This commit is contained in:
parent
b63e9aa45c
commit
5355d0398d
GPG Key ID:
41A00BF15EA7E5F3
235
Cargo.lock
generated
235
Cargo.lock
generated
@ -14,21 +14,12 @@ version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"cipher",
|
||||
"cpufeatures",
|
||||
"opaque-debug",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.7.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anes"
|
||||
version = "0.1.6"
|
||||
@ -52,12 +43,6 @@ version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
|
||||
|
||||
[[package]]
|
||||
name = "base64ct"
|
||||
version = "1.0.1"
|
||||
@ -83,12 +68,6 @@ dependencies = [
|
||||
"virtue",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
@ -152,12 +131,6 @@ dependencies = [
|
||||
"jobserver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "0.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
@ -172,9 +145,6 @@ dependencies = [
|
||||
"chgk_ledb_lib",
|
||||
"clap",
|
||||
"criterion",
|
||||
"ledb",
|
||||
"ledb-derive",
|
||||
"ledb-types",
|
||||
"rand",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
@ -188,9 +158,6 @@ name = "chgk_ledb_lib"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"ledb",
|
||||
"ledb-derive",
|
||||
"ledb-types",
|
||||
"memmap",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
@ -243,7 +210,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"bitflags 1.3.2",
|
||||
"bitflags",
|
||||
"clap_derive",
|
||||
"clap_lex",
|
||||
"indexmap",
|
||||
@ -296,7 +263,7 @@ version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -341,7 +308,7 @@ version = "0.5.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
@ -351,7 +318,7 @@ version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
@ -363,7 +330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"memoffset",
|
||||
"scopeguard",
|
||||
@ -375,7 +342,7 @@ version = "0.8.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -399,33 +366,6 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs"
|
||||
version = "2.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
"dirs-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"redox_users",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dunce"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0bd4b30a6560bbd9b4620f4de34c3f14f60848e58a9b7216801afcb4c7b31c3c"
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.8.1"
|
||||
@ -472,12 +412,6 @@ dependencies = [
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gcc"
|
||||
version = "0.3.55"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2"
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.6"
|
||||
@ -494,7 +428,7 @@ version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
@ -560,7 +494,7 @@ version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -612,94 +546,27 @@ version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "ledb"
|
||||
version = "0.4.0"
|
||||
source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"dirs",
|
||||
"dunce",
|
||||
"ledb-derive",
|
||||
"ledb-types",
|
||||
"lmdb-zero",
|
||||
"ordered-float",
|
||||
"regex",
|
||||
"ron",
|
||||
"serde",
|
||||
"serde_cbor",
|
||||
"supercow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ledb-derive"
|
||||
version = "0.4.0"
|
||||
source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ledb-types"
|
||||
version = "0.4.0"
|
||||
source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_cbor",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.139"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
|
||||
|
||||
[[package]]
|
||||
name = "liblmdb-sys"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "feed38a3a580f60bf61aaa067b0ff4123395966839adeaf67258a9e50c4d2e49"
|
||||
dependencies = [
|
||||
"gcc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
|
||||
|
||||
[[package]]
|
||||
name = "lmdb-zero"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13416eee745b087c22934f35f1f24da22da41ba2a5ce197143d168ce055cc58d"
|
||||
dependencies = [
|
||||
"bitflags 0.9.1",
|
||||
"libc",
|
||||
"liblmdb-sys",
|
||||
"supercow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
||||
|
||||
[[package]]
|
||||
name = "memmap"
|
||||
version = "0.7.0"
|
||||
@ -765,16 +632,6 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
|
||||
|
||||
[[package]]
|
||||
name = "ordered-float"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "os_str_bytes"
|
||||
version = "6.4.1"
|
||||
@ -944,18 +801,7 @@ version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"redox_syscall",
|
||||
"thiserror",
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -964,8 +810,6 @@ version = "1.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
@ -975,24 +819,13 @@ version = "0.6.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
|
||||
|
||||
[[package]]
|
||||
name = "ron"
|
||||
version = "0.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86018df177b1beef6c7c8ef949969c4f7cb9a9344181b92486b23c79995bdaa4"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bitflags 1.3.2",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.36.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"bitflags",
|
||||
"errno",
|
||||
"io-lifetimes",
|
||||
"libc",
|
||||
@ -1030,16 +863,6 @@ dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_cbor"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
|
||||
dependencies = [
|
||||
"half",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.152"
|
||||
@ -1068,7 +891,7 @@ version = "0.10.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
@ -1079,7 +902,7 @@ version = "0.10.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
@ -1096,12 +919,6 @@ version = "2.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
|
||||
|
||||
[[package]]
|
||||
name = "supercow"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "171758edb47aa306a78dfa4ab9aeb5167405bd4e3dc2b64e88f6a84bbe98bd63"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
@ -1119,7 +936,7 @@ version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"fastrand",
|
||||
"redox_syscall",
|
||||
"rustix",
|
||||
@ -1141,26 +958,6 @@ version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.38"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "1.0.38"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.20"
|
||||
@ -1245,7 +1042,7 @@ version = "0.2.84"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"cfg-if",
|
||||
"wasm-bindgen-macro",
|
||||
]
|
||||
|
||||
|
||||
@ -13,7 +13,6 @@ harness = false
|
||||
[dependencies]
|
||||
chgk_ledb_lib = {path = "../lib"}
|
||||
serde_json="1.0"
|
||||
ledb = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb"}
|
||||
zip="0.6"
|
||||
rand="0.8"
|
||||
clap = { version = "3.2.22", features = ["derive"] }
|
||||
@ -22,8 +21,5 @@ clap = { version = "3.2.22", features = ["derive"] }
|
||||
criterion = "0.4.0"
|
||||
tempfile = "3.3"
|
||||
bincode = "^2.0.0-rc.2"
|
||||
ledb = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb"}
|
||||
ledb-derive = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-derive"}
|
||||
ledb-types = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-types"}
|
||||
serde="1.0"
|
||||
serde_derive="1.0"
|
||||
|
||||
@ -1,13 +1,10 @@
|
||||
#[macro_use]
|
||||
extern crate criterion;
|
||||
extern crate bincode;
|
||||
extern crate ledb;
|
||||
extern crate ledb_types;
|
||||
extern crate serde;
|
||||
extern crate serde_derive;
|
||||
extern crate tempfile;
|
||||
#[macro_use]
|
||||
extern crate serde_json;
|
||||
extern crate tempfile;
|
||||
|
||||
use chgk_ledb_lib::db;
|
||||
use std::path::PathBuf;
|
||||
@ -17,7 +14,6 @@ use db::{Reader, Writer, WriterOpts};
|
||||
use criterion::{BatchSize, Criterion};
|
||||
use tempfile::tempdir;
|
||||
|
||||
use ledb::{Document, Options, Storage};
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
|
||||
#[derive(
|
||||
@ -31,10 +27,8 @@ use serde_derive::{Deserialize, Serialize};
|
||||
Ord,
|
||||
Serialize,
|
||||
Deserialize,
|
||||
Document,
|
||||
)]
|
||||
struct TestData {
|
||||
#[document(primary)]
|
||||
num1: u64,
|
||||
num2: u64,
|
||||
test: String,
|
||||
@ -48,7 +42,7 @@ fn gen_data(count: usize) -> impl Iterator<Item = TestData> {
|
||||
.map(|i| 143 + i as u64)
|
||||
.map(|i| TestData {
|
||||
num1: i,
|
||||
num2: i*100 ^ 0xDF0E441122334455,
|
||||
num2: i * 100 ^ 0xDF0E441122334455,
|
||||
test: "test ---- Test ____".repeat(123 + i as usize % 15),
|
||||
})
|
||||
}
|
||||
@ -108,83 +102,9 @@ fn db_write(c: &mut Criterion) {
|
||||
});
|
||||
}
|
||||
|
||||
fn ledb_write(c: &mut Criterion) {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
let tmp_dir = dir.as_ref();
|
||||
c.bench_function("ledb_write", |b| {
|
||||
b.iter_batched(
|
||||
|| {
|
||||
let src = gen_data(N).collect::<Vec<TestData>>().into_iter();
|
||||
let options: Options = serde_json::from_value(json!({
|
||||
"map_size": 100 * 1024 * 1024, // 100mb
|
||||
"write_map": true,
|
||||
"map_async": true,
|
||||
"no_lock": true,
|
||||
"no_meta_sync": true,
|
||||
"no_sync": true,
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let storage = Storage::new(tmp_dir, options).unwrap();
|
||||
let collection = storage.collection("test").unwrap();
|
||||
(src, collection)
|
||||
},
|
||||
|(src, collection)| collection.load(src).expect("load"),
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
}
|
||||
|
||||
fn ledb_read(c: &mut Criterion) {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
let tmp_dir = dir.as_ref();
|
||||
|
||||
let write_options: Options = serde_json::from_value(json!({
|
||||
"map_size": 100 * 1024 * 1024, // 100mb
|
||||
"write_map": true,
|
||||
"map_async": true,
|
||||
"no_lock": true,
|
||||
"no_meta_sync": true,
|
||||
"no_sync": true,
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let storage = Storage::new(&tmp_dir, write_options).unwrap();
|
||||
let collection = storage.collection("test").unwrap();
|
||||
let items_iter = gen_data(N).collect::<Vec<TestData>>().into_iter();
|
||||
collection.load(items_iter).expect("load");
|
||||
drop(collection);
|
||||
drop(storage);
|
||||
|
||||
c.bench_function("ledb_read", |b| {
|
||||
b.iter_batched(
|
||||
|| {
|
||||
let options: Options = serde_json::from_value(json!({
|
||||
"read_only": true,
|
||||
"map_async": true,
|
||||
"no_lock": true,
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let storage = Storage::new(tmp_dir, options).unwrap();
|
||||
let collection = storage.collection("test").unwrap();
|
||||
collection
|
||||
},
|
||||
|collection| {
|
||||
let mut collection_iter = collection.dump::<TestData>().expect("dump");
|
||||
while let Some(item) = collection_iter.next() {
|
||||
drop(item);
|
||||
}
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
}
|
||||
|
||||
fn config() -> Criterion {
|
||||
Criterion::default().sample_size(40)
|
||||
}
|
||||
|
||||
criterion_group! {name=ledb; config = config(); targets = ledb_read, ledb_write}
|
||||
criterion_group! {name=benches; config = config(); targets = db_read, db_write}
|
||||
criterion_main!(benches, ledb);
|
||||
criterion_main!(benches);
|
||||
|
||||
116
app/src/main.rs
116
app/src/main.rs
@ -1,15 +1,11 @@
|
||||
#[macro_use]
|
||||
extern crate serde_json;
|
||||
use clap::{Parser, Subcommand};
|
||||
use rand::seq::IteratorRandom;
|
||||
|
||||
use std::io;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Instant;
|
||||
use std::{fs, sync::mpsc, thread};
|
||||
|
||||
use ledb::{Options, Storage};
|
||||
|
||||
use chgk_ledb_lib::db;
|
||||
use chgk_ledb_lib::questions;
|
||||
use chgk_ledb_lib::source;
|
||||
@ -19,12 +15,10 @@ use crate::source::ReadSourceQuestionsBatches;
|
||||
|
||||
const ZIP_FILENAME: &str = "json.zip";
|
||||
const NEW_DB_FILENAME: &str = "db.dat";
|
||||
const DB_DIR: &str = "db";
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum Command {
|
||||
Write,
|
||||
Compact,
|
||||
Print {
|
||||
#[clap(value_parser, default_value = "0")]
|
||||
id: u32,
|
||||
@ -35,11 +29,6 @@ enum Command {
|
||||
#[clap(value_parser, default_value = "0")]
|
||||
num: usize,
|
||||
},
|
||||
Write2,
|
||||
Print2 {
|
||||
#[clap(value_parser, default_value = "0")]
|
||||
id: u32,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
@ -73,60 +62,12 @@ fn zip_reader_task(tx: mpsc::Sender<Question>) {
|
||||
}
|
||||
println!("read done");
|
||||
}
|
||||
fn db_writer_task(rx: mpsc::Receiver<Question>) {
|
||||
let out_file: PathBuf = [DB_DIR, "data.mdb"].into_iter().collect();
|
||||
match fs::metadata(&out_file) {
|
||||
Ok(x) if x.is_file() => {
|
||||
fs::remove_file(&out_file).unwrap();
|
||||
println!(r#""{}" removed"#, out_file.to_str().unwrap());
|
||||
}
|
||||
_ => {}
|
||||
};
|
||||
|
||||
let options: Options = serde_json::from_value(json!({
|
||||
"map_size": 900 * 1024 * 1024, // 900mb
|
||||
"write_map": true,
|
||||
"map_async": true,
|
||||
"no_lock": true,
|
||||
"no_meta_sync": true,
|
||||
"no_sync": true,
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let storage = Storage::new(DB_DIR, options).unwrap();
|
||||
let collection = storage.collection("questions").unwrap();
|
||||
|
||||
let count = collection.load(rx).expect("load");
|
||||
|
||||
println!("loaded {count}");
|
||||
|
||||
println!("syncing to disk...");
|
||||
storage.sync(true).unwrap();
|
||||
|
||||
print!("stats: ");
|
||||
let stats = storage.stat().unwrap();
|
||||
println!("{:?}", stats);
|
||||
|
||||
drop(storage);
|
||||
println!("write done");
|
||||
}
|
||||
|
||||
fn write_db() {
|
||||
let (tx, rx) = mpsc::channel::<Question>();
|
||||
[
|
||||
thread::spawn(move || zip_reader_task(tx)),
|
||||
thread::spawn(move || db_writer_task(rx)),
|
||||
]
|
||||
.into_iter()
|
||||
.for_each(|handle| handle.join().expect("thread panic"));
|
||||
println!("all done");
|
||||
}
|
||||
|
||||
fn print_question_from<F>(get_q: F)
|
||||
where
|
||||
F: FnOnce() -> Option<Question>,
|
||||
{
|
||||
let q = get_q().unwrap();
|
||||
let q = get_q().expect("question not found");
|
||||
println!("{:#?}", q)
|
||||
}
|
||||
|
||||
@ -151,45 +92,6 @@ fn read_from_zip(file_num: usize, mut num: usize) -> Option<Question> {
|
||||
Some(questions[num - 1].clone())
|
||||
}
|
||||
|
||||
fn compact_db() {
|
||||
let options: Options = serde_json::from_value(json!({
|
||||
"write_map": true,
|
||||
"map_async": true,
|
||||
"no_lock": true,
|
||||
"no_meta_sync": true,
|
||||
"no_sync": true,
|
||||
"compact": true,
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let storage = Storage::new(DB_DIR, options).unwrap();
|
||||
|
||||
storage.sync(true).unwrap();
|
||||
let stats = storage.stat().unwrap();
|
||||
println!("{:?}", stats);
|
||||
drop(storage);
|
||||
}
|
||||
|
||||
fn read_from_db(mut id: u32) -> Option<Question> {
|
||||
let options: Options = serde_json::from_value(json!({
|
||||
"read_only": true,
|
||||
"map_async": true,
|
||||
"no_lock": true,
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let storage = Storage::new(DB_DIR, options).unwrap();
|
||||
let collection = storage.collection("questions").unwrap();
|
||||
let mut rng = rand::thread_rng();
|
||||
|
||||
if id == 0 {
|
||||
let last_id = collection.last_id().unwrap();
|
||||
id = (1..=last_id).choose(&mut rng).unwrap();
|
||||
}
|
||||
|
||||
collection.get::<Question>(id).unwrap()
|
||||
}
|
||||
|
||||
// measure and return time elapsed in `func` in seconds
|
||||
pub fn measure<F: FnOnce()>(func: F) -> f64 {
|
||||
let start = Instant::now();
|
||||
@ -208,7 +110,6 @@ fn main() {
|
||||
|
||||
let mut action: Box<dyn FnOnce()> = match &args.command {
|
||||
Command::Write => Box::new(write_db),
|
||||
Command::Compact => Box::new(compact_db),
|
||||
Command::Print { id } => {
|
||||
let get_question = Box::new(|| read_from_db(*id));
|
||||
Box::new(|| print_question_from(get_question))
|
||||
@ -217,11 +118,6 @@ fn main() {
|
||||
let get_question = Box::new(|| read_from_zip(*file_num, *num));
|
||||
Box::new(|| print_question_from(get_question))
|
||||
}
|
||||
Command::Write2 => Box::new(write_db2),
|
||||
Command::Print2 { id } => {
|
||||
let get_question = Box::new(|| read_from_db2(*id));
|
||||
Box::new(|| print_question_from(get_question))
|
||||
}
|
||||
};
|
||||
|
||||
if args.measure {
|
||||
@ -231,11 +127,11 @@ fn main() {
|
||||
action();
|
||||
}
|
||||
|
||||
fn read_from_db2(id: u32) -> Option<Question> {
|
||||
fn read_from_db(id: u32) -> Option<Question> {
|
||||
let reader: db::Reader<Question> =
|
||||
db::Reader::new(NEW_DB_FILENAME, 2048).expect("new db reader");
|
||||
|
||||
let mut questions = reader.iter();
|
||||
let mut questions = reader.into_iter();
|
||||
|
||||
match id {
|
||||
0 => {
|
||||
@ -245,17 +141,17 @@ fn read_from_db2(id: u32) -> Option<Question> {
|
||||
_ => questions.nth((id - 1) as usize),
|
||||
}
|
||||
}
|
||||
fn write_db2() {
|
||||
fn write_db() {
|
||||
let (tx, rx) = mpsc::channel::<Question>();
|
||||
[
|
||||
thread::spawn(move || zip_reader_task(tx)),
|
||||
thread::spawn(move || db_writer2_task(rx)),
|
||||
thread::spawn(move || db_writer_task(rx)),
|
||||
]
|
||||
.into_iter()
|
||||
.for_each(|handle| handle.join().expect("thread panic"));
|
||||
println!("all done");
|
||||
}
|
||||
fn db_writer2_task(rx: mpsc::Receiver<Question>) {
|
||||
fn db_writer_task(rx: mpsc::Receiver<Question>) {
|
||||
let writer_opts = db::WriterOpts::default();
|
||||
let mut writer: db::Writer<Question> =
|
||||
db::Writer::new(NEW_DB_FILENAME, writer_opts).expect("new db writer");
|
||||
|
||||
@ -10,9 +10,6 @@ edition = "2021"
|
||||
serde="1.0"
|
||||
serde_derive="1.0"
|
||||
serde_json="1.0"
|
||||
ledb = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb"}
|
||||
ledb-derive = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-derive"}
|
||||
ledb-types = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-types"}
|
||||
zip="0.6"
|
||||
bincode = "^2.0.0-rc.2"
|
||||
zstd = "^0.10"
|
||||
|
||||
@ -3,7 +3,7 @@ use std::{
|
||||
io::{self, Cursor, Read, Write},
|
||||
marker::PhantomData,
|
||||
path::Path,
|
||||
sync::Arc
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use memmap::{Mmap, MmapOptions};
|
||||
@ -158,7 +158,7 @@ where
|
||||
mmap: Mmap,
|
||||
count: usize,
|
||||
first_pos: LSize,
|
||||
_t: Option<Arc<T>> // PhantomData replacement
|
||||
_t: Option<Arc<T>>, // PhantomData replacement
|
||||
}
|
||||
|
||||
impl<T> Reader<T>
|
||||
@ -179,7 +179,7 @@ where
|
||||
mmap,
|
||||
count,
|
||||
first_pos,
|
||||
_t: None
|
||||
_t: None,
|
||||
})
|
||||
}
|
||||
|
||||
@ -226,10 +226,6 @@ where
|
||||
pub fn iter(&self) -> ReaderIter<'_, T> {
|
||||
ReaderIter::new(self)
|
||||
}
|
||||
|
||||
pub fn into_iter(self) -> ReaderIntoIter<T> {
|
||||
ReaderIntoIter::new(self)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ReaderIter<'a, T>
|
||||
@ -396,7 +392,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> IntoIterator for Reader<T>
|
||||
impl<T> IntoIterator for Reader<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
{
|
||||
@ -404,7 +400,7 @@ where
|
||||
type IntoIter = ReaderIntoIter<Self::Item>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
self.into_iter()
|
||||
Self::IntoIter::new(self)
|
||||
}
|
||||
}
|
||||
|
||||
@ -534,9 +530,12 @@ mod test {
|
||||
let cur_items = items.clone();
|
||||
let cur_reader = Arc::clone(&reader);
|
||||
thread::spawn(move || {
|
||||
cur_items.into_iter().zip(cur_reader.iter()).for_each(|pair| {
|
||||
assert_eq!(pair.0, pair.1);
|
||||
});
|
||||
cur_items
|
||||
.into_iter()
|
||||
.zip(cur_reader.iter())
|
||||
.for_each(|pair| {
|
||||
assert_eq!(pair.0, pair.1);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
use ledb::Document;
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
|
||||
use crate::source::{SourceQuestion, SourceQuestionsBatch};
|
||||
@ -12,11 +11,8 @@ macro_rules! make {
|
||||
),+ ,..$Target::default()}}
|
||||
}
|
||||
|
||||
#[derive(
|
||||
Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode,
|
||||
)]
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, bincode::Decode, bincode::Encode)]
|
||||
pub struct BatchInfo {
|
||||
#[document(primary)]
|
||||
#[serde(default)]
|
||||
pub filename: String,
|
||||
#[serde(default)]
|
||||
@ -45,14 +41,10 @@ pub struct BatchInfo {
|
||||
pub rating: String,
|
||||
}
|
||||
|
||||
#[derive(
|
||||
Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode,
|
||||
)]
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, bincode::Decode, bincode::Encode)]
|
||||
pub struct Question {
|
||||
#[document(primary)]
|
||||
#[serde(default)]
|
||||
pub num: u32,
|
||||
#[document(index)]
|
||||
pub id: String,
|
||||
|
||||
pub description: String,
|
||||
@ -84,7 +76,6 @@ pub struct Question {
|
||||
pub source: String,
|
||||
#[serde(default)]
|
||||
pub rating: String,
|
||||
#[document(nested)]
|
||||
#[serde(default)]
|
||||
pub batch_info: BatchInfo,
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user