diff --git a/Cargo.lock b/Cargo.lock index 4f3033d..7e057c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,21 +14,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cipher", "cpufeatures", "opaque-debug", ] -[[package]] -name = "aho-corasick" -version = "0.7.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" -dependencies = [ - "memchr", -] - [[package]] name = "anes" version = "0.1.6" @@ -52,12 +43,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64ct" version = "1.0.1" @@ -83,12 +68,6 @@ dependencies = [ "virtue", ] -[[package]] -name = "bitflags" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" - [[package]] name = "bitflags" version = "1.3.2" @@ -152,12 +131,6 @@ dependencies = [ "jobserver", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.0" @@ -172,9 +145,6 @@ dependencies = [ "chgk_ledb_lib", "clap", "criterion", - "ledb", - "ledb-derive", - "ledb-types", "rand", "serde", "serde_derive", @@ -188,9 +158,6 @@ name = "chgk_ledb_lib" version = "0.1.0" dependencies = [ "bincode", - "ledb", - "ledb-derive", - "ledb-types", "memmap", "serde", "serde_derive", @@ -243,7 +210,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "atty", - "bitflags 1.3.2", + "bitflags", "clap_derive", "clap_lex", "indexmap", @@ -296,7 +263,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -341,7 +308,7 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", ] @@ -351,7 +318,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] @@ -363,7 +330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg", - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", "memoffset", "scopeguard", @@ -375,7 +342,7 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -399,33 +366,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dirs" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" -dependencies = [ - "cfg-if 0.1.10", - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - -[[package]] -name = "dunce" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd4b30a6560bbd9b4620f4de34c3f14f60848e58a9b7216801afcb4c7b31c3c" - [[package]] name = "either" version = "1.8.1" @@ -472,12 +412,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "gcc" -version = "0.3.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" - [[package]] name = "generic-array" version = "0.14.6" @@ -494,7 +428,7 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi", ] @@ -560,7 +494,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -612,94 +546,27 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "ledb" -version = "0.4.0" -source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b" -dependencies = [ - "byteorder", - "dirs", - "dunce", - "ledb-derive", - "ledb-types", - "lmdb-zero", - "ordered-float", - "regex", - "ron", - "serde", - "serde_cbor", - "supercow", -] - -[[package]] -name = "ledb-derive" -version = "0.4.0" -source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "ledb-types" -version = "0.4.0" -source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b" -dependencies = [ - "serde", - "serde_cbor", - "serde_json", -] - [[package]] name = "libc" version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" -[[package]] -name = "liblmdb-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feed38a3a580f60bf61aaa067b0ff4123395966839adeaf67258a9e50c4d2e49" -dependencies = [ - "gcc", - "libc", -] - [[package]] name = "linux-raw-sys" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" -[[package]] -name = "lmdb-zero" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13416eee745b087c22934f35f1f24da22da41ba2a5ce197143d168ce055cc58d" -dependencies = [ - "bitflags 0.9.1", - "libc", - "liblmdb-sys", - "supercow", -] - [[package]] name = "log" version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] -[[package]] -name = "memchr" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - [[package]] name = "memmap" version = "0.7.0" @@ -765,16 +632,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" -[[package]] -name = "ordered-float" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" -dependencies = [ - "num-traits", - "serde", -] - [[package]] name = "os_str_bytes" version = "6.4.1" @@ -944,18 +801,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom", - "redox_syscall", - "thiserror", + "bitflags", ] [[package]] @@ -964,8 +810,6 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ - "aho-corasick", - "memchr", "regex-syntax", ] @@ -975,24 +819,13 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" -[[package]] -name = "ron" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86018df177b1beef6c7c8ef949969c4f7cb9a9344181b92486b23c79995bdaa4" -dependencies = [ - "base64", - "bitflags 1.3.2", - "serde", -] - [[package]] name = "rustix" version = "0.36.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" dependencies = [ - "bitflags 1.3.2", + "bitflags", "errno", "io-lifetimes", "libc", @@ -1030,16 +863,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_cbor" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" -dependencies = [ - "half", - "serde", -] - [[package]] name = "serde_derive" version = "1.0.152" @@ -1068,7 +891,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest", ] @@ -1079,7 +902,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest", ] @@ -1096,12 +919,6 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" -[[package]] -name = "supercow" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171758edb47aa306a78dfa4ab9aeb5167405bd4e3dc2b64e88f6a84bbe98bd63" - [[package]] name = "syn" version = "1.0.109" @@ -1119,7 +936,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "fastrand", "redox_syscall", "rustix", @@ -1141,26 +958,6 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" -[[package]] -name = "thiserror" -version = "1.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "time" version = "0.3.20" @@ -1245,7 +1042,7 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen-macro", ] diff --git a/app/Cargo.toml b/app/Cargo.toml index 80f1d1c..c41f4fb 100644 --- a/app/Cargo.toml +++ b/app/Cargo.toml @@ -13,7 +13,6 @@ harness = false [dependencies] chgk_ledb_lib = {path = "../lib"} serde_json="1.0" -ledb = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb"} zip="0.6" rand="0.8" clap = { version = "3.2.22", features = ["derive"] } @@ -22,8 +21,5 @@ clap = { version = "3.2.22", features = ["derive"] } criterion = "0.4.0" tempfile = "3.3" bincode = "^2.0.0-rc.2" -ledb = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb"} -ledb-derive = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-derive"} -ledb-types = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-types"} serde="1.0" serde_derive="1.0" diff --git a/app/benches/db_bench.rs b/app/benches/db_bench.rs index 87a3035..ca0f06c 100644 --- a/app/benches/db_bench.rs +++ b/app/benches/db_bench.rs @@ -1,13 +1,10 @@ #[macro_use] extern crate criterion; extern crate bincode; -extern crate ledb; -extern crate ledb_types; extern crate serde; extern crate serde_derive; -extern crate tempfile; -#[macro_use] extern crate serde_json; +extern crate tempfile; use chgk_ledb_lib::db; use std::path::PathBuf; @@ -17,7 +14,6 @@ use db::{Reader, Writer, WriterOpts}; use criterion::{BatchSize, Criterion}; use tempfile::tempdir; -use ledb::{Document, Options, Storage}; use serde_derive::{Deserialize, Serialize}; #[derive( @@ -31,10 +27,8 @@ use serde_derive::{Deserialize, Serialize}; Ord, Serialize, Deserialize, - Document, )] struct TestData { - #[document(primary)] num1: u64, num2: u64, test: String, @@ -48,7 +42,7 @@ fn gen_data(count: usize) -> impl Iterator { .map(|i| 143 + i as u64) .map(|i| TestData { num1: i, - num2: i*100 ^ 0xDF0E441122334455, + num2: i * 100 ^ 0xDF0E441122334455, test: "test ---- Test ____".repeat(123 + i as usize % 15), }) } @@ -108,83 +102,9 @@ fn db_write(c: &mut Criterion) { }); } -fn ledb_write(c: &mut Criterion) { - let dir = tempdir().expect("tempdir"); - let tmp_dir = dir.as_ref(); - c.bench_function("ledb_write", |b| { - b.iter_batched( - || { - let src = gen_data(N).collect::>().into_iter(); - let options: Options = serde_json::from_value(json!({ - "map_size": 100 * 1024 * 1024, // 100mb - "write_map": true, - "map_async": true, - "no_lock": true, - "no_meta_sync": true, - "no_sync": true, - })) - .unwrap(); - - let storage = Storage::new(tmp_dir, options).unwrap(); - let collection = storage.collection("test").unwrap(); - (src, collection) - }, - |(src, collection)| collection.load(src).expect("load"), - BatchSize::SmallInput, - ) - }); -} - -fn ledb_read(c: &mut Criterion) { - let dir = tempdir().expect("tempdir"); - let tmp_dir = dir.as_ref(); - - let write_options: Options = serde_json::from_value(json!({ - "map_size": 100 * 1024 * 1024, // 100mb - "write_map": true, - "map_async": true, - "no_lock": true, - "no_meta_sync": true, - "no_sync": true, - })) - .unwrap(); - - let storage = Storage::new(&tmp_dir, write_options).unwrap(); - let collection = storage.collection("test").unwrap(); - let items_iter = gen_data(N).collect::>().into_iter(); - collection.load(items_iter).expect("load"); - drop(collection); - drop(storage); - - c.bench_function("ledb_read", |b| { - b.iter_batched( - || { - let options: Options = serde_json::from_value(json!({ - "read_only": true, - "map_async": true, - "no_lock": true, - })) - .unwrap(); - - let storage = Storage::new(tmp_dir, options).unwrap(); - let collection = storage.collection("test").unwrap(); - collection - }, - |collection| { - let mut collection_iter = collection.dump::().expect("dump"); - while let Some(item) = collection_iter.next() { - drop(item); - } - }, - BatchSize::SmallInput, - ) - }); -} - fn config() -> Criterion { Criterion::default().sample_size(40) } -criterion_group! {name=ledb; config = config(); targets = ledb_read, ledb_write} criterion_group! {name=benches; config = config(); targets = db_read, db_write} -criterion_main!(benches, ledb); +criterion_main!(benches); diff --git a/app/src/main.rs b/app/src/main.rs index 52bd56c..fbcb2e7 100644 --- a/app/src/main.rs +++ b/app/src/main.rs @@ -1,15 +1,11 @@ -#[macro_use] extern crate serde_json; use clap::{Parser, Subcommand}; use rand::seq::IteratorRandom; use std::io; -use std::path::PathBuf; use std::time::Instant; use std::{fs, sync::mpsc, thread}; -use ledb::{Options, Storage}; - use chgk_ledb_lib::db; use chgk_ledb_lib::questions; use chgk_ledb_lib::source; @@ -19,12 +15,10 @@ use crate::source::ReadSourceQuestionsBatches; const ZIP_FILENAME: &str = "json.zip"; const NEW_DB_FILENAME: &str = "db.dat"; -const DB_DIR: &str = "db"; #[derive(Subcommand, Debug)] enum Command { Write, - Compact, Print { #[clap(value_parser, default_value = "0")] id: u32, @@ -35,11 +29,6 @@ enum Command { #[clap(value_parser, default_value = "0")] num: usize, }, - Write2, - Print2 { - #[clap(value_parser, default_value = "0")] - id: u32, - }, } #[derive(Parser, Debug)] @@ -73,60 +62,12 @@ fn zip_reader_task(tx: mpsc::Sender) { } println!("read done"); } -fn db_writer_task(rx: mpsc::Receiver) { - let out_file: PathBuf = [DB_DIR, "data.mdb"].into_iter().collect(); - match fs::metadata(&out_file) { - Ok(x) if x.is_file() => { - fs::remove_file(&out_file).unwrap(); - println!(r#""{}" removed"#, out_file.to_str().unwrap()); - } - _ => {} - }; - - let options: Options = serde_json::from_value(json!({ - "map_size": 900 * 1024 * 1024, // 900mb - "write_map": true, - "map_async": true, - "no_lock": true, - "no_meta_sync": true, - "no_sync": true, - })) - .unwrap(); - - let storage = Storage::new(DB_DIR, options).unwrap(); - let collection = storage.collection("questions").unwrap(); - - let count = collection.load(rx).expect("load"); - - println!("loaded {count}"); - - println!("syncing to disk..."); - storage.sync(true).unwrap(); - - print!("stats: "); - let stats = storage.stat().unwrap(); - println!("{:?}", stats); - - drop(storage); - println!("write done"); -} - -fn write_db() { - let (tx, rx) = mpsc::channel::(); - [ - thread::spawn(move || zip_reader_task(tx)), - thread::spawn(move || db_writer_task(rx)), - ] - .into_iter() - .for_each(|handle| handle.join().expect("thread panic")); - println!("all done"); -} fn print_question_from(get_q: F) where F: FnOnce() -> Option, { - let q = get_q().unwrap(); + let q = get_q().expect("question not found"); println!("{:#?}", q) } @@ -151,45 +92,6 @@ fn read_from_zip(file_num: usize, mut num: usize) -> Option { Some(questions[num - 1].clone()) } -fn compact_db() { - let options: Options = serde_json::from_value(json!({ - "write_map": true, - "map_async": true, - "no_lock": true, - "no_meta_sync": true, - "no_sync": true, - "compact": true, - })) - .unwrap(); - - let storage = Storage::new(DB_DIR, options).unwrap(); - - storage.sync(true).unwrap(); - let stats = storage.stat().unwrap(); - println!("{:?}", stats); - drop(storage); -} - -fn read_from_db(mut id: u32) -> Option { - let options: Options = serde_json::from_value(json!({ - "read_only": true, - "map_async": true, - "no_lock": true, - })) - .unwrap(); - - let storage = Storage::new(DB_DIR, options).unwrap(); - let collection = storage.collection("questions").unwrap(); - let mut rng = rand::thread_rng(); - - if id == 0 { - let last_id = collection.last_id().unwrap(); - id = (1..=last_id).choose(&mut rng).unwrap(); - } - - collection.get::(id).unwrap() -} - // measure and return time elapsed in `func` in seconds pub fn measure(func: F) -> f64 { let start = Instant::now(); @@ -208,7 +110,6 @@ fn main() { let mut action: Box = match &args.command { Command::Write => Box::new(write_db), - Command::Compact => Box::new(compact_db), Command::Print { id } => { let get_question = Box::new(|| read_from_db(*id)); Box::new(|| print_question_from(get_question)) @@ -217,11 +118,6 @@ fn main() { let get_question = Box::new(|| read_from_zip(*file_num, *num)); Box::new(|| print_question_from(get_question)) } - Command::Write2 => Box::new(write_db2), - Command::Print2 { id } => { - let get_question = Box::new(|| read_from_db2(*id)); - Box::new(|| print_question_from(get_question)) - } }; if args.measure { @@ -231,11 +127,11 @@ fn main() { action(); } -fn read_from_db2(id: u32) -> Option { +fn read_from_db(id: u32) -> Option { let reader: db::Reader = db::Reader::new(NEW_DB_FILENAME, 2048).expect("new db reader"); - let mut questions = reader.iter(); + let mut questions = reader.into_iter(); match id { 0 => { @@ -245,17 +141,17 @@ fn read_from_db2(id: u32) -> Option { _ => questions.nth((id - 1) as usize), } } -fn write_db2() { +fn write_db() { let (tx, rx) = mpsc::channel::(); [ thread::spawn(move || zip_reader_task(tx)), - thread::spawn(move || db_writer2_task(rx)), + thread::spawn(move || db_writer_task(rx)), ] .into_iter() .for_each(|handle| handle.join().expect("thread panic")); println!("all done"); } -fn db_writer2_task(rx: mpsc::Receiver) { +fn db_writer_task(rx: mpsc::Receiver) { let writer_opts = db::WriterOpts::default(); let mut writer: db::Writer = db::Writer::new(NEW_DB_FILENAME, writer_opts).expect("new db writer"); diff --git a/lib/Cargo.toml b/lib/Cargo.toml index a67d49a..1440ab7 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -10,9 +10,6 @@ edition = "2021" serde="1.0" serde_derive="1.0" serde_json="1.0" -ledb = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb"} -ledb-derive = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-derive"} -ledb-types = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-types"} zip="0.6" bincode = "^2.0.0-rc.2" zstd = "^0.10" diff --git a/lib/src/db.rs b/lib/src/db.rs index cdb98c8..2336be0 100644 --- a/lib/src/db.rs +++ b/lib/src/db.rs @@ -3,7 +3,7 @@ use std::{ io::{self, Cursor, Read, Write}, marker::PhantomData, path::Path, - sync::Arc + sync::Arc, }; use memmap::{Mmap, MmapOptions}; @@ -158,7 +158,7 @@ where mmap: Mmap, count: usize, first_pos: LSize, - _t: Option> // PhantomData replacement + _t: Option>, // PhantomData replacement } impl Reader @@ -179,7 +179,7 @@ where mmap, count, first_pos, - _t: None + _t: None, }) } @@ -226,10 +226,6 @@ where pub fn iter(&self) -> ReaderIter<'_, T> { ReaderIter::new(self) } - - pub fn into_iter(self) -> ReaderIntoIter { - ReaderIntoIter::new(self) - } } pub struct ReaderIter<'a, T> @@ -396,7 +392,7 @@ where } } -impl IntoIterator for Reader +impl IntoIterator for Reader where T: bincode::Decode, { @@ -404,7 +400,7 @@ where type IntoIter = ReaderIntoIter; fn into_iter(self) -> Self::IntoIter { - self.into_iter() + Self::IntoIter::new(self) } } @@ -534,9 +530,12 @@ mod test { let cur_items = items.clone(); let cur_reader = Arc::clone(&reader); thread::spawn(move || { - cur_items.into_iter().zip(cur_reader.iter()).for_each(|pair| { - assert_eq!(pair.0, pair.1); - }); + cur_items + .into_iter() + .zip(cur_reader.iter()) + .for_each(|pair| { + assert_eq!(pair.0, pair.1); + }); }); } } diff --git a/lib/src/questions.rs b/lib/src/questions.rs index b0853fc..6fd27f7 100644 --- a/lib/src/questions.rs +++ b/lib/src/questions.rs @@ -1,4 +1,3 @@ -use ledb::Document; use serde_derive::{Deserialize, Serialize}; use crate::source::{SourceQuestion, SourceQuestionsBatch}; @@ -12,11 +11,8 @@ macro_rules! make { ),+ ,..$Target::default()}} } -#[derive( - Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode, -)] +#[derive(Debug, Default, Clone, Serialize, Deserialize, bincode::Decode, bincode::Encode)] pub struct BatchInfo { - #[document(primary)] #[serde(default)] pub filename: String, #[serde(default)] @@ -45,14 +41,10 @@ pub struct BatchInfo { pub rating: String, } -#[derive( - Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode, -)] +#[derive(Debug, Default, Clone, Serialize, Deserialize, bincode::Decode, bincode::Encode)] pub struct Question { - #[document(primary)] #[serde(default)] pub num: u32, - #[document(index)] pub id: String, pub description: String, @@ -84,7 +76,6 @@ pub struct Question { pub source: String, #[serde(default)] pub rating: String, - #[document(nested)] #[serde(default)] pub batch_info: BatchInfo, }