remove ledb

Cargo.lock

@@ -14,21 +14,12 @@ version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "cipher",
  "cpufeatures",
  "opaque-debug",
 ]
 
-[[package]]
-name = "aho-corasick"
-version = "0.7.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "anes"
 version = "0.1.6"
@@ -52,12 +43,6 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
-[[package]]
-name = "base64"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
-
 [[package]]
 name = "base64ct"
 version = "1.0.1"
@@ -83,12 +68,6 @@ dependencies = [
  "virtue",
 ]
 
-[[package]]
-name = "bitflags"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5"
-
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -152,12 +131,6 @@ dependencies = [
  "jobserver",
 ]
 
-[[package]]
-name = "cfg-if"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
-
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -172,9 +145,6 @@ dependencies = [
  "chgk_ledb_lib",
  "clap",
  "criterion",
- "ledb",
- "ledb-derive",
- "ledb-types",
  "rand",
  "serde",
  "serde_derive",
@@ -188,9 +158,6 @@ name = "chgk_ledb_lib"
 version = "0.1.0"
 dependencies = [
  "bincode",
- "ledb",
- "ledb-derive",
- "ledb-types",
  "memmap",
  "serde",
  "serde_derive",
@@ -243,7 +210,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
 dependencies = [
  "atty",
- "bitflags 1.3.2",
+ "bitflags",
  "clap_derive",
  "clap_lex",
  "indexmap",
@@ -296,7 +263,7 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -341,7 +308,7 @@ version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "crossbeam-utils",
 ]
 
@@ -351,7 +318,7 @@ version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "crossbeam-epoch",
  "crossbeam-utils",
 ]
@@ -363,7 +330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
 dependencies = [
  "autocfg",
- "cfg-if 1.0.0",
+ "cfg-if",
  "crossbeam-utils",
  "memoffset",
  "scopeguard",
@@ -375,7 +342,7 @@ version = "0.8.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -399,33 +366,6 @@ dependencies = [
  "subtle",
 ]
 
-[[package]]
-name = "dirs"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3"
-dependencies = [
- "cfg-if 0.1.10",
- "dirs-sys",
-]
-
-[[package]]
-name = "dirs-sys"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
-dependencies = [
- "libc",
- "redox_users",
- "winapi",
-]
-
-[[package]]
-name = "dunce"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bd4b30a6560bbd9b4620f4de34c3f14f60848e58a9b7216801afcb4c7b31c3c"
-
 [[package]]
 name = "either"
 version = "1.8.1"
@@ -472,12 +412,6 @@ dependencies = [
  "miniz_oxide",
 ]
 
-[[package]]
-name = "gcc"
-version = "0.3.55"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2"
-
 [[package]]
 name = "generic-array"
 version = "0.14.6"
@@ -494,7 +428,7 @@ version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "wasi",
 ]
@@ -560,7 +494,7 @@ version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -612,94 +546,27 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
-[[package]]
-name = "ledb"
-version = "0.4.0"
-source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b"
-dependencies = [
- "byteorder",
- "dirs",
- "dunce",
- "ledb-derive",
- "ledb-types",
- "lmdb-zero",
- "ordered-float",
- "regex",
- "ron",
- "serde",
- "serde_cbor",
- "supercow",
-]
-
-[[package]]
-name = "ledb-derive"
-version = "0.4.0"
-source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "ledb-types"
-version = "0.4.0"
-source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b"
-dependencies = [
- "serde",
- "serde_cbor",
- "serde_json",
-]
-
 [[package]]
 name = "libc"
 version = "0.2.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
 
-[[package]]
-name = "liblmdb-sys"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "feed38a3a580f60bf61aaa067b0ff4123395966839adeaf67258a9e50c4d2e49"
-dependencies = [
- "gcc",
- "libc",
-]
-
 [[package]]
 name = "linux-raw-sys"
 version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
 
-[[package]]
-name = "lmdb-zero"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13416eee745b087c22934f35f1f24da22da41ba2a5ce197143d168ce055cc58d"
-dependencies = [
- "bitflags 0.9.1",
- "libc",
- "liblmdb-sys",
- "supercow",
-]
-
 [[package]]
 name = "log"
 version = "0.4.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
-[[package]]
-name = "memchr"
-version = "2.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
-
 [[package]]
 name = "memmap"
 version = "0.7.0"
@@ -765,16 +632,6 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
-[[package]]
-name = "ordered-float"
-version = "1.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7"
-dependencies = [
- "num-traits",
- "serde",
-]
-
 [[package]]
 name = "os_str_bytes"
 version = "6.4.1"
@@ -944,18 +801,7 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
-]
-
-[[package]]
-name = "redox_users"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
-dependencies = [
- "getrandom",
- "redox_syscall",
- "thiserror",
 ]
 
 [[package]]
@@ -964,8 +810,6 @@ version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
 dependencies = [
- "aho-corasick",
- "memchr",
  "regex-syntax",
 ]
 
@@ -975,24 +819,13 @@ version = "0.6.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
 
-[[package]]
-name = "ron"
-version = "0.6.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86018df177b1beef6c7c8ef949969c4f7cb9a9344181b92486b23c79995bdaa4"
-dependencies = [
- "base64",
- "bitflags 1.3.2",
- "serde",
-]
-
 [[package]]
 name = "rustix"
 version = "0.36.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
  "errno",
  "io-lifetimes",
  "libc",
@@ -1030,16 +863,6 @@ dependencies = [
  "serde_derive",
 ]
 
-[[package]]
-name = "serde_cbor"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
-dependencies = [
- "half",
- "serde",
-]
-
 [[package]]
 name = "serde_derive"
 version = "1.0.152"
@@ -1068,7 +891,7 @@ version = "0.10.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "cpufeatures",
  "digest",
 ]
@@ -1079,7 +902,7 @@ version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "cpufeatures",
  "digest",
 ]
@@ -1096,12 +919,6 @@ version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
-[[package]]
-name = "supercow"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "171758edb47aa306a78dfa4ab9aeb5167405bd4e3dc2b64e88f6a84bbe98bd63"
-
 [[package]]
 name = "syn"
 version = "1.0.109"
@@ -1119,7 +936,7 @@ version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "fastrand",
  "redox_syscall",
  "rustix",
@@ -1141,26 +958,6 @@ version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
 
-[[package]]
-name = "thiserror"
-version = "1.0.38"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "1.0.38"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "time"
 version = "0.3.20"
@@ -1245,7 +1042,7 @@ version = "0.2.84"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "wasm-bindgen-macro",
 ]
 

app/Cargo.toml

@@ -13,7 +13,6 @@ harness = false
 [dependencies]
 chgk_ledb_lib = {path = "../lib"}
 serde_json="1.0"
-ledb = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb"}
 zip="0.6"
 rand="0.8"
 clap = { version = "3.2.22", features = ["derive"] }
@@ -22,8 +21,5 @@ clap = { version = "3.2.22", features = ["derive"] }
 criterion = "0.4.0"
 tempfile = "3.3"
 bincode = "^2.0.0-rc.2"
-ledb = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb"}
-ledb-derive = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-derive"}
-ledb-types = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-types"}
 serde="1.0"
 serde_derive="1.0"

app/benches/db_bench.rs

@@ -1,13 +1,10 @@
 #[macro_use]
 extern crate criterion;
 extern crate bincode;
-extern crate ledb;
-extern crate ledb_types;
 extern crate serde;
 extern crate serde_derive;
-extern crate tempfile;
-#[macro_use]
 extern crate serde_json;
+extern crate tempfile;
 
 use chgk_ledb_lib::db;
 use std::path::PathBuf;
@@ -17,7 +14,6 @@ use db::{Reader, Writer, WriterOpts};
 use criterion::{BatchSize, Criterion};
 use tempfile::tempdir;
 
-use ledb::{Document, Options, Storage};
 use serde_derive::{Deserialize, Serialize};
 
 #[derive(
@@ -31,10 +27,8 @@ use serde_derive::{Deserialize, Serialize};
     Ord,
     Serialize,
     Deserialize,
-    Document,
 )]
 struct TestData {
-    #[document(primary)]
     num1: u64,
     num2: u64,
     test: String,
@@ -48,7 +42,7 @@ fn gen_data(count: usize) -> impl Iterator<Item = TestData> {
         .map(|i| 143 + i as u64)
         .map(|i| TestData {
             num1: i,
-            num2: i*100 ^ 0xDF0E441122334455,
+            num2: i * 100 ^ 0xDF0E441122334455,
             test: "test ---- Test ____".repeat(123 + i as usize % 15),
         })
 }
@@ -108,83 +102,9 @@ fn db_write(c: &mut Criterion) {
     });
 }
 
-fn ledb_write(c: &mut Criterion) {
-    let dir = tempdir().expect("tempdir");
-    let tmp_dir = dir.as_ref();
-    c.bench_function("ledb_write", |b| {
-        b.iter_batched(
-            || {
-                let src = gen_data(N).collect::<Vec<TestData>>().into_iter();
-                let options: Options = serde_json::from_value(json!({
-                    "map_size": 100 * 1024 * 1024, // 100mb
-                    "write_map": true,
-                    "map_async": true,
-                    "no_lock": true,
-                    "no_meta_sync": true,
-                    "no_sync": true,
-                }))
-                .unwrap();
-
-                let storage = Storage::new(tmp_dir, options).unwrap();
-                let collection = storage.collection("test").unwrap();
-                (src, collection)
-            },
-            |(src, collection)| collection.load(src).expect("load"),
-            BatchSize::SmallInput,
-        )
-    });
-}
-
-fn ledb_read(c: &mut Criterion) {
-    let dir = tempdir().expect("tempdir");
-    let tmp_dir = dir.as_ref();
-
-    let write_options: Options = serde_json::from_value(json!({
-        "map_size": 100 * 1024 * 1024, // 100mb
-        "write_map": true,
-        "map_async": true,
-        "no_lock": true,
-        "no_meta_sync": true,
-        "no_sync": true,
-    }))
-    .unwrap();
-
-    let storage = Storage::new(&tmp_dir, write_options).unwrap();
-    let collection = storage.collection("test").unwrap();
-    let items_iter = gen_data(N).collect::<Vec<TestData>>().into_iter();
-    collection.load(items_iter).expect("load");
-    drop(collection);
-    drop(storage);
-
-    c.bench_function("ledb_read", |b| {
-        b.iter_batched(
-            || {
-                let options: Options = serde_json::from_value(json!({
-                    "read_only": true,
-                    "map_async": true,
-                    "no_lock": true,
-                }))
-                .unwrap();
-
-                let storage = Storage::new(tmp_dir, options).unwrap();
-                let collection = storage.collection("test").unwrap();
-                collection
-            },
-            |collection| {
-                let mut collection_iter = collection.dump::<TestData>().expect("dump");
-                while let Some(item) = collection_iter.next() {
-                    drop(item);
-                }
-            },
-            BatchSize::SmallInput,
-        )
-    });
-}
-
 fn config() -> Criterion {
     Criterion::default().sample_size(40)
 }
 
-criterion_group! {name=ledb; config = config(); targets = ledb_read, ledb_write}
 criterion_group! {name=benches; config = config(); targets = db_read, db_write}
-criterion_main!(benches, ledb);
+criterion_main!(benches);

app/src/main.rs

@@ -1,15 +1,11 @@
-#[macro_use]
 extern crate serde_json;
 use clap::{Parser, Subcommand};
 use rand::seq::IteratorRandom;
 
 use std::io;
-use std::path::PathBuf;
 use std::time::Instant;
 use std::{fs, sync::mpsc, thread};
 
-use ledb::{Options, Storage};
-
 use chgk_ledb_lib::db;
 use chgk_ledb_lib::questions;
 use chgk_ledb_lib::source;
@@ -19,12 +15,10 @@ use crate::source::ReadSourceQuestionsBatches;
 
 const ZIP_FILENAME: &str = "json.zip";
 const NEW_DB_FILENAME: &str = "db.dat";
-const DB_DIR: &str = "db";
 
 #[derive(Subcommand, Debug)]
 enum Command {
     Write,
-    Compact,
     Print {
         #[clap(value_parser, default_value = "0")]
         id: u32,
@@ -35,11 +29,6 @@ enum Command {
         #[clap(value_parser, default_value = "0")]
         num: usize,
     },
-    Write2,
-    Print2 {
-        #[clap(value_parser, default_value = "0")]
-        id: u32,
-    },
 }
 
 #[derive(Parser, Debug)]
@@ -73,60 +62,12 @@ fn zip_reader_task(tx: mpsc::Sender<Question>) {
     }
     println!("read done");
 }
-fn db_writer_task(rx: mpsc::Receiver<Question>) {
-    let out_file: PathBuf = [DB_DIR, "data.mdb"].into_iter().collect();
-    match fs::metadata(&out_file) {
-        Ok(x) if x.is_file() => {
-            fs::remove_file(&out_file).unwrap();
-            println!(r#""{}" removed"#, out_file.to_str().unwrap());
-        }
-        _ => {}
-    };
-
-    let options: Options = serde_json::from_value(json!({
-        "map_size": 900 * 1024 * 1024, // 900mb
-        "write_map": true,
-        "map_async": true,
-        "no_lock": true,
-        "no_meta_sync": true,
-        "no_sync": true,
-    }))
-    .unwrap();
-
-    let storage = Storage::new(DB_DIR, options).unwrap();
-    let collection = storage.collection("questions").unwrap();
-
-    let count = collection.load(rx).expect("load");
-
-    println!("loaded {count}");
-
-    println!("syncing to disk...");
-    storage.sync(true).unwrap();
-
-    print!("stats: ");
-    let stats = storage.stat().unwrap();
-    println!("{:?}", stats);
-
-    drop(storage);
-    println!("write done");
-}
-
-fn write_db() {
-    let (tx, rx) = mpsc::channel::<Question>();
-    [
-        thread::spawn(move || zip_reader_task(tx)),
-        thread::spawn(move || db_writer_task(rx)),
-    ]
-    .into_iter()
-    .for_each(|handle| handle.join().expect("thread panic"));
-    println!("all done");
-}
 
 fn print_question_from<F>(get_q: F)
 where
     F: FnOnce() -> Option<Question>,
 {
-    let q = get_q().unwrap();
+    let q = get_q().expect("question not found");
     println!("{:#?}", q)
 }
 
@@ -151,45 +92,6 @@ fn read_from_zip(file_num: usize, mut num: usize) -> Option<Question> {
     Some(questions[num - 1].clone())
 }
 
-fn compact_db() {
-    let options: Options = serde_json::from_value(json!({
-        "write_map": true,
-        "map_async": true,
-        "no_lock": true,
-        "no_meta_sync": true,
-        "no_sync": true,
-        "compact": true,
-    }))
-    .unwrap();
-
-    let storage = Storage::new(DB_DIR, options).unwrap();
-
-    storage.sync(true).unwrap();
-    let stats = storage.stat().unwrap();
-    println!("{:?}", stats);
-    drop(storage);
-}
-
-fn read_from_db(mut id: u32) -> Option<Question> {
-    let options: Options = serde_json::from_value(json!({
-        "read_only": true,
-        "map_async": true,
-        "no_lock": true,
-    }))
-    .unwrap();
-
-    let storage = Storage::new(DB_DIR, options).unwrap();
-    let collection = storage.collection("questions").unwrap();
-    let mut rng = rand::thread_rng();
-
-    if id == 0 {
-        let last_id = collection.last_id().unwrap();
-        id = (1..=last_id).choose(&mut rng).unwrap();
-    }
-
-    collection.get::<Question>(id).unwrap()
-}
-
 // measure and return time elapsed in `func` in seconds
 pub fn measure<F: FnOnce()>(func: F) -> f64 {
     let start = Instant::now();
@@ -208,7 +110,6 @@ fn main() {
 
     let mut action: Box<dyn FnOnce()> = match &args.command {
         Command::Write => Box::new(write_db),
-        Command::Compact => Box::new(compact_db),
         Command::Print { id } => {
             let get_question = Box::new(|| read_from_db(*id));
             Box::new(|| print_question_from(get_question))
@@ -217,11 +118,6 @@ fn main() {
             let get_question = Box::new(|| read_from_zip(*file_num, *num));
             Box::new(|| print_question_from(get_question))
         }
-        Command::Write2 => Box::new(write_db2),
-        Command::Print2 { id } => {
-            let get_question = Box::new(|| read_from_db2(*id));
-            Box::new(|| print_question_from(get_question))
-        }
     };
 
     if args.measure {
@@ -231,11 +127,11 @@ fn main() {
     action();
 }
 
-fn read_from_db2(id: u32) -> Option<Question> {
+fn read_from_db(id: u32) -> Option<Question> {
     let reader: db::Reader<Question> =
         db::Reader::new(NEW_DB_FILENAME, 2048).expect("new db reader");
 
-    let mut questions = reader.iter();
+    let mut questions = reader.into_iter();
 
     match id {
         0 => {
@@ -245,17 +141,17 @@ fn read_from_db2(id: u32) -> Option<Question> {
         _ => questions.nth((id - 1) as usize),
     }
 }
-fn write_db2() {
+fn write_db() {
     let (tx, rx) = mpsc::channel::<Question>();
     [
         thread::spawn(move || zip_reader_task(tx)),
-        thread::spawn(move || db_writer2_task(rx)),
+        thread::spawn(move || db_writer_task(rx)),
     ]
     .into_iter()
     .for_each(|handle| handle.join().expect("thread panic"));
     println!("all done");
 }
-fn db_writer2_task(rx: mpsc::Receiver<Question>) {
+fn db_writer_task(rx: mpsc::Receiver<Question>) {
     let writer_opts = db::WriterOpts::default();
     let mut writer: db::Writer<Question> =
         db::Writer::new(NEW_DB_FILENAME, writer_opts).expect("new db writer");

lib/Cargo.toml

@@ -10,9 +10,6 @@ edition = "2021"
 serde="1.0"
 serde_derive="1.0"
 serde_json="1.0"
-ledb = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb"}
-ledb-derive = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-derive"}
-ledb-types = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-types"}
 zip="0.6"
 bincode = "^2.0.0-rc.2"
 zstd = "^0.10"

lib/src/db.rs

@@ -3,7 +3,7 @@ use std::{
     io::{self, Cursor, Read, Write},
     marker::PhantomData,
     path::Path,
-    sync::Arc
+    sync::Arc,
 };
 
 use memmap::{Mmap, MmapOptions};
@@ -158,7 +158,7 @@ where
     mmap: Mmap,
     count: usize,
     first_pos: LSize,
-    _t: Option<Arc<T>> // PhantomData replacement
+    _t: Option<Arc<T>>, // PhantomData replacement
 }
 
 impl<T> Reader<T>
@@ -179,7 +179,7 @@ where
             mmap,
             count,
             first_pos,
-            _t: None
+            _t: None,
         })
     }
 
@@ -226,10 +226,6 @@ where
     pub fn iter(&self) -> ReaderIter<'_, T> {
         ReaderIter::new(self)
     }
-
-    pub fn into_iter(self) -> ReaderIntoIter<T> {
-        ReaderIntoIter::new(self)
-    }
 }
 
 pub struct ReaderIter<'a, T>
@@ -404,7 +400,7 @@ where
     type IntoIter = ReaderIntoIter<Self::Item>;
 
     fn into_iter(self) -> Self::IntoIter {
-        self.into_iter()
+        Self::IntoIter::new(self)
     }
 }
 
@@ -534,7 +530,10 @@ mod test {
             let cur_items = items.clone();
             let cur_reader = Arc::clone(&reader);
             thread::spawn(move || {
-                cur_items.into_iter().zip(cur_reader.iter()).for_each(|pair| {
+                cur_items
+                    .into_iter()
+                    .zip(cur_reader.iter())
+                    .for_each(|pair| {
                         assert_eq!(pair.0, pair.1);
                     });
             });

lib/src/questions.rs

@@ -1,4 +1,3 @@
-use ledb::Document;
 use serde_derive::{Deserialize, Serialize};
 
 use crate::source::{SourceQuestion, SourceQuestionsBatch};
@@ -12,11 +11,8 @@ macro_rules! make {
     ),+ ,..$Target::default()}}
 }
 
-#[derive(
+#[derive(Debug, Default, Clone, Serialize, Deserialize, bincode::Decode, bincode::Encode)]
-    Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode,
-)]
 pub struct BatchInfo {
-    #[document(primary)]
     #[serde(default)]
     pub filename: String,
     #[serde(default)]
@@ -45,14 +41,10 @@ pub struct BatchInfo {
     pub rating: String,
 }
 
-#[derive(
+#[derive(Debug, Default, Clone, Serialize, Deserialize, bincode::Decode, bincode::Encode)]
-    Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode,
-)]
 pub struct Question {
-    #[document(primary)]
     #[serde(default)]
     pub num: u32,
-    #[document(index)]
     pub id: String,
 
     pub description: String,
@@ -84,7 +76,6 @@ pub struct Question {
     pub source: String,
     #[serde(default)]
     pub rating: String,
-    #[document(nested)]
     #[serde(default)]
     pub batch_info: BatchInfo,
 }