db::read -- io::Cursor instead of take

with fixed collection.load()

.drone.yml

@@ -0,0 +1,11 @@
+kind: pipeline
+name: default
+
+steps:
+- name: test
+  image: rust:1-alpine
+  commands:
+  - apk add --no-cache musl-dev
+  - cargo build --verbose --all
+  - cargo test --verbose --all
+

.gitignore

@@ -5,3 +5,4 @@ test?.zip
 json.zip
 /exp
 /.vscode
+test*.bin

Cargo.lock

@@ -29,6 +29,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -58,6 +64,25 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a32fd6af2b5827bce66c29053ba0e7c42b9dcab01835835058558c10851a46b"
 
+[[package]]
+name = "bincode"
+version = "2.0.0-rc.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bb50c5a2ef4b9b1e7ae73e3a73b52ea24b20312d629f9c4df28260b7ad2c3c4"
+dependencies = [
+ "bincode_derive",
+ "serde",
+]
+
+[[package]]
+name = "bincode_derive"
+version = "2.0.0-rc.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a45a23389446d2dd25dc8e73a7a3b3c43522b630cac068927f0649d43d719d2"
+dependencies = [
+ "virtue",
+]
+
 [[package]]
 name = "bitflags"
 version = "0.9.1"
@@ -79,6 +104,12 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "bumpalo"
+version = "3.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d"
+
 [[package]]
 name = "byteorder"
 version = "1.4.3"
@@ -106,6 +137,12 @@ dependencies = [
  "pkg-config",
 ]
 
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "cc"
 version = "1.0.73"
@@ -131,15 +168,47 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 name = "chgk_ledb"
 version = "0.1.0"
 dependencies = [
+ "bincode",
  "clap",
+ "criterion",
  "ledb",
  "ledb-derive",
  "ledb-types",
+ "memmap",
  "rand",
  "serde",
  "serde_derive",
  "serde_json",
+ "tempfile",
  "zip",
+ "zstd",
+]
+
+[[package]]
+name = "ciborium"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b"
+dependencies = [
+ "ciborium-io",
+ "half",
 ]
 
 [[package]]
@@ -214,6 +283,76 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb"
+dependencies = [
+ "anes",
+ "atty",
+ "cast",
+ "ciborium",
+ "clap",
+ "criterion-plot",
+ "itertools",
+ "lazy_static",
+ "num-traits",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348"
+dependencies = [
+ "autocfg",
+ "cfg-if 1.0.0",
+ "crossbeam-utils",
+ "memoffset",
+ "scopeguard",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.11"
@@ -272,6 +411,21 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "453440c271cf5577fd2a40e4942540cb7d0d2f85e27c8d07dd0023c925a67541"
 
+[[package]]
+name = "either"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
+
+[[package]]
+name = "fastrand"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499"
+dependencies = [
+ "instant",
+]
+
 [[package]]
 name = "flate2"
 version = "1.0.24"
@@ -355,6 +509,24 @@ dependencies = [
  "hashbrown",
 ]
 
+[[package]]
+name = "instant"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
+dependencies = [
+ "cfg-if 1.0.0",
+]
+
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.3"
@@ -370,11 +542,25 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "js-sys"
+version = "0.3.60"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
+dependencies = [
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
 [[package]]
 name = "ledb"
 version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b"
-checksum = "035e7ab9b9b6de6b16bf9ddd6ef4ab764161c2024e53fa11f592368610128114"
 dependencies = [
  "byteorder",
  "dirs",
@@ -393,8 +579,7 @@ dependencies = [
 [[package]]
 name = "ledb-derive"
 version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b"
-checksum = "01e3e32cca609ef18496c4300c59263964ba8ceb89d7fc5307e1cb9a645484de"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -404,8 +589,7 @@ dependencies = [
 [[package]]
 name = "ledb-types"
 version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b"
-checksum = "20d1de30724b365b84e65498d91197524c67a8c0919f045bb6af9c471b2d048e"
 dependencies = [
  "serde",
  "serde_cbor",
@@ -440,12 +624,40 @@ dependencies = [
  "supercow",
 ]
 
+[[package]]
+name = "log"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
+dependencies = [
+ "cfg-if 1.0.0",
+]
+
 [[package]]
 name = "memchr"
 version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
+[[package]]
+name = "memmap"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "memoffset"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.5.4"
@@ -464,6 +676,16 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "num_cpus"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
 [[package]]
 name = "num_threads"
 version = "0.1.6"
@@ -479,6 +701,12 @@ version = "1.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0"
 
+[[package]]
+name = "oorandom"
+version = "11.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
+
 [[package]]
 name = "opaque-debug"
 version = "0.3.0"
@@ -530,6 +758,34 @@ version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
 
+[[package]]
+name = "plotters"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.16"
@@ -608,6 +864,30 @@ dependencies = [
  "getrandom",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d"
+dependencies = [
+ "autocfg",
+ "crossbeam-deque",
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "num_cpus",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"
@@ -645,6 +925,15 @@ version = "0.6.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
 
+[[package]]
+name = "remove_dir_all"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "ron"
 version = "0.6.6"
@@ -662,6 +951,21 @@ version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
 
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+
 [[package]]
 name = "serde"
 version = "1.0.144"
@@ -754,6 +1058,20 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "tempfile"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
+dependencies = [
+ "cfg-if 1.0.0",
+ "fastrand",
+ "libc",
+ "redox_syscall",
+ "remove_dir_all",
+ "winapi",
+]
+
 [[package]]
 name = "termcolor"
 version = "1.1.3"
@@ -807,6 +1125,16 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792"
 
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "typenum"
 version = "1.15.0"
@@ -825,12 +1153,93 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
+[[package]]
+name = "virtue"
+version = "0.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b60dcd6a64dd45abf9bd426970c9843726da7fc08f44cd6fcebf68c21220a63"
+
+[[package]]
+name = "walkdir"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
+dependencies = [
+ "same-file",
+ "winapi",
+ "winapi-util",
+]
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.83"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
+dependencies = [
+ "cfg-if 1.0.0",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.83"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
+dependencies = [
+ "bumpalo",
+ "log",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.83"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.83"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.83"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
+
+[[package]]
+name = "web-sys"
+version = "0.3.60"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "winapi"
 version = "0.3.9"

Cargo.toml

@@ -6,16 +6,27 @@ edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[[bench]]
+name = "db_bench"
+harness = false
+
 [dependencies]
 serde="1.0"
 serde_derive="1.0"
 serde_json="1.0"
-ledb="0.4"
+ledb = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb"}
-ledb-derive="0.4"
+ledb-derive = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-derive"}
-ledb-types="0.4"
+ledb-types = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-types"}
 zip="0.6"
 rand="0.8"
 clap = { version = "3.2.22", features = ["derive"] }
+bincode = "^2.0.0-rc.2"
+zstd = "^0.10"
+memmap = "0.7.0"
+
+[dev-dependencies]
+criterion = "0.4.0"
+tempfile = "3.3"
 
 [profile.release]
 opt-level = 3

README.md

@@ -4,12 +4,58 @@
 
 Исходный файл вопросов: `json.zip`, кодировка `UTF-8`.
 
-Выходная база: файл `./db/data.mdb`, формат базы [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database).
-
-Работа с базой выполняется с помощью [ledb](https://crates.io/crates/ledb).
-
 При загрузке базы информация о пакете(файле/турнире) дублируется в каждом вопросе.
 
+## Выходной формат данных
+
+Для хранения данных используется [bincode](https://crates.io/crates/bincode) и [zstd](https://crates.io/crates/zstd).
+Данные вопросов в виде структуры сериализуются через `bincode` в бинарные данные и сжимаются `zstd`. Каждый вопрос сериализуется и сжимается отдельно.
+В файле сжатые данные храняться последовательно, после заголовка файла.
+
+### Заголовок файла
+
+В заголовке хранятся только смещения каждого сжатого блока данных, и дополнительно, смещение указывающее на конец файла.
+Смещение указывается в виде 32-битного беззнакового целового числа `u32`, сохраненного в виде 4-х байтов от младшего к старшему ([Little Endian](https://ru.wikipedia.org/wiki/Порядок_байтов#Порядок_от_младшего_к_старшему)).
+
+Пример заголовка:
+
+~~~
+00:  10 00 00 00
+04:  1A 00 00 00
+08:  2E 00 00 00
+0А:  3A 00 00 00
+~~~
+
+данные:
+
+~~~
+10:  00 00 00 00 00 00 00 00 00 00
+1A:  00 00 00 00 00 00 00 00 00 00
+     00 00 00 00 00 00 00 00 00 00
+2E:  00 00 00 00 00 00 00 00 00 00
+     00 00
+3A:  (EOF)
+~~~
+
+В этом примере сохранены 3 записи:
+
+1. смещение **0x10**, длина **10** байт
+2. смещение **0x1A**, длина **20** байт
+3. смещение **0x2E**, длина **12** байт
+
+Размер файла - 58 байт (0x3A), размер заголовка 16 байт (0x10).
+
+### Чтение данных
+
+Пусть размер записи заголовка в байтах = **`M`**, количество записей в файле = **`N`**.
+
+Тогда для того чтобы:
+
+- Найти **`N`**, нужно прочитать первую запись (**`M`** байт) в начале файла и разделить её значение на **`M`**;
+- Найти элемент данных с индексом **`i`**, нужно последовательно прочитать 2 записи (**`M * 2`** байт), начиная c индекса **`i`** (по смещению в файле: **`i * M`**). Смещением будет значение по индексу **`i`**, длинной - разница между значениями по индексам **`i + 1`** и **`i`**.
+
+Далее для каждого вопроса отдельно предполагается распаковка данных через `zstd` и десериализация через `bincode`.
+
 ## Ссылки
 
 - Источник вопросов: http://db.chgk.info

bench.txt

@@ -0,0 +1,23 @@
+866 MB db/data.mdb
+232 MB test.bin
+ 95 MB json.zip
+---
+
+hyperfine  -n print -n print2 -n zip-print -w 400 -m 400 ".\target\release\chgk_ledb.exe print 444" ".\target\release\chgk_ledb.exe print2 444" ".\target\release\chgk_ledb.exe zip-print 4 84"
+
+Benchmark 1: print
+  Time (mean ± σ):      19.0 ms ±   1.5 ms    [User: 5.6 ms, System: 13.1 ms]
+  Range (min … max):    16.8 ms …  24.5 ms    400 runs
+
+Benchmark 2: print2
+  Time (mean ± σ):      18.6 ms ±   1.6 ms    [User: 5.5 ms, System: 12.6 ms]
+  Range (min … max):    16.1 ms …  29.5 ms    400 runs
+
+Benchmark 3: zip-print
+  Time (mean ± σ):      40.8 ms ±   3.3 ms    [User: 15.4 ms, System: 21.6 ms]
+  Range (min … max):    36.5 ms …  67.5 ms    400 runs
+
+Summary
+  'print2' ran
+    1.02 ± 0.12 times faster than 'print'
+    2.20 ± 0.26 times faster than 'zip-print'

benches/db_bench.rs

@@ -0,0 +1,92 @@
+#[macro_use]
+extern crate criterion;
+
+#[path = "../src/db.rs"]
+mod db;
+use std::path::PathBuf;
+
+use db::{Reader, Writer, WriterOpts};
+
+use criterion::{BatchSize, Criterion};
+use tempfile::tempdir;
+
+#[derive(bincode::Encode, bincode::Decode, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+struct TestData {
+    num: u128,
+    test: String,
+}
+
+const N: usize = 4096;
+
+fn gen_data(count: usize) -> impl Iterator<Item = TestData> {
+    (0..count)
+        .into_iter()
+        .map(|i| 143 + i as u128)
+        .map(|i| TestData {
+            num: i,
+            test: "test ---- Test ____".repeat(123 + i as usize % 15),
+        })
+}
+
+fn prepare_db_writer(path: &PathBuf) -> Writer<TestData> {
+    let opts = WriterOpts {
+        compress_lvl: 1,
+        data_buf_size: 100 * 1024 * 1024,
+        out_buf_size: 100 * 1024 * 1024,
+        current_buf_size: 10240,
+    };
+
+    Writer::new(path, opts).expect("new writer")
+}
+
+fn db_read(c: &mut Criterion) {
+    let dir = tempdir().expect("tempdir");
+    let tmpfile = dir.path().join("test.tmp");
+    let mut writer = prepare_db_writer(&tmpfile);
+
+    let mut items_iter = gen_data(N).collect::<Vec<TestData>>().into_iter();
+    writer.load(&mut items_iter).unwrap();
+    writer.finish().unwrap();
+
+    c.bench_function("read", |b| {
+        b.iter_batched(
+            || {
+                let reader: Reader<TestData> = Reader::new(&tmpfile, 2048).expect("new reader");
+                reader
+            },
+            |mut reader| {
+                let mut reader_iter = reader.iter();
+                while let Some(item) = reader_iter.next() {
+                    drop(item);
+                }
+            },
+            BatchSize::SmallInput,
+        )
+    });
+}
+
+fn db_write(c: &mut Criterion) {
+    let dir = tempdir().expect("tempdir");
+    let tmpfile = dir.path().join("test.tmp");
+    c.bench_function("write", |b| {
+        b.iter_batched(
+            || {
+                let src = gen_data(N).collect::<Vec<TestData>>().into_iter();
+                let writer = prepare_db_writer(&tmpfile);
+                (src, writer)
+            },
+            |(mut src, mut writer)| {
+                writer.load(&mut src).unwrap();
+                writer.finish().unwrap();
+            },
+            BatchSize::SmallInput,
+        )
+    });
+}
+
+fn config() -> Criterion {
+    Criterion::default().sample_size(40)
+}
+
+criterion_group! {name=benches; config = config(); targets = db_read, db_write}
+criterion_main!(benches);

src/db.rs

@@ -0,0 +1,378 @@
+use std::{
+    fs,
+    io::{self, Cursor, Read, Write},
+    marker::PhantomData,
+    path::Path,
+};
+
+use memmap::{Mmap, MmapOptions};
+
+type LSize = u32;
+const LEN_SIZE: usize = std::mem::size_of::<LSize>();
+const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard();
+
+trait ErrorToString {
+    type Output;
+    fn str_err(self) -> std::result::Result<Self::Output, String>;
+}
+
+impl<T, E> ErrorToString for std::result::Result<T, E>
+where
+    E: std::error::Error,
+{
+    type Output = T;
+    fn str_err(self) -> std::result::Result<Self::Output, String> {
+        self.map_err(|e| e.to_string())
+    }
+}
+
+pub struct WriterOpts {
+    pub compress_lvl: i32,
+    pub data_buf_size: usize,
+    pub out_buf_size: usize,
+    pub current_buf_size: usize,
+}
+
+impl Default for WriterOpts {
+    fn default() -> Self {
+        Self {
+            compress_lvl: 1,
+            data_buf_size: 500 * 1024 * 1024,
+            out_buf_size: 200 * 1024 * 1024,
+            current_buf_size: 100 * 1024,
+        }
+    }
+}
+
+pub struct Writer<T>
+where
+    T: bincode::Encode,
+{
+    out: io::BufWriter<fs::File>,
+    data_buf: Cursor<Vec<u8>>,
+    cur_buf_raw: Cursor<Vec<u8>>,
+    table: Vec<LSize>,
+    compress_lvl: i32,
+    _t: PhantomData<*const T>,
+}
+
+impl<T> Writer<T>
+where
+    T: bincode::Encode,
+{
+    pub fn new<P: AsRef<Path>>(path: P, opts: WriterOpts) -> Result<Self, String> {
+        let out = fs::File::create(path).str_err()?;
+        let out = io::BufWriter::with_capacity(opts.out_buf_size, out);
+        let data_buf: Vec<u8> = Vec::with_capacity(opts.data_buf_size);
+        let data_buf = Cursor::new(data_buf);
+
+        let cur_buf_raw: Vec<u8> = Vec::with_capacity(opts.current_buf_size);
+        let cur_buf_raw = Cursor::new(cur_buf_raw);
+
+        let compress_lvl = opts.compress_lvl;
+
+        let table: Vec<LSize> = vec![];
+
+        Ok(Self {
+            out,
+            data_buf,
+            cur_buf_raw,
+            table,
+            compress_lvl,
+            _t: PhantomData,
+        })
+    }
+
+    pub fn push(&mut self, item: T) -> Result<(), String> {
+        let pos: LSize = self.data_buf.position() as LSize;
+
+        let item_data = bincode::encode_to_vec(item, BINCODE_CFG).str_err()?;
+
+        let mut zencoder = zstd::stream::raw::Encoder::new(self.compress_lvl).str_err()?;
+        zencoder
+            .set_pledged_src_size(item_data.len() as u64)
+            .str_err()?;
+
+        self.cur_buf_raw.set_position(0);
+        let mut cur_buf_z = zstd::stream::zio::Writer::new(&mut self.cur_buf_raw, zencoder);
+        cur_buf_z.write_all(&item_data).str_err()?;
+        cur_buf_z.finish().str_err()?;
+        cur_buf_z.flush().str_err()?;
+
+        self.table.push(pos);
+        let (cur_buf_raw, _) = cur_buf_z.into_inner();
+        let size = cur_buf_raw.position();
+
+        cur_buf_raw.set_position(0);
+        let mut chunk = cur_buf_raw.take(size);
+        io::copy(&mut chunk, &mut self.data_buf).str_err()?;
+
+        Ok(())
+    }
+
+    pub fn load<I>(&mut self, iter: &mut I) -> Result<(), String>
+    where
+        I: Iterator<Item = T>,
+    {
+        let hint = iter.size_hint();
+        let hint = std::cmp::max(hint.0, hint.1.unwrap_or(0));
+        if hint > 0 {
+            self.table.reserve(hint);
+        }
+
+        for item in iter {
+            self.push(item)?;
+        }
+
+        Ok(())
+    }
+
+    pub fn finish(mut self) -> Result<(), String> {
+        // finish tab
+        let pos: LSize = self.data_buf.position() as LSize;
+        self.table.push(pos);
+
+        // write tab
+        let tab_size = (self.table.len() * LEN_SIZE) as LSize;
+        for pos in self.table {
+            let pos_data = (pos + tab_size).to_le_bytes();
+            self.out.write_all(&pos_data).str_err()?;
+        }
+
+        // copy data
+        let data_size = self.data_buf.position();
+        self.data_buf.set_position(0);
+        let mut data = self.data_buf.take(data_size);
+        io::copy(&mut data, &mut self.out).str_err()?;
+
+        self.out.flush().str_err()?;
+        Ok(())
+    }
+}
+
+pub struct Reader<T>
+where
+    T: bincode::Decode,
+{
+    mmap: Mmap,
+    count: usize,
+    first_pos: LSize,
+    _t: PhantomData<*const T>,
+}
+
+impl<T> Reader<T>
+where
+    T: bincode::Decode,
+{
+    pub fn new<P: AsRef<Path>>(path: P, _buf_size: usize) -> Result<Self, String> {
+        let file = fs::File::open(path).str_err()?;
+        let mmap = unsafe { MmapOptions::new().map(&file).str_err()? };
+
+        // read first pos and records count
+        let first_data: [u8; LEN_SIZE] = mmap[0..LEN_SIZE].try_into().str_err()?;
+        let first_pos = LSize::from_le_bytes(first_data);
+        let tab_len = (first_pos as usize) / LEN_SIZE;
+        let count = tab_len - 1;
+
+        Ok(Self {
+            mmap,
+            count,
+            first_pos,
+            _t: PhantomData,
+        })
+    }
+
+    pub fn len(&self) -> usize {
+        self.count
+    }
+
+    pub fn get(&mut self, index: usize) -> Result<T, String> {
+        if index >= self.len() {
+            return Err("index out of range".into());
+        }
+
+        let next_pos: usize = (index + 1) * LEN_SIZE;
+        let next_end: usize = next_pos + LEN_SIZE;
+
+        // read item data pos
+        let data_pos = if 0 == index {
+            self.first_pos
+        } else {
+            let tab_pos: usize = index * LEN_SIZE;
+            let pos_curr_data: [u8; LEN_SIZE] =
+                self.mmap[tab_pos..next_pos].try_into().str_err()?;
+            LSize::from_le_bytes(pos_curr_data)
+        } as usize;
+
+        // read next item pos
+        let pos_next_data: [u8; LEN_SIZE] = self.mmap[next_pos..next_end].try_into().str_err()?;
+        let data_pos_next = LSize::from_le_bytes(pos_next_data) as usize;
+
+        // read & unpack item data
+        let reader = io::Cursor::new(self.mmap[data_pos..data_pos_next].as_ref());
+        let data = zstd::decode_all(reader).str_err()?;
+
+        // decode item
+        let item: (T, usize) = bincode::decode_from_slice(&data, BINCODE_CFG).str_err()?;
+
+        Ok(item.0)
+    }
+
+    pub fn iter(&mut self) -> ReaderIter<'_, T> {
+        ReaderIter::new(self)
+    }
+}
+
+pub struct ReaderIter<'a, T>
+where
+    T: bincode::Decode,
+{
+    reader: &'a mut Reader<T>,
+    index: Option<usize>,
+}
+
+impl<'a, T> ReaderIter<'a, T>
+where
+    T: bincode::Decode,
+{
+    fn new(reader: &'a mut Reader<T>) -> Self {
+        ReaderIter {
+            reader,
+            index: None,
+        }
+    }
+}
+
+impl<'a, T> Iterator for ReaderIter<'a, T>
+where
+    T: bincode::Decode,
+{
+    type Item = T;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.index.is_none() && self.reader.len() != 0 {
+            self.index = Some(0);
+        }
+
+        match self.index {
+            Some(i) if i < self.reader.len() => self.nth(i),
+            _ => None,
+        }
+    }
+
+    fn nth(&mut self, n: usize) -> Option<Self::Item> {
+        if self.reader.len() <= n {
+            return None;
+        }
+        self.index = Some(n + 1);
+
+        let item = self.reader.get(n);
+        match item {
+            Ok(item) => Some(item),
+            Err(_) => None,
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let len = self.reader.len();
+        if self.index.is_none() {
+            return (len, Some(len));
+        }
+
+        let index = self.index.unwrap();
+        let rem = if len > index + 1 {
+            len - (index + 1)
+        } else {
+            0
+        };
+        (rem, Some(rem))
+    }
+
+    fn count(self) -> usize
+    where
+        Self: Sized,
+    {
+        self.reader.len()
+    }
+}
+
+impl<'a, T> ExactSizeIterator for ReaderIter<'a, T>
+where
+    T: bincode::Decode,
+{
+    fn len(&self) -> usize {
+        self.reader.len()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use tempfile::tempdir;
+
+    #[derive(bincode::Encode, bincode::Decode, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+    struct TestData {
+        num: u64,
+        test: String,
+    }
+
+    fn gen_data(count: usize) -> impl Iterator<Item = TestData> {
+        (0..count).into_iter().map(|i| TestData {
+            num: i as u64,
+            test: "test".repeat(i),
+        })
+    }
+
+    #[test]
+    fn test_write_read() {
+        let dir = tempdir().expect("tempdir");
+        let tmpfile = dir.path().join("test.tmp");
+        let opts = WriterOpts {
+            compress_lvl: 1,
+            data_buf_size: 10 * 1024 * 1024,
+            out_buf_size: 10 * 1024 * 1024,
+            current_buf_size: 4096,
+        };
+        let mut writer: Writer<TestData> = Writer::new(&tmpfile, opts).expect("new writer");
+
+        let items_iter = gen_data(5);
+        let items: Vec<TestData> = items_iter.collect();
+
+        writer.load(&mut items.clone().into_iter()).expect("load");
+        writer.finish().expect("finish write");
+
+        let mut reader: Reader<TestData> = Reader::new(&tmpfile, 2048).expect("new reader");
+        assert_eq!(items.len(), reader.len());
+
+        for (idx, item) in items.iter().enumerate() {
+            let ritem = reader.get(idx).expect("get");
+            assert_eq!(*item, ritem);
+        }
+    }
+
+    #[test]
+    fn test_write_read_iter() {
+        let dir = tempdir().expect("tempdir");
+        let tmpfile = dir.path().join("test.tmp");
+        let opts = WriterOpts {
+            compress_lvl: 1,
+            data_buf_size: 10 * 1024 * 1024,
+            out_buf_size: 10 * 1024 * 1024,
+            current_buf_size: 4096,
+        };
+        let mut writer: Writer<TestData> = Writer::new(&tmpfile, opts).expect("new writer");
+
+        let items_iter = gen_data(10);
+        let items: Vec<TestData> = items_iter.collect();
+
+        writer.load(&mut items.clone().into_iter()).expect("load");
+        writer.finish().expect("finish write");
+
+        let mut reader: Reader<TestData> = Reader::new(&tmpfile, 2048).expect("new reader");
+        assert_eq!(items.len(), reader.len());
+
+        items.into_iter().zip(reader.iter()).for_each(|pair| {
+            assert_eq!(pair.0, pair.1);
+        });
+    }
+}

src/main.rs

@@ -1,35 +1,31 @@
 extern crate serde;
-#[macro_use]
 extern crate serde_derive;
-// This allows inserting JSON documents
 #[macro_use]
 extern crate serde_json;
 extern crate ledb;
-// This allows define typed documents easy
-#[macro_use]
-extern crate ledb_derive;
 extern crate ledb_types;
 extern crate zip;
 
 use clap::{Parser, Subcommand};
 use rand::seq::IteratorRandom;
+
+use std::io;
 use std::path::PathBuf;
 use std::time::Instant;
-use std::{fs, io};
+use std::{fs, sync::mpsc, thread};
 
 use ledb::{Options, Storage};
 
-const ZIP_FILENAME: &str = "json.zip";
+mod db;
-const DB_DIR: &str = "db";
+mod questions;
+mod source;
 
-macro_rules! make {
+use crate::questions::{Question, QuestionsConverter};
-    ($Target:ident; by {$($field:ident),+}; from $src:expr) => {$Target {$(
+use crate::source::ReadSourceQuestionsBatches;
-        $field: $src.$field
+
-    ),+}};
+const ZIP_FILENAME: &str = "json.zip";
-    ($Target:ident; with defaults and by {$($field:ident),+}; from $src:expr) => {$Target {$(
+const NEW_DB_FILENAME: &str = "test.bin";
-        $field: $src.$field
+const DB_DIR: &str = "db";
-    ),+ ,..$Target::default()}}
-}
 
 #[derive(Subcommand, Debug)]
 enum Command {
@@ -45,6 +41,11 @@ enum Command {
         #[clap(value_parser, default_value = "0")]
         num: usize,
     },
+    Write2,
+    Print2 {
+        #[clap(value_parser, default_value = "0")]
+        id: u32,
+    },
 }
 
 #[derive(Parser, Debug)]
@@ -57,230 +58,28 @@ struct Cli {
     measure: bool,
 }
 
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+fn zip_reader_task(tx: mpsc::Sender<Question>) {
-struct SourceQuestion {
+    let zip_file = fs::File::open(ZIP_FILENAME).unwrap();
-    #[serde(default)]
+    let zip_reader = io::BufReader::new(zip_file);
-    num: u32,
+    let archive = zip::ZipArchive::new(zip_reader).unwrap();
-    id: String,
+    let mut source_questions = archive.source_questions();
 
-    #[serde(alias = "Вопрос")]
+    let questions = source_questions
-    description: String,
+        .convert()
-    #[serde(alias = "Ответ")]
+        .enumerate()
-    answer: String,
+        .map(|(num, mut question)| {
-
+            question.num = 1 + num as u32;
-    #[serde(alias = "Автор")]
+            question
-    #[serde(default)]
-    author: String,
-    #[serde(alias = "Комментарий")]
-    #[serde(default)]
-    comment: String,
-    #[serde(alias = "Комментарии")]
-    #[serde(alias = "Инфо")]
-    #[serde(default)]
-    comment1: String,
-    #[serde(alias = "Тур")]
-    #[serde(default)]
-    tour: String,
-    #[serde(alias = "Ссылка")]
-    #[serde(alias = "URL")]
-    #[serde(default)]
-    url: String,
-    #[serde(alias = "Дата")]
-    #[serde(default)]
-    date: String,
-    #[serde(alias = "Обработан")]
-    #[serde(default)]
-    processed_by: String,
-    #[serde(alias = "Редактор")]
-    #[serde(default)]
-    redacted_by: String,
-    #[serde(alias = "Копирайт")]
-    #[serde(default)]
-    copyright: String,
-    #[serde(alias = "Тема")]
-    #[serde(default)]
-    theme: String,
-    #[serde(alias = "Вид")]
-    #[serde(alias = "Тип")]
-    #[serde(default)]
-    kind: String,
-    #[serde(alias = "Источник")]
-    #[serde(default)]
-    source: String,
-    #[serde(alias = "Рейтинг")]
-    #[serde(default)]
-    rating: String,
-}
-
-#[derive(Debug, Default, Clone, Serialize, Deserialize)]
-struct SourceQuestionsBatch {
-    #[serde(default)]
-    filename: String,
-    #[serde(alias = "Пакет")]
-    #[serde(alias = "Чемпионат")]
-    description: String,
-    #[serde(alias = "Автор")]
-    #[serde(default)]
-    author: String,
-    #[serde(alias = "Комментарий")]
-    #[serde(alias = "Комментарии")]
-    #[serde(alias = "Инфо")]
-    #[serde(default)]
-    comment: String,
-    #[serde(alias = "Ссылка")]
-    #[serde(alias = "URL")]
-    #[serde(default)]
-    url: String,
-    #[serde(alias = "Дата")]
-    #[serde(default)]
-    date: String,
-    #[serde(alias = "Обработан")]
-    #[serde(default)]
-    processed_by: String,
-    #[serde(alias = "Редактор")]
-    #[serde(default)]
-    redacted_by: String,
-    #[serde(alias = "Копирайт")]
-    #[serde(default)]
-    copyright: String,
-    #[serde(alias = "Тема")]
-    #[serde(default)]
-    theme: String,
-    #[serde(alias = "Вид")]
-    #[serde(alias = "Тип")]
-    #[serde(default)]
-    kind: String,
-    #[serde(alias = "Источник")]
-    #[serde(default)]
-    source: String,
-    #[serde(alias = "Рейтинг")]
-    #[serde(default)]
-    rating: String,
-    #[serde(alias = "Вопросы")]
-    questions: Vec<SourceQuestion>,
-}
-
-#[derive(Debug, Default, Clone, Serialize, Deserialize, Document)]
-struct BatchInfo {
-    #[document(primary)]
-    #[serde(default)]
-    filename: String,
-    #[serde(default)]
-    description: String,
-    #[serde(default)]
-    author: String,
-    #[serde(default)]
-    comment: String,
-    #[serde(default)]
-    url: String,
-    #[serde(default)]
-    date: String,
-    #[serde(default)]
-    processed_by: String,
-    #[serde(default)]
-    redacted_by: String,
-    #[serde(default)]
-    copyright: String,
-    #[serde(default)]
-    theme: String,
-    #[serde(default)]
-    kind: String,
-    #[serde(default)]
-    source: String,
-    #[serde(default)]
-    rating: String,
-}
-
-#[derive(Debug, Default, Clone, Serialize, Deserialize, Document)]
-struct Question {
-    #[document(primary)]
-    #[serde(default)]
-    num: u32,
-    #[document(index)]
-    id: String,
-
-    description: String,
-    answer: String,
-
-    #[serde(default)]
-    author: String,
-    #[serde(default)]
-    comment: String,
-    #[serde(default)]
-    comment1: String,
-    #[serde(default)]
-    tour: String,
-    #[serde(default)]
-    url: String,
-    #[serde(default)]
-    date: String,
-    #[serde(default)]
-    processed_by: String,
-    #[serde(default)]
-    redacted_by: String,
-    #[serde(default)]
-    copyright: String,
-    #[serde(default)]
-    theme: String,
-    #[serde(default)]
-    kind: String,
-    #[serde(default)]
-    source: String,
-    #[serde(default)]
-    rating: String,
-    #[document(nested)]
-    #[serde(default)]
-    batch_info: BatchInfo,
-}
-
-impl From<SourceQuestion> for Question {
-    fn from(src: SourceQuestion) -> Self {
-        make! {Self; with defaults and by {
-            num, id, description, answer, author, comment, comment1, tour, url,
-            date, processed_by, redacted_by, copyright, theme, kind, source, rating
-        }; from src}
-    }
-}
-
-impl From<SourceQuestionsBatch> for BatchInfo {
-    fn from(src: SourceQuestionsBatch) -> Self {
-        make! {Self; by {
-            filename, description, author, comment, url, date,
-            processed_by, redacted_by, copyright, theme, kind, source, rating
-        }; from src}
-    }
-}
-
-impl From<SourceQuestionsBatch> for Vec<Question> {
-    fn from(src: SourceQuestionsBatch) -> Self {
-        let mut result: Vec<Question> = src
-            .questions
-            .iter()
-            .map(|item| item.clone().into())
-            .collect();
-        let batch_info = BatchInfo::from(src);
-        result.iter_mut().for_each(|mut question| {
-            question.batch_info = batch_info.clone();
         });
-
+    for question in questions {
-        result
+        let res = tx.send(question);
+        if res.is_err() {
+            break;
         }
     }
-
+    println!("read done");
-// measure and return time elapsed in `func` in seconds
-pub fn measure<F: FnOnce()>(func: F) -> f64 {
-    let start = Instant::now();
-    func();
-    let elapsed = start.elapsed();
-    (elapsed.as_secs() as f64) + (elapsed.subsec_nanos() as f64 / 1_000_000_000.0)
 }
-
+fn db_writer_task(rx: mpsc::Receiver<Question>) {
-pub fn measure_and_print<F: FnOnce()>(func: F) {
-    let m = measure(func);
-    eprintln!("{}", m);
-}
-
-fn write_db() {
     let out_file: PathBuf = [DB_DIR, "data.mdb"].into_iter().collect();
     match fs::metadata(&out_file) {
         Ok(x) if x.is_file() => {
@@ -290,10 +89,6 @@ fn write_db() {
         _ => {}
     };
 
-    let zip_file = fs::File::open(ZIP_FILENAME).unwrap();
-    let zip_reader = io::BufReader::new(zip_file);
-    let mut archive = zip::ZipArchive::new(zip_reader).unwrap();
-
     let options: Options = serde_json::from_value(json!({
         "map_size": 900 * 1024 * 1024, // 900mb
         "write_map": true,
@@ -307,43 +102,30 @@ fn write_db() {
     let storage = Storage::new(DB_DIR, options).unwrap();
     let collection = storage.collection("questions").unwrap();
 
-    println!("converting...");
+    let count = collection.load(rx).expect("load");
 
-    let mut count: usize = 0;
+    println!("loaded {count}");
-    let count = &mut count;
-    (0..archive.len())
-        .map(|i| {
-            let file = archive.by_index(i).unwrap();
-            let name = file.mangled_name();
-            let name_str = name.to_str().unwrap();
 
-            let data: Result<SourceQuestionsBatch, _> = serde_json::from_reader(file);
+    println!("syncing to disk...");
-
-            (String::from(name_str), data)
-        })
-        .filter(|(_, data)| data.is_ok())
-        .flat_map(|(filename, data)| {
-            let mut data = data.unwrap();
-            data.filename = filename;
-            let questions: Vec<Question> = data.into();
-            questions
-        })
-        .for_each(|question| {
-            let result = collection.insert(&question);
-            if result.is_err() {
-                println!("-- {:#?}", question);
-                panic!("{:#?}", result);
-            } else {
-                *count += 1;
-            }
-        });
-
-    println!("inserted {}\nwriting...", count);
     storage.sync(true).unwrap();
+
     print!("stats: ");
     let stats = storage.stat().unwrap();
     println!("{:?}", stats);
+
     drop(storage);
+    println!("write done");
+}
+
+fn write_db() {
+    let (tx, rx) = mpsc::channel::<Question>();
+    [
+        thread::spawn(move || zip_reader_task(tx)),
+        thread::spawn(move || db_writer_task(rx)),
+    ]
+    .into_iter()
+    .for_each(|handle| handle.join().expect("thread panic"));
+    println!("all done");
 }
 
 fn print_question_from<F>(get_q: F)
@@ -354,20 +136,21 @@ where
     println!("{:#?}", q)
 }
 
-fn read_from_zip(mut file_num: usize, mut num: usize) -> Option<Question> {
+fn read_from_zip(file_num: usize, mut num: usize) -> Option<Question> {
+    let mut rng = rand::thread_rng();
     let zip_file = fs::File::open(ZIP_FILENAME).unwrap();
     let zip_reader = io::BufReader::new(zip_file);
-    let mut archive = zip::ZipArchive::new(zip_reader).unwrap();
+    let archive = zip::ZipArchive::new(zip_reader).unwrap();
-
-    let mut rng = rand::thread_rng();
-    if file_num == 0 {
-        file_num = (1..=archive.len()).choose(&mut rng).unwrap();
-    }
-    let file = archive.by_index(file_num - 1).unwrap();
-    let data: Result<SourceQuestionsBatch, _> = serde_json::from_reader(file);
-    let data = data.unwrap();
-    let questions: Vec<Question> = data.into();
 
+    let mut source_questions = archive.source_questions();
+    let (filename, batch) = if file_num == 0 {
+        source_questions.choose(&mut rng).unwrap()
+    } else {
+        source_questions.nth(file_num - 1).unwrap()
+    };
+    let mut batch = batch.unwrap();
+    batch.filename = filename;
+    let questions: Vec<Question> = batch.into();
     if num == 0 {
         num = (1..=questions.len()).choose(&mut rng).unwrap();
     }
@@ -413,6 +196,19 @@ fn read_from_db(mut id: u32) -> Option<Question> {
     collection.get::<Question>(id).unwrap()
 }
 
+// measure and return time elapsed in `func` in seconds
+pub fn measure<F: FnOnce()>(func: F) -> f64 {
+    let start = Instant::now();
+    func();
+    let elapsed = start.elapsed();
+    (elapsed.as_secs() as f64) + (elapsed.subsec_nanos() as f64 / 1_000_000_000.0)
+}
+
+pub fn measure_and_print<F: FnOnce()>(func: F) {
+    let m = measure(func);
+    eprintln!("{}", m);
+}
+
 fn main() {
     let args = Cli::parse();
 
@@ -427,6 +223,11 @@ fn main() {
             let get_question = Box::new(|| read_from_zip(*file_num, *num));
             Box::new(|| print_question_from(get_question))
         }
+        Command::Write2 => Box::new(write_db2),
+        Command::Print2 { id } => {
+            let get_question = Box::new(|| read_from_db2(*id));
+            Box::new(|| print_question_from(get_question))
+        }
     };
 
     if args.measure {
@@ -435,3 +236,41 @@ fn main() {
 
     action();
 }
+
+fn read_from_db2(id: u32) -> Option<Question> {
+    let mut reader: db::Reader<Question> =
+        db::Reader::new(NEW_DB_FILENAME, 2048).expect("new db reader");
+
+    let mut questions = reader.iter();
+
+    match id {
+        0 => {
+            let mut rng = rand::thread_rng();
+            questions.choose(&mut rng)
+        }
+        _ => questions.nth((id - 1) as usize),
+    }
+}
+fn write_db2() {
+    let (tx, rx) = mpsc::channel::<Question>();
+    [
+        thread::spawn(move || zip_reader_task(tx)),
+        thread::spawn(move || db_writer2_task(rx)),
+    ]
+    .into_iter()
+    .for_each(|handle| handle.join().expect("thread panic"));
+    println!("all done");
+}
+fn db_writer2_task(rx: mpsc::Receiver<Question>) {
+    let writer_opts = db::WriterOpts::default();
+    let mut writer: db::Writer<Question> =
+        db::Writer::new(NEW_DB_FILENAME, writer_opts).expect("new db writer");
+
+    writer
+        .load(&mut rx.iter())
+        .unwrap_or_else(|e| panic!("db writer load, {e:#?}"));
+
+    writer.finish().expect("db writer finish");
+
+    println!("write done");
+}

src/questions.rs

@@ -0,0 +1,145 @@
+use ledb::Document;
+use serde_derive::{Deserialize, Serialize};
+
+use crate::source::{SourceQuestion, SourceQuestionsBatch};
+
+macro_rules! make {
+    ($Target:ident; by {$($field:ident),+}; from $src:expr) => {$Target {$(
+        $field: $src.$field
+    ),+}};
+    ($Target:ident; with defaults and by {$($field:ident),+}; from $src:expr) => {$Target {$(
+        $field: $src.$field
+    ),+ ,..$Target::default()}}
+}
+
+#[derive(
+    Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode,
+)]
+pub struct BatchInfo {
+    #[document(primary)]
+    #[serde(default)]
+    pub filename: String,
+    #[serde(default)]
+    pub description: String,
+    #[serde(default)]
+    pub author: String,
+    #[serde(default)]
+    pub comment: String,
+    #[serde(default)]
+    pub url: String,
+    #[serde(default)]
+    pub date: String,
+    #[serde(default)]
+    pub processed_by: String,
+    #[serde(default)]
+    pub redacted_by: String,
+    #[serde(default)]
+    pub copyright: String,
+    #[serde(default)]
+    pub theme: String,
+    #[serde(default)]
+    pub kind: String,
+    #[serde(default)]
+    pub source: String,
+    #[serde(default)]
+    pub rating: String,
+}
+
+#[derive(
+    Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode,
+)]
+pub struct Question {
+    #[document(primary)]
+    #[serde(default)]
+    pub num: u32,
+    #[document(index)]
+    pub id: String,
+
+    pub description: String,
+    pub answer: String,
+
+    #[serde(default)]
+    pub author: String,
+    #[serde(default)]
+    pub comment: String,
+    #[serde(default)]
+    pub comment1: String,
+    #[serde(default)]
+    pub tour: String,
+    #[serde(default)]
+    pub url: String,
+    #[serde(default)]
+    pub date: String,
+    #[serde(default)]
+    pub processed_by: String,
+    #[serde(default)]
+    pub redacted_by: String,
+    #[serde(default)]
+    pub copyright: String,
+    #[serde(default)]
+    pub theme: String,
+    #[serde(default)]
+    pub kind: String,
+    #[serde(default)]
+    pub source: String,
+    #[serde(default)]
+    pub rating: String,
+    #[document(nested)]
+    #[serde(default)]
+    pub batch_info: BatchInfo,
+}
+
+impl From<SourceQuestion> for Question {
+    fn from(src: SourceQuestion) -> Self {
+        make! {Self; with defaults and by {
+            num, id, description, answer, author, comment, comment1, tour, url,
+            date, processed_by, redacted_by, copyright, theme, kind, source, rating
+        }; from src}
+    }
+}
+
+impl From<SourceQuestionsBatch> for BatchInfo {
+    fn from(src: SourceQuestionsBatch) -> Self {
+        make! {Self; by {
+            filename, description, author, comment, url, date,
+            processed_by, redacted_by, copyright, theme, kind, source, rating
+        }; from src}
+    }
+}
+
+impl From<SourceQuestionsBatch> for Vec<Question> {
+    fn from(src: SourceQuestionsBatch) -> Self {
+        let mut result: Vec<Question> = src
+            .questions
+            .iter()
+            .map(|item| item.clone().into())
+            .collect();
+        let batch_info = BatchInfo::from(src);
+        result.iter_mut().for_each(|mut question| {
+            question.batch_info = batch_info.clone();
+        });
+
+        result
+    }
+}
+
+pub trait QuestionsConverter {
+    fn convert<'a>(&'a mut self) -> Box<dyn Iterator<Item = Question> + 'a>;
+}
+
+impl<T> QuestionsConverter for T
+where
+    T: Iterator<Item = (String, Result<SourceQuestionsBatch, serde_json::Error>)>,
+{
+    fn convert<'a>(&'a mut self) -> Box<dyn Iterator<Item = Question> + 'a> {
+        let iter = self
+            .filter(|(_, data)| data.is_ok())
+            .flat_map(|(filename, data)| {
+                let mut batch = data.unwrap();
+                batch.filename = filename;
+                let questions: Vec<Question> = batch.into();
+                questions
+            });
+        Box::new(iter)
+    }
+}

src/source.rs

@@ -0,0 +1,206 @@
+use serde_derive::{Deserialize, Serialize};
+use std::io::{Read, Seek};
+use zip::ZipArchive;
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct SourceQuestion {
+    #[serde(default)]
+    pub num: u32,
+    pub id: String,
+
+    #[serde(alias = "Вопрос")]
+    pub description: String,
+    #[serde(alias = "Ответ")]
+    pub answer: String,
+
+    #[serde(alias = "Автор")]
+    #[serde(default)]
+    pub author: String,
+    #[serde(alias = "Комментарий")]
+    #[serde(default)]
+    pub comment: String,
+    #[serde(alias = "Комментарии")]
+    #[serde(alias = "Инфо")]
+    #[serde(default)]
+    pub comment1: String,
+    #[serde(alias = "Тур")]
+    #[serde(default)]
+    pub tour: String,
+    #[serde(alias = "Ссылка")]
+    #[serde(alias = "URL")]
+    #[serde(default)]
+    pub url: String,
+    #[serde(alias = "Дата")]
+    #[serde(default)]
+    pub date: String,
+    #[serde(alias = "Обработан")]
+    #[serde(default)]
+    pub processed_by: String,
+    #[serde(alias = "Редактор")]
+    #[serde(default)]
+    pub redacted_by: String,
+    #[serde(alias = "Копирайт")]
+    #[serde(default)]
+    pub copyright: String,
+    #[serde(alias = "Тема")]
+    #[serde(default)]
+    pub theme: String,
+    #[serde(alias = "Вид")]
+    #[serde(alias = "Тип")]
+    #[serde(default)]
+    pub kind: String,
+    #[serde(alias = "Источник")]
+    #[serde(default)]
+    pub source: String,
+    #[serde(alias = "Рейтинг")]
+    #[serde(default)]
+    pub rating: String,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct SourceQuestionsBatch {
+    #[serde(default)]
+    pub filename: String,
+    #[serde(alias = "Пакет")]
+    #[serde(alias = "Чемпионат")]
+    pub description: String,
+    #[serde(alias = "Автор")]
+    #[serde(default)]
+    pub author: String,
+    #[serde(alias = "Комментарий")]
+    #[serde(alias = "Комментарии")]
+    #[serde(alias = "Инфо")]
+    #[serde(default)]
+    pub comment: String,
+    #[serde(alias = "Ссылка")]
+    #[serde(alias = "URL")]
+    #[serde(default)]
+    pub url: String,
+    #[serde(alias = "Дата")]
+    #[serde(default)]
+    pub date: String,
+    #[serde(alias = "Обработан")]
+    #[serde(default)]
+    pub processed_by: String,
+    #[serde(alias = "Редактор")]
+    #[serde(default)]
+    pub redacted_by: String,
+    #[serde(alias = "Копирайт")]
+    #[serde(default)]
+    pub copyright: String,
+    #[serde(alias = "Тема")]
+    #[serde(default)]
+    pub theme: String,
+    #[serde(alias = "Вид")]
+    #[serde(alias = "Тип")]
+    #[serde(default)]
+    pub kind: String,
+    #[serde(alias = "Источник")]
+    #[serde(default)]
+    pub source: String,
+    #[serde(alias = "Рейтинг")]
+    #[serde(default)]
+    pub rating: String,
+    #[serde(alias = "Вопросы")]
+    pub questions: Vec<SourceQuestion>,
+}
+
+pub struct SourceQuestionsZipReader<R>
+where
+    R: Read + Seek,
+{
+    zipfile: ZipArchive<R>,
+    index: Option<usize>,
+}
+
+impl<R> SourceQuestionsZipReader<R>
+where
+    R: Read + Seek,
+{
+    fn new(zipfile: ZipArchive<R>) -> Self {
+        SourceQuestionsZipReader {
+            zipfile,
+            index: None,
+        }
+    }
+}
+
+impl<R> Iterator for SourceQuestionsZipReader<R>
+where
+    R: Read + Seek,
+{
+    type Item = (String, Result<SourceQuestionsBatch, serde_json::Error>);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.index.is_none() && !self.zipfile.is_empty() {
+            self.index = Some(0);
+        }
+
+        match self.index {
+            Some(i) if i < self.zipfile.len() => {
+                self.index = Some(i + 1);
+
+                self.nth(i)
+            }
+            _ => None,
+        }
+    }
+
+    fn nth(&mut self, n: usize) -> Option<Self::Item> {
+        if self.zipfile.len() <= n {
+            return None;
+        }
+        self.index = Some(n + 1);
+
+        let file = self.zipfile.by_index(n).unwrap();
+        let name = file.mangled_name();
+        let name_str = name.to_str().unwrap();
+
+        let data: Result<SourceQuestionsBatch, _> = serde_json::from_reader(file);
+
+        Some((String::from(name_str), data))
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let len = self.zipfile.len();
+        let index = self.index.unwrap_or(0);
+        let rem = if len > index + 1 {
+            len - (index + 1)
+        } else {
+            0
+        };
+        (rem, Some(rem))
+    }
+
+    fn count(self) -> usize
+    where
+        Self: Sized,
+    {
+        self.zipfile.len()
+    }
+}
+
+impl<R> ExactSizeIterator for SourceQuestionsZipReader<R>
+where
+    R: Read + Seek,
+{
+    fn len(&self) -> usize {
+        self.zipfile.len()
+    }
+}
+
+pub trait ReadSourceQuestionsBatches<R>
+where
+    R: Read + Seek,
+{
+    fn source_questions(self) -> SourceQuestionsZipReader<R>;
+}
+
+impl<R> ReadSourceQuestionsBatches<R> for ZipArchive<R>
+where
+    R: Read + Seek,
+{
+    fn source_questions(self) -> SourceQuestionsZipReader<R> {
+        SourceQuestionsZipReader::new(self)
+    }
+}