Compare commits

...

2 Commits

Author SHA1 Message Date
5355d0398d
remove ledb
All checks were successful
continuous-integration/drone Build is passing
2023-03-29 13:00:17 +03:00
b63e9aa45c
add publish pipeline 2023-03-29 12:59:35 +03:00
8 changed files with 58 additions and 442 deletions

View File

@ -11,3 +11,23 @@ steps:
environment:
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
---
kind: pipeline
name: publish
steps:
- name: push
image: rust:1-alpine
commands:
- cargo publish --registry gitea -p chgk_ledb_lib
environment:
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
CARGO_REGISTRIES_GITEA_INDEX: https://gitea.b4tman.ru/b4tman/_cargo-index
CARGO_REGISTRIES_GITEA_TOKEN:
from_secret: gitea_password
trigger:
branch:
- master
event:
- tag

235
Cargo.lock generated
View File

@ -14,21 +14,12 @@ version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
"cipher",
"cpufeatures",
"opaque-debug",
]
[[package]]
name = "aho-corasick"
version = "0.7.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
dependencies = [
"memchr",
]
[[package]]
name = "anes"
version = "0.1.6"
@ -52,12 +43,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "base64"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "base64ct"
version = "1.0.1"
@ -83,12 +68,6 @@ dependencies = [
"virtue",
]
[[package]]
name = "bitflags"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5"
[[package]]
name = "bitflags"
version = "1.3.2"
@ -152,12 +131,6 @@ dependencies = [
"jobserver",
]
[[package]]
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "cfg-if"
version = "1.0.0"
@ -172,9 +145,6 @@ dependencies = [
"chgk_ledb_lib",
"clap",
"criterion",
"ledb",
"ledb-derive",
"ledb-types",
"rand",
"serde",
"serde_derive",
@ -188,9 +158,6 @@ name = "chgk_ledb_lib"
version = "0.1.0"
dependencies = [
"bincode",
"ledb",
"ledb-derive",
"ledb-types",
"memmap",
"serde",
"serde_derive",
@ -243,7 +210,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
dependencies = [
"atty",
"bitflags 1.3.2",
"bitflags",
"clap_derive",
"clap_lex",
"indexmap",
@ -296,7 +263,7 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
]
[[package]]
@ -341,7 +308,7 @@ version = "0.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
"crossbeam-utils",
]
@ -351,7 +318,7 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
"crossbeam-epoch",
"crossbeam-utils",
]
@ -363,7 +330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
dependencies = [
"autocfg",
"cfg-if 1.0.0",
"cfg-if",
"crossbeam-utils",
"memoffset",
"scopeguard",
@ -375,7 +342,7 @@ version = "0.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
]
[[package]]
@ -399,33 +366,6 @@ dependencies = [
"subtle",
]
[[package]]
name = "dirs"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3"
dependencies = [
"cfg-if 0.1.10",
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
dependencies = [
"libc",
"redox_users",
"winapi",
]
[[package]]
name = "dunce"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bd4b30a6560bbd9b4620f4de34c3f14f60848e58a9b7216801afcb4c7b31c3c"
[[package]]
name = "either"
version = "1.8.1"
@ -472,12 +412,6 @@ dependencies = [
"miniz_oxide",
]
[[package]]
name = "gcc"
version = "0.3.55"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2"
[[package]]
name = "generic-array"
version = "0.14.6"
@ -494,7 +428,7 @@ version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
"libc",
"wasi",
]
@ -560,7 +494,7 @@ version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
]
[[package]]
@ -612,94 +546,27 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "ledb"
version = "0.4.0"
source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b"
dependencies = [
"byteorder",
"dirs",
"dunce",
"ledb-derive",
"ledb-types",
"lmdb-zero",
"ordered-float",
"regex",
"ron",
"serde",
"serde_cbor",
"supercow",
]
[[package]]
name = "ledb-derive"
version = "0.4.0"
source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "ledb-types"
version = "0.4.0"
source = "git+https://github.com/b4tman/ledb.git?rev=a646b90e#a646b90eea1bf9fa8d4c2f1c5ff9153f8c1ad44b"
dependencies = [
"serde",
"serde_cbor",
"serde_json",
]
[[package]]
name = "libc"
version = "0.2.139"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
[[package]]
name = "liblmdb-sys"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "feed38a3a580f60bf61aaa067b0ff4123395966839adeaf67258a9e50c4d2e49"
dependencies = [
"gcc",
"libc",
]
[[package]]
name = "linux-raw-sys"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
[[package]]
name = "lmdb-zero"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13416eee745b087c22934f35f1f24da22da41ba2a5ce197143d168ce055cc58d"
dependencies = [
"bitflags 0.9.1",
"libc",
"liblmdb-sys",
"supercow",
]
[[package]]
name = "log"
version = "0.4.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
]
[[package]]
name = "memchr"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "memmap"
version = "0.7.0"
@ -765,16 +632,6 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "ordered-float"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7"
dependencies = [
"num-traits",
"serde",
]
[[package]]
name = "os_str_bytes"
version = "6.4.1"
@ -944,18 +801,7 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_users"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
dependencies = [
"getrandom",
"redox_syscall",
"thiserror",
"bitflags",
]
[[package]]
@ -964,8 +810,6 @@ version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
@ -975,24 +819,13 @@ version = "0.6.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
[[package]]
name = "ron"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86018df177b1beef6c7c8ef949969c4f7cb9a9344181b92486b23c79995bdaa4"
dependencies = [
"base64",
"bitflags 1.3.2",
"serde",
]
[[package]]
name = "rustix"
version = "0.36.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644"
dependencies = [
"bitflags 1.3.2",
"bitflags",
"errno",
"io-lifetimes",
"libc",
@ -1030,16 +863,6 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde_cbor"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
dependencies = [
"half",
"serde",
]
[[package]]
name = "serde_derive"
version = "1.0.152"
@ -1068,7 +891,7 @@ version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
"cpufeatures",
"digest",
]
@ -1079,7 +902,7 @@ version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
"cpufeatures",
"digest",
]
@ -1096,12 +919,6 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]]
name = "supercow"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "171758edb47aa306a78dfa4ab9aeb5167405bd4e3dc2b64e88f6a84bbe98bd63"
[[package]]
name = "syn"
version = "1.0.109"
@ -1119,7 +936,7 @@ version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
"fastrand",
"redox_syscall",
"rustix",
@ -1141,26 +958,6 @@ version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
[[package]]
name = "thiserror"
version = "1.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "time"
version = "0.3.20"
@ -1245,7 +1042,7 @@ version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
"wasm-bindgen-macro",
]

View File

@ -13,7 +13,6 @@ harness = false
[dependencies]
chgk_ledb_lib = {path = "../lib"}
serde_json="1.0"
ledb = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb"}
zip="0.6"
rand="0.8"
clap = { version = "3.2.22", features = ["derive"] }
@ -22,8 +21,5 @@ clap = { version = "3.2.22", features = ["derive"] }
criterion = "0.4.0"
tempfile = "3.3"
bincode = "^2.0.0-rc.2"
ledb = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb"}
ledb-derive = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-derive"}
ledb-types = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-types"}
serde="1.0"
serde_derive="1.0"

View File

@ -1,13 +1,10 @@
#[macro_use]
extern crate criterion;
extern crate bincode;
extern crate ledb;
extern crate ledb_types;
extern crate serde;
extern crate serde_derive;
extern crate tempfile;
#[macro_use]
extern crate serde_json;
extern crate tempfile;
use chgk_ledb_lib::db;
use std::path::PathBuf;
@ -17,7 +14,6 @@ use db::{Reader, Writer, WriterOpts};
use criterion::{BatchSize, Criterion};
use tempfile::tempdir;
use ledb::{Document, Options, Storage};
use serde_derive::{Deserialize, Serialize};
#[derive(
@ -31,10 +27,8 @@ use serde_derive::{Deserialize, Serialize};
Ord,
Serialize,
Deserialize,
Document,
)]
struct TestData {
#[document(primary)]
num1: u64,
num2: u64,
test: String,
@ -48,7 +42,7 @@ fn gen_data(count: usize) -> impl Iterator<Item = TestData> {
.map(|i| 143 + i as u64)
.map(|i| TestData {
num1: i,
num2: i*100 ^ 0xDF0E441122334455,
num2: i * 100 ^ 0xDF0E441122334455,
test: "test ---- Test ____".repeat(123 + i as usize % 15),
})
}
@ -108,83 +102,9 @@ fn db_write(c: &mut Criterion) {
});
}
fn ledb_write(c: &mut Criterion) {
let dir = tempdir().expect("tempdir");
let tmp_dir = dir.as_ref();
c.bench_function("ledb_write", |b| {
b.iter_batched(
|| {
let src = gen_data(N).collect::<Vec<TestData>>().into_iter();
let options: Options = serde_json::from_value(json!({
"map_size": 100 * 1024 * 1024, // 100mb
"write_map": true,
"map_async": true,
"no_lock": true,
"no_meta_sync": true,
"no_sync": true,
}))
.unwrap();
let storage = Storage::new(tmp_dir, options).unwrap();
let collection = storage.collection("test").unwrap();
(src, collection)
},
|(src, collection)| collection.load(src).expect("load"),
BatchSize::SmallInput,
)
});
}
fn ledb_read(c: &mut Criterion) {
let dir = tempdir().expect("tempdir");
let tmp_dir = dir.as_ref();
let write_options: Options = serde_json::from_value(json!({
"map_size": 100 * 1024 * 1024, // 100mb
"write_map": true,
"map_async": true,
"no_lock": true,
"no_meta_sync": true,
"no_sync": true,
}))
.unwrap();
let storage = Storage::new(&tmp_dir, write_options).unwrap();
let collection = storage.collection("test").unwrap();
let items_iter = gen_data(N).collect::<Vec<TestData>>().into_iter();
collection.load(items_iter).expect("load");
drop(collection);
drop(storage);
c.bench_function("ledb_read", |b| {
b.iter_batched(
|| {
let options: Options = serde_json::from_value(json!({
"read_only": true,
"map_async": true,
"no_lock": true,
}))
.unwrap();
let storage = Storage::new(tmp_dir, options).unwrap();
let collection = storage.collection("test").unwrap();
collection
},
|collection| {
let mut collection_iter = collection.dump::<TestData>().expect("dump");
while let Some(item) = collection_iter.next() {
drop(item);
}
},
BatchSize::SmallInput,
)
});
}
fn config() -> Criterion {
Criterion::default().sample_size(40)
}
criterion_group! {name=ledb; config = config(); targets = ledb_read, ledb_write}
criterion_group! {name=benches; config = config(); targets = db_read, db_write}
criterion_main!(benches, ledb);
criterion_main!(benches);

View File

@ -1,15 +1,11 @@
#[macro_use]
extern crate serde_json;
use clap::{Parser, Subcommand};
use rand::seq::IteratorRandom;
use std::io;
use std::path::PathBuf;
use std::time::Instant;
use std::{fs, sync::mpsc, thread};
use ledb::{Options, Storage};
use chgk_ledb_lib::db;
use chgk_ledb_lib::questions;
use chgk_ledb_lib::source;
@ -19,12 +15,10 @@ use crate::source::ReadSourceQuestionsBatches;
const ZIP_FILENAME: &str = "json.zip";
const NEW_DB_FILENAME: &str = "db.dat";
const DB_DIR: &str = "db";
#[derive(Subcommand, Debug)]
enum Command {
Write,
Compact,
Print {
#[clap(value_parser, default_value = "0")]
id: u32,
@ -35,11 +29,6 @@ enum Command {
#[clap(value_parser, default_value = "0")]
num: usize,
},
Write2,
Print2 {
#[clap(value_parser, default_value = "0")]
id: u32,
},
}
#[derive(Parser, Debug)]
@ -73,60 +62,12 @@ fn zip_reader_task(tx: mpsc::Sender<Question>) {
}
println!("read done");
}
fn db_writer_task(rx: mpsc::Receiver<Question>) {
let out_file: PathBuf = [DB_DIR, "data.mdb"].into_iter().collect();
match fs::metadata(&out_file) {
Ok(x) if x.is_file() => {
fs::remove_file(&out_file).unwrap();
println!(r#""{}" removed"#, out_file.to_str().unwrap());
}
_ => {}
};
let options: Options = serde_json::from_value(json!({
"map_size": 900 * 1024 * 1024, // 900mb
"write_map": true,
"map_async": true,
"no_lock": true,
"no_meta_sync": true,
"no_sync": true,
}))
.unwrap();
let storage = Storage::new(DB_DIR, options).unwrap();
let collection = storage.collection("questions").unwrap();
let count = collection.load(rx).expect("load");
println!("loaded {count}");
println!("syncing to disk...");
storage.sync(true).unwrap();
print!("stats: ");
let stats = storage.stat().unwrap();
println!("{:?}", stats);
drop(storage);
println!("write done");
}
fn write_db() {
let (tx, rx) = mpsc::channel::<Question>();
[
thread::spawn(move || zip_reader_task(tx)),
thread::spawn(move || db_writer_task(rx)),
]
.into_iter()
.for_each(|handle| handle.join().expect("thread panic"));
println!("all done");
}
fn print_question_from<F>(get_q: F)
where
F: FnOnce() -> Option<Question>,
{
let q = get_q().unwrap();
let q = get_q().expect("question not found");
println!("{:#?}", q)
}
@ -151,45 +92,6 @@ fn read_from_zip(file_num: usize, mut num: usize) -> Option<Question> {
Some(questions[num - 1].clone())
}
fn compact_db() {
let options: Options = serde_json::from_value(json!({
"write_map": true,
"map_async": true,
"no_lock": true,
"no_meta_sync": true,
"no_sync": true,
"compact": true,
}))
.unwrap();
let storage = Storage::new(DB_DIR, options).unwrap();
storage.sync(true).unwrap();
let stats = storage.stat().unwrap();
println!("{:?}", stats);
drop(storage);
}
fn read_from_db(mut id: u32) -> Option<Question> {
let options: Options = serde_json::from_value(json!({
"read_only": true,
"map_async": true,
"no_lock": true,
}))
.unwrap();
let storage = Storage::new(DB_DIR, options).unwrap();
let collection = storage.collection("questions").unwrap();
let mut rng = rand::thread_rng();
if id == 0 {
let last_id = collection.last_id().unwrap();
id = (1..=last_id).choose(&mut rng).unwrap();
}
collection.get::<Question>(id).unwrap()
}
// measure and return time elapsed in `func` in seconds
pub fn measure<F: FnOnce()>(func: F) -> f64 {
let start = Instant::now();
@ -208,7 +110,6 @@ fn main() {
let mut action: Box<dyn FnOnce()> = match &args.command {
Command::Write => Box::new(write_db),
Command::Compact => Box::new(compact_db),
Command::Print { id } => {
let get_question = Box::new(|| read_from_db(*id));
Box::new(|| print_question_from(get_question))
@ -217,11 +118,6 @@ fn main() {
let get_question = Box::new(|| read_from_zip(*file_num, *num));
Box::new(|| print_question_from(get_question))
}
Command::Write2 => Box::new(write_db2),
Command::Print2 { id } => {
let get_question = Box::new(|| read_from_db2(*id));
Box::new(|| print_question_from(get_question))
}
};
if args.measure {
@ -231,11 +127,11 @@ fn main() {
action();
}
fn read_from_db2(id: u32) -> Option<Question> {
fn read_from_db(id: u32) -> Option<Question> {
let reader: db::Reader<Question> =
db::Reader::new(NEW_DB_FILENAME, 2048).expect("new db reader");
let mut questions = reader.iter();
let mut questions = reader.into_iter();
match id {
0 => {
@ -245,17 +141,17 @@ fn read_from_db2(id: u32) -> Option<Question> {
_ => questions.nth((id - 1) as usize),
}
}
fn write_db2() {
fn write_db() {
let (tx, rx) = mpsc::channel::<Question>();
[
thread::spawn(move || zip_reader_task(tx)),
thread::spawn(move || db_writer2_task(rx)),
thread::spawn(move || db_writer_task(rx)),
]
.into_iter()
.for_each(|handle| handle.join().expect("thread panic"));
println!("all done");
}
fn db_writer2_task(rx: mpsc::Receiver<Question>) {
fn db_writer_task(rx: mpsc::Receiver<Question>) {
let writer_opts = db::WriterOpts::default();
let mut writer: db::Writer<Question> =
db::Writer::new(NEW_DB_FILENAME, writer_opts).expect("new db writer");

View File

@ -10,9 +10,6 @@ edition = "2021"
serde="1.0"
serde_derive="1.0"
serde_json="1.0"
ledb = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb"}
ledb-derive = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-derive"}
ledb-types = {git = "https://github.com/b4tman/ledb.git", rev="a646b90e", package="ledb-types"}
zip="0.6"
bincode = "^2.0.0-rc.2"
zstd = "^0.10"

View File

@ -3,7 +3,7 @@ use std::{
io::{self, Cursor, Read, Write},
marker::PhantomData,
path::Path,
sync::Arc
sync::Arc,
};
use memmap::{Mmap, MmapOptions};
@ -158,7 +158,7 @@ where
mmap: Mmap,
count: usize,
first_pos: LSize,
_t: Option<Arc<T>> // PhantomData replacement
_t: Option<Arc<T>>, // PhantomData replacement
}
impl<T> Reader<T>
@ -179,7 +179,7 @@ where
mmap,
count,
first_pos,
_t: None
_t: None,
})
}
@ -226,10 +226,6 @@ where
pub fn iter(&self) -> ReaderIter<'_, T> {
ReaderIter::new(self)
}
pub fn into_iter(self) -> ReaderIntoIter<T> {
ReaderIntoIter::new(self)
}
}
pub struct ReaderIter<'a, T>
@ -396,7 +392,7 @@ where
}
}
impl<T> IntoIterator for Reader<T>
impl<T> IntoIterator for Reader<T>
where
T: bincode::Decode,
{
@ -404,7 +400,7 @@ where
type IntoIter = ReaderIntoIter<Self::Item>;
fn into_iter(self) -> Self::IntoIter {
self.into_iter()
Self::IntoIter::new(self)
}
}
@ -534,9 +530,12 @@ mod test {
let cur_items = items.clone();
let cur_reader = Arc::clone(&reader);
thread::spawn(move || {
cur_items.into_iter().zip(cur_reader.iter()).for_each(|pair| {
assert_eq!(pair.0, pair.1);
});
cur_items
.into_iter()
.zip(cur_reader.iter())
.for_each(|pair| {
assert_eq!(pair.0, pair.1);
});
});
}
}

View File

@ -1,4 +1,3 @@
use ledb::Document;
use serde_derive::{Deserialize, Serialize};
use crate::source::{SourceQuestion, SourceQuestionsBatch};
@ -12,11 +11,8 @@ macro_rules! make {
),+ ,..$Target::default()}}
}
#[derive(
Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode,
)]
#[derive(Debug, Default, Clone, Serialize, Deserialize, bincode::Decode, bincode::Encode)]
pub struct BatchInfo {
#[document(primary)]
#[serde(default)]
pub filename: String,
#[serde(default)]
@ -45,14 +41,10 @@ pub struct BatchInfo {
pub rating: String,
}
#[derive(
Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode,
)]
#[derive(Debug, Default, Clone, Serialize, Deserialize, bincode::Decode, bincode::Encode)]
pub struct Question {
#[document(primary)]
#[serde(default)]
pub num: u32,
#[document(index)]
pub id: String,
pub description: String,
@ -84,7 +76,6 @@ pub struct Question {
pub source: String,
#[serde(default)]
pub rating: String,
#[document(nested)]
#[serde(default)]
pub batch_info: BatchInfo,
}