test fmt 1
This commit is contained in:
parent
2996aefa7b
commit
5a4086cfdf
1
.gitignore
vendored
1
.gitignore
vendored
@ -5,3 +5,4 @@ test?.zip
|
|||||||
json.zip
|
json.zip
|
||||||
/exp
|
/exp
|
||||||
/.vscode
|
/.vscode
|
||||||
|
test*.bin
|
||||||
|
27
Cargo.lock
generated
27
Cargo.lock
generated
@ -58,6 +58,25 @@ version = "1.0.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8a32fd6af2b5827bce66c29053ba0e7c42b9dcab01835835058558c10851a46b"
|
checksum = "8a32fd6af2b5827bce66c29053ba0e7c42b9dcab01835835058558c10851a46b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bincode"
|
||||||
|
version = "2.0.0-rc.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7bb50c5a2ef4b9b1e7ae73e3a73b52ea24b20312d629f9c4df28260b7ad2c3c4"
|
||||||
|
dependencies = [
|
||||||
|
"bincode_derive",
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bincode_derive"
|
||||||
|
version = "2.0.0-rc.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0a45a23389446d2dd25dc8e73a7a3b3c43522b630cac068927f0649d43d719d2"
|
||||||
|
dependencies = [
|
||||||
|
"virtue",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bitflags"
|
name = "bitflags"
|
||||||
version = "0.9.1"
|
version = "0.9.1"
|
||||||
@ -131,6 +150,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
|||||||
name = "chgk_ledb"
|
name = "chgk_ledb"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"bincode",
|
||||||
"clap",
|
"clap",
|
||||||
"ledb",
|
"ledb",
|
||||||
"ledb-derive",
|
"ledb-derive",
|
||||||
@ -140,6 +160,7 @@ dependencies = [
|
|||||||
"serde_derive",
|
"serde_derive",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"zip",
|
"zip",
|
||||||
|
"zstd",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@ -825,6 +846,12 @@ version = "0.9.4"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
|
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "virtue"
|
||||||
|
version = "0.0.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7b60dcd6a64dd45abf9bd426970c9843726da7fc08f44cd6fcebf68c21220a63"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wasi"
|
name = "wasi"
|
||||||
version = "0.11.0+wasi-snapshot-preview1"
|
version = "0.11.0+wasi-snapshot-preview1"
|
||||||
|
@ -16,6 +16,8 @@ ledb-types="0.4"
|
|||||||
zip="0.6"
|
zip="0.6"
|
||||||
rand="0.8"
|
rand="0.8"
|
||||||
clap = { version = "3.2.22", features = ["derive"] }
|
clap = { version = "3.2.22", features = ["derive"] }
|
||||||
|
bincode = "^2.0.0-rc.2"
|
||||||
|
zstd = "^0.10"
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
opt-level = 3
|
opt-level = 3
|
||||||
|
20
bench.txt
Normal file
20
bench.txt
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
hyperfine -n print -n print2 -n zip-print -w 100 -m 100 ".\target\release\chgk_ledb.exe print 444" ".\target\release\chgk_ledb.exe print2 444" ".\target\release\chgk_ledb.exe zip-print 444 4"
|
||||||
|
|
||||||
|
Benchmark 1: print
|
||||||
|
Time (mean ± σ): 19.9 ms ± 1.8 ms [User: 5.3 ms, System: 14.3 ms]
|
||||||
|
Range (min … max): 17.4 ms … 26.2 ms 100 runs
|
||||||
|
|
||||||
|
Benchmark 2: print2
|
||||||
|
Time (mean ± σ): 79.1 ms ± 5.5 ms [User: 10.4 ms, System: 61.4 ms]
|
||||||
|
Range (min … max): 71.6 ms … 122.1 ms 100 runs
|
||||||
|
|
||||||
|
Warning: Statistical outliers were detected. Consider re-running this benchmark on a quiet PC without any interferences from other programs. It might help to use the '--warmup' or '--prepare' options.
|
||||||
|
|
||||||
|
Benchmark 3: zip-print
|
||||||
|
Time (mean ± σ): 38.0 ms ± 2.2 ms [User: 10.0 ms, System: 23.4 ms]
|
||||||
|
Range (min … max): 34.3 ms … 43.1 ms 100 runs
|
||||||
|
|
||||||
|
Summary
|
||||||
|
'print' ran
|
||||||
|
1.91 ± 0.20 times faster than 'zip-print'
|
||||||
|
3.98 ± 0.45 times faster than 'print2'
|
109
src/main.rs
109
src/main.rs
@ -9,6 +9,7 @@ extern crate zip;
|
|||||||
use clap::{Parser, Subcommand};
|
use clap::{Parser, Subcommand};
|
||||||
use rand::seq::IteratorRandom;
|
use rand::seq::IteratorRandom;
|
||||||
|
|
||||||
|
use std::io::{Read, Write};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
use std::{fs, io, sync::mpsc, thread};
|
use std::{fs, io, sync::mpsc, thread};
|
||||||
@ -38,6 +39,11 @@ enum Command {
|
|||||||
#[clap(value_parser, default_value = "0")]
|
#[clap(value_parser, default_value = "0")]
|
||||||
num: usize,
|
num: usize,
|
||||||
},
|
},
|
||||||
|
Write2,
|
||||||
|
Print2 {
|
||||||
|
#[clap(value_parser, default_value = "0")]
|
||||||
|
id: u32,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Parser, Debug)]
|
#[derive(Parser, Debug)]
|
||||||
@ -210,6 +216,11 @@ fn main() {
|
|||||||
let get_question = Box::new(|| read_from_zip(*file_num, *num));
|
let get_question = Box::new(|| read_from_zip(*file_num, *num));
|
||||||
Box::new(|| print_question_from(get_question))
|
Box::new(|| print_question_from(get_question))
|
||||||
}
|
}
|
||||||
|
Command::Write2 => Box::new(write_db2),
|
||||||
|
Command::Print2 { id } => {
|
||||||
|
let get_question = Box::new(|| read_from_db2(*id));
|
||||||
|
Box::new(|| print_question_from(get_question))
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if args.measure {
|
if args.measure {
|
||||||
@ -218,3 +229,101 @@ fn main() {
|
|||||||
|
|
||||||
action();
|
action();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(bincode::Decode, bincode::Encode)]
|
||||||
|
struct VEntry {
|
||||||
|
pos: u32,
|
||||||
|
len: u16,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_from_db2(id: u32) -> Option<Question> {
|
||||||
|
use std::io::Cursor;
|
||||||
|
const LEN_SIZE: usize = std::mem::size_of::<u64>();
|
||||||
|
let cfg = bincode::config::standard().with_fixed_int_encoding();
|
||||||
|
|
||||||
|
let input = fs::File::open("test.bin").expect("open input");
|
||||||
|
let mut input = std::io::BufReader::with_capacity(100 * 1024 * 1024, input);
|
||||||
|
|
||||||
|
let mut len_data: [u8; LEN_SIZE] = [0; LEN_SIZE];
|
||||||
|
input.read_exact(&mut len_data).expect("read len");
|
||||||
|
let len = u64::from_le_bytes(len_data) as usize;
|
||||||
|
|
||||||
|
//println!("read len done");
|
||||||
|
|
||||||
|
let mut zdata = vec![0u8; len];
|
||||||
|
input.read_exact(&mut zdata[..len]).expect("read ztab");
|
||||||
|
let tab_data = zstd::decode_all(Cursor::new(zdata)).expect("zstd decode table");
|
||||||
|
let tab: (Vec<VEntry>, usize) =
|
||||||
|
bincode::decode_from_slice(&tab_data, cfg).expect("bincode decode tab");
|
||||||
|
let tab = tab.0;
|
||||||
|
|
||||||
|
//println!("read tab done");
|
||||||
|
|
||||||
|
let choosen = tab.get(id as usize).expect("get by id");
|
||||||
|
input.seek_relative(choosen.pos as i64).expect("q seek");
|
||||||
|
let reader = input.take(choosen.len as u64);
|
||||||
|
let data = zstd::decode_all(reader).expect("zstd decode data");
|
||||||
|
|
||||||
|
let question: (Question, usize) =
|
||||||
|
bincode::decode_from_slice(&data, cfg).expect("bincode decode q");
|
||||||
|
let question = question.0;
|
||||||
|
|
||||||
|
//println!("read done");
|
||||||
|
|
||||||
|
Some(question)
|
||||||
|
}
|
||||||
|
fn write_db2() {
|
||||||
|
let (tx, rx) = mpsc::channel::<Question>();
|
||||||
|
[
|
||||||
|
thread::spawn(move || reader_task(tx)),
|
||||||
|
thread::spawn(move || db_writer2_task(rx)),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.for_each(|handle| handle.join().expect("thread panic"));
|
||||||
|
println!("all done");
|
||||||
|
}
|
||||||
|
fn db_writer2_task(rx: mpsc::Receiver<Question>) {
|
||||||
|
const LEN_SIZE: usize = std::mem::size_of::<u64>();
|
||||||
|
|
||||||
|
let cfg = bincode::config::standard().with_fixed_int_encoding();
|
||||||
|
let mut table: Vec<VEntry> = vec![];
|
||||||
|
|
||||||
|
let buf_data: Vec<u8> = Vec::with_capacity(500 * 1024 * 1024);
|
||||||
|
use std::io::Cursor;
|
||||||
|
let mut buf = Cursor::new(buf_data);
|
||||||
|
let mut pos: u32 = 0;
|
||||||
|
|
||||||
|
rx.into_iter().for_each(|q| {
|
||||||
|
let data = bincode::encode_to_vec(q, cfg).expect("bincode q encode");
|
||||||
|
let data = zstd::encode_all(Cursor::new(data), 3).expect("zstd q encode");
|
||||||
|
let len = buf.write(&data).expect("write question");
|
||||||
|
table.push(VEntry {
|
||||||
|
pos,
|
||||||
|
len: len.try_into().expect("len try_into len"),
|
||||||
|
});
|
||||||
|
pos += len as u32;
|
||||||
|
});
|
||||||
|
|
||||||
|
buf.set_position(0);
|
||||||
|
|
||||||
|
println!("zbuf done");
|
||||||
|
|
||||||
|
let tab_data = bincode::encode_to_vec(&table, cfg).expect("encode table");
|
||||||
|
let zdata = zstd::encode_all(Cursor::new(tab_data), 3).expect("zstd enc table");
|
||||||
|
let zlen = zdata.len() as u64;
|
||||||
|
|
||||||
|
println!("z tab done");
|
||||||
|
|
||||||
|
let out = fs::File::create("test.bin").expect("out create");
|
||||||
|
let mut out = std::io::BufWriter::with_capacity(500 * 1024 * 1024, out);
|
||||||
|
let len_writed = out.write(&zlen.to_le_bytes()).expect("write zlen");
|
||||||
|
assert_eq!(len_writed, LEN_SIZE);
|
||||||
|
let ztab_writed = out.write(&zdata).expect("write tab zdata");
|
||||||
|
assert_eq!(ztab_writed, zdata.len());
|
||||||
|
drop(zdata);
|
||||||
|
std::io::copy(&mut buf, &mut out).expect("copy z buf");
|
||||||
|
drop(buf);
|
||||||
|
out.flush().expect("out flush");
|
||||||
|
|
||||||
|
println!("write done");
|
||||||
|
}
|
||||||
|
@ -12,7 +12,9 @@ macro_rules! make {
|
|||||||
),+ ,..$Target::default()}}
|
),+ ,..$Target::default()}}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, Document)]
|
#[derive(
|
||||||
|
Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode,
|
||||||
|
)]
|
||||||
pub struct BatchInfo {
|
pub struct BatchInfo {
|
||||||
#[document(primary)]
|
#[document(primary)]
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@ -43,7 +45,9 @@ pub struct BatchInfo {
|
|||||||
pub rating: String,
|
pub rating: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, Document)]
|
#[derive(
|
||||||
|
Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode,
|
||||||
|
)]
|
||||||
pub struct Question {
|
pub struct Question {
|
||||||
#[document(primary)]
|
#[document(primary)]
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
Loading…
Reference in New Issue
Block a user