This commit is contained in:
parent
7efd03d624
commit
6a3b3647b3
11
Cargo.lock
generated
11
Cargo.lock
generated
@ -174,6 +174,7 @@ dependencies = [
|
|||||||
"ledb",
|
"ledb",
|
||||||
"ledb-derive",
|
"ledb-derive",
|
||||||
"ledb-types",
|
"ledb-types",
|
||||||
|
"memmap",
|
||||||
"rand",
|
"rand",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_derive",
|
"serde_derive",
|
||||||
@ -638,6 +639,16 @@ version = "2.5.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memmap"
|
||||||
|
version = "0.7.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "memoffset"
|
name = "memoffset"
|
||||||
version = "0.6.5"
|
version = "0.6.5"
|
||||||
|
@ -22,6 +22,7 @@ rand="0.8"
|
|||||||
clap = { version = "3.2.22", features = ["derive"] }
|
clap = { version = "3.2.22", features = ["derive"] }
|
||||||
bincode = "^2.0.0-rc.2"
|
bincode = "^2.0.0-rc.2"
|
||||||
zstd = "^0.10"
|
zstd = "^0.10"
|
||||||
|
memmap = "0.7.0"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
criterion = "0.4.0"
|
criterion = "0.4.0"
|
||||||
|
16
bench.txt
16
bench.txt
@ -6,18 +6,18 @@
|
|||||||
hyperfine -n print -n print2 -n zip-print -w 100 -m 100 ".\target\release\chgk_ledb.exe print 444" ".\target\release\chgk_ledb.exe print2 444" ".\target\release\chgk_ledb.exe zip-print 4 84"
|
hyperfine -n print -n print2 -n zip-print -w 100 -m 100 ".\target\release\chgk_ledb.exe print 444" ".\target\release\chgk_ledb.exe print2 444" ".\target\release\chgk_ledb.exe zip-print 4 84"
|
||||||
|
|
||||||
Benchmark 1: print
|
Benchmark 1: print
|
||||||
Time (mean ± σ): 20.0 ms ± 1.5 ms [User: 3.4 ms, System: 11.7 ms]
|
Time (mean ± σ): 20.0 ms ± 1.7 ms [User: 5.8 ms, System: 12.3 ms]
|
||||||
Range (min … max): 17.4 ms … 24.1 ms 100 runs
|
Range (min … max): 18.0 ms … 30.9 ms 100 runs
|
||||||
|
|
||||||
Benchmark 2: print2
|
Benchmark 2: print2
|
||||||
Time (mean ± σ): 19.4 ms ± 1.5 ms [User: 5.5 ms, System: 9.8 ms]
|
Time (mean ± σ): 19.1 ms ± 1.2 ms [User: 5.7 ms, System: 12.3 ms]
|
||||||
Range (min … max): 17.0 ms … 23.9 ms 100 runs
|
Range (min … max): 17.2 ms … 22.1 ms 100 runs
|
||||||
|
|
||||||
Benchmark 3: zip-print
|
Benchmark 3: zip-print
|
||||||
Time (mean ± σ): 40.2 ms ± 2.3 ms [User: 15.1 ms, System: 18.9 ms]
|
Time (mean ± σ): 41.1 ms ± 1.4 ms [User: 16.1 ms, System: 22.2 ms]
|
||||||
Range (min … max): 36.6 ms … 48.1 ms 100 runs
|
Range (min … max): 38.9 ms … 45.4 ms 100 runs
|
||||||
|
|
||||||
Summary
|
Summary
|
||||||
'print2' ran
|
'print2' ran
|
||||||
1.03 ± 0.11 times faster than 'print'
|
1.05 ± 0.11 times faster than 'print'
|
||||||
2.07 ± 0.20 times faster than 'zip-print'
|
2.16 ± 0.15 times faster than 'zip-print'
|
||||||
|
@ -3,7 +3,7 @@ extern crate criterion;
|
|||||||
|
|
||||||
#[path = "../src/db.rs"]
|
#[path = "../src/db.rs"]
|
||||||
mod db;
|
mod db;
|
||||||
use std::{path::PathBuf};
|
use std::path::PathBuf;
|
||||||
|
|
||||||
use db::{Reader, Writer, WriterOpts};
|
use db::{Reader, Writer, WriterOpts};
|
||||||
|
|
||||||
@ -88,6 +88,5 @@ fn config() -> Criterion {
|
|||||||
Criterion::default().sample_size(40)
|
Criterion::default().sample_size(40)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
criterion_group! {name=benches; config = config(); targets = db_read, db_write}
|
||||||
criterion_group!{name=benches; config = config(); targets = db_read, db_write}
|
|
||||||
criterion_main!(benches);
|
criterion_main!(benches);
|
||||||
|
43
src/db.rs
43
src/db.rs
@ -1,10 +1,12 @@
|
|||||||
use std::{
|
use std::{
|
||||||
fs,
|
fs,
|
||||||
io::{self, Cursor, Read, Seek, Write},
|
io::{self, Cursor, Read, Write},
|
||||||
marker::PhantomData,
|
marker::PhantomData,
|
||||||
path::Path,
|
path::Path,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use memmap::{Mmap, MmapOptions};
|
||||||
|
|
||||||
type LSize = u32;
|
type LSize = u32;
|
||||||
const LEN_SIZE: usize = std::mem::size_of::<LSize>();
|
const LEN_SIZE: usize = std::mem::size_of::<LSize>();
|
||||||
const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard();
|
const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard();
|
||||||
@ -152,7 +154,7 @@ pub struct Reader<T>
|
|||||||
where
|
where
|
||||||
T: bincode::Decode,
|
T: bincode::Decode,
|
||||||
{
|
{
|
||||||
input: io::BufReader<fs::File>,
|
mmap: Mmap,
|
||||||
count: usize,
|
count: usize,
|
||||||
first_pos: LSize,
|
first_pos: LSize,
|
||||||
_t: PhantomData<*const T>,
|
_t: PhantomData<*const T>,
|
||||||
@ -162,19 +164,18 @@ impl<T> Reader<T>
|
|||||||
where
|
where
|
||||||
T: bincode::Decode,
|
T: bincode::Decode,
|
||||||
{
|
{
|
||||||
pub fn new<P: AsRef<Path>>(path: P, buf_size: usize) -> Result<Self, String> {
|
pub fn new<P: AsRef<Path>>(path: P, _buf_size: usize) -> Result<Self, String> {
|
||||||
let input = fs::File::open(path).str_err()?;
|
let file = fs::File::open(path).str_err()?;
|
||||||
let mut input = io::BufReader::with_capacity(buf_size, input);
|
let mmap = unsafe { MmapOptions::new().map(&file).str_err()? };
|
||||||
|
|
||||||
// read first pos and records count
|
// read first pos and records count
|
||||||
let mut first_data: [u8; LEN_SIZE] = [0; LEN_SIZE];
|
let first_data: [u8; LEN_SIZE] = mmap[0..LEN_SIZE].try_into().str_err()?;
|
||||||
input.read_exact(&mut first_data).str_err()?;
|
|
||||||
let first_pos = LSize::from_le_bytes(first_data);
|
let first_pos = LSize::from_le_bytes(first_data);
|
||||||
let tab_len = (first_pos as usize) / LEN_SIZE;
|
let tab_len = (first_pos as usize) / LEN_SIZE;
|
||||||
let count = tab_len - 1;
|
let count = tab_len - 1;
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
input,
|
mmap,
|
||||||
count,
|
count,
|
||||||
first_pos,
|
first_pos,
|
||||||
_t: PhantomData,
|
_t: PhantomData,
|
||||||
@ -194,30 +195,24 @@ where
|
|||||||
let data_pos = if 0 == index {
|
let data_pos = if 0 == index {
|
||||||
self.first_pos
|
self.first_pos
|
||||||
} else {
|
} else {
|
||||||
let tab_pos: u64 = (index * LEN_SIZE).try_into().str_err()?;
|
let tab_pos: usize = index * LEN_SIZE;
|
||||||
let mut pos_curr_data: [u8; LEN_SIZE] = [0; LEN_SIZE];
|
let pos_curr_data: [u8; LEN_SIZE] = self.mmap[tab_pos..(tab_pos + LEN_SIZE)]
|
||||||
let cur_pos = self.input.stream_position().str_err()? as i64;
|
.try_into()
|
||||||
self.input
|
|
||||||
.seek_relative((tab_pos as i64) - cur_pos)
|
|
||||||
.str_err()?;
|
.str_err()?;
|
||||||
|
|
||||||
self.input.read_exact(&mut pos_curr_data).str_err()?;
|
|
||||||
LSize::from_le_bytes(pos_curr_data)
|
LSize::from_le_bytes(pos_curr_data)
|
||||||
};
|
} as usize;
|
||||||
|
|
||||||
// read next item pos
|
// read next item pos
|
||||||
let mut pos_next_data: [u8; LEN_SIZE] = [0; LEN_SIZE];
|
let next_pos: usize = (index + 1) * LEN_SIZE;
|
||||||
self.input.read_exact(&mut pos_next_data).str_err()?;
|
let pos_next_data: [u8; LEN_SIZE] = self.mmap[next_pos..(next_pos + LEN_SIZE)]
|
||||||
let data_pos_next = LSize::from_le_bytes(pos_next_data);
|
.try_into()
|
||||||
|
.str_err()?;
|
||||||
|
let data_pos_next = LSize::from_le_bytes(pos_next_data) as usize;
|
||||||
// calc item data length
|
// calc item data length
|
||||||
let data_len = data_pos_next - data_pos;
|
let data_len = data_pos_next - data_pos;
|
||||||
|
|
||||||
// read & unpack item data
|
// read & unpack item data
|
||||||
let cur_pos = self.input.stream_position().str_err()? as i64;
|
let reader = self.mmap[data_pos..data_pos_next].take(data_len as u64);
|
||||||
self.input
|
|
||||||
.seek_relative((data_pos as i64) - cur_pos)
|
|
||||||
.str_err()?;
|
|
||||||
let reader = self.input.by_ref().take(data_len as u64);
|
|
||||||
let data = zstd::decode_all(reader).str_err()?;
|
let data = zstd::decode_all(reader).str_err()?;
|
||||||
|
|
||||||
// decode item
|
// decode item
|
||||||
|
Loading…
Reference in New Issue
Block a user