db read with mmap
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
2022-11-09 20:57:21 +03:00
parent 7efd03d624
commit 6a3b3647b3
5 changed files with 41 additions and 35 deletions

View File

@@ -1,10 +1,12 @@
use std::{
fs,
io::{self, Cursor, Read, Seek, Write},
io::{self, Cursor, Read, Write},
marker::PhantomData,
path::Path,
};
use memmap::{Mmap, MmapOptions};
type LSize = u32;
const LEN_SIZE: usize = std::mem::size_of::<LSize>();
const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard();
@@ -152,7 +154,7 @@ pub struct Reader<T>
where
T: bincode::Decode,
{
input: io::BufReader<fs::File>,
mmap: Mmap,
count: usize,
first_pos: LSize,
_t: PhantomData<*const T>,
@@ -162,19 +164,18 @@ impl<T> Reader<T>
where
T: bincode::Decode,
{
pub fn new<P: AsRef<Path>>(path: P, buf_size: usize) -> Result<Self, String> {
let input = fs::File::open(path).str_err()?;
let mut input = io::BufReader::with_capacity(buf_size, input);
pub fn new<P: AsRef<Path>>(path: P, _buf_size: usize) -> Result<Self, String> {
let file = fs::File::open(path).str_err()?;
let mmap = unsafe { MmapOptions::new().map(&file).str_err()? };
// read first pos and records count
let mut first_data: [u8; LEN_SIZE] = [0; LEN_SIZE];
input.read_exact(&mut first_data).str_err()?;
let first_data: [u8; LEN_SIZE] = mmap[0..LEN_SIZE].try_into().str_err()?;
let first_pos = LSize::from_le_bytes(first_data);
let tab_len = (first_pos as usize) / LEN_SIZE;
let count = tab_len - 1;
Ok(Self {
input,
mmap,
count,
first_pos,
_t: PhantomData,
@@ -194,30 +195,24 @@ where
let data_pos = if 0 == index {
self.first_pos
} else {
let tab_pos: u64 = (index * LEN_SIZE).try_into().str_err()?;
let mut pos_curr_data: [u8; LEN_SIZE] = [0; LEN_SIZE];
let cur_pos = self.input.stream_position().str_err()? as i64;
self.input
.seek_relative((tab_pos as i64) - cur_pos)
let tab_pos: usize = index * LEN_SIZE;
let pos_curr_data: [u8; LEN_SIZE] = self.mmap[tab_pos..(tab_pos + LEN_SIZE)]
.try_into()
.str_err()?;
self.input.read_exact(&mut pos_curr_data).str_err()?;
LSize::from_le_bytes(pos_curr_data)
};
} as usize;
// read next item pos
let mut pos_next_data: [u8; LEN_SIZE] = [0; LEN_SIZE];
self.input.read_exact(&mut pos_next_data).str_err()?;
let data_pos_next = LSize::from_le_bytes(pos_next_data);
let next_pos: usize = (index + 1) * LEN_SIZE;
let pos_next_data: [u8; LEN_SIZE] = self.mmap[next_pos..(next_pos + LEN_SIZE)]
.try_into()
.str_err()?;
let data_pos_next = LSize::from_le_bytes(pos_next_data) as usize;
// calc item data length
let data_len = data_pos_next - data_pos;
// read & unpack item data
let cur_pos = self.input.stream_position().str_err()? as i64;
self.input
.seek_relative((data_pos as i64) - cur_pos)
.str_err()?;
let reader = self.input.by_ref().take(data_len as u64);
let reader = self.mmap[data_pos..data_pos_next].take(data_len as u64);
let data = zstd::decode_all(reader).str_err()?;
// decode item