b4tman
/
chgk_ledb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use BincodeVecWriter in (sync) db
continuous-integration/drone/push Build is passing Details 

for reusing current item buf
This commit is contained in:
Dmitry Belyaev 2023-08-14 15:54:26 +03:00
parent e074108560
commit 645b654dac
Signed by: b4tman
GPG Key ID: 41A00BF15EA7E5F3
1 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lib/src/db.rs
View File

@ -12,6 +12,7 @@ type LSize = u32;
const LEN_SIZE: usize = std::mem::size_of::<LSize>();
const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard();
use crate::util::BincodeVecWriter;
use crate::util::ErrorToString;
pub struct WriterOpts {
@ -39,6 +40,7 @@ where
out: io::BufWriter<fs::File>,
data_buf: Cursor<Vec<u8>>,
cur_buf_raw: Cursor<Vec<u8>>,
cur_buf_item: BincodeVecWriter,
table: Vec<LSize>,
compress_lvl: i32,
_t: PhantomData<Arc<T>>,
@ -56,6 +58,8 @@ where
let cur_buf_raw: Vec<u8> = Vec::with_capacity(opts.current_buf_size);
let cur_buf_raw = Cursor::new(cur_buf_raw);
let cur_buf_item: Vec<u8> = Vec::with_capacity(opts.current_buf_size);
let cur_buf_item = BincodeVecWriter::new(cur_buf_item);
let compress_lvl = opts.compress_lvl;
@ -65,6 +69,7 @@ where
out,
data_buf,
cur_buf_raw,
cur_buf_item,
table,
compress_lvl,
_t: PhantomData,
@ -74,18 +79,19 @@ where
pub fn push(&mut self, item: T) -> Result<(), String> {
let pos: LSize = self.data_buf.position() as LSize;
let item_data = bincode::encode_to_vec(item, BINCODE_CFG).str_err()?;
bincode::encode_into_writer(item, &mut self.cur_buf_item, BINCODE_CFG).str_err()?;
let mut zencoder = zstd::stream::raw::Encoder::new(self.compress_lvl).str_err()?;
zencoder
.set_pledged_src_size(Some(item_data.len() as u64))
.set_pledged_src_size(Some(self.cur_buf_item.len() as u64))
.str_err()?;
self.cur_buf_raw.set_position(0);
let mut cur_buf_z = zstd::stream::zio::Writer::new(&mut self.cur_buf_raw, zencoder);
cur_buf_z.write_all(&item_data).str_err()?;
cur_buf_z.write_all(&self.cur_buf_item).str_err()?;
cur_buf_z.finish().str_err()?;
cur_buf_z.flush().str_err()?;
self.cur_buf_item.clear();
self.table.push(pos);
let (cur_buf_raw, _) = cur_buf_z.into_inner();