From 7e6b513179b21d2733901c92ff1ff032e9d32cd0 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Wed, 23 Aug 2023 21:40:07 +0300 Subject: [PATCH] sync db: postcard --- lib/src/db.rs | 61 ++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/lib/src/db.rs b/lib/src/db.rs index 17d94f9..6b751b2 100644 --- a/lib/src/db.rs +++ b/lib/src/db.rs @@ -6,13 +6,13 @@ use std::{ sync::Arc, }; +use serde::{de::DeserializeOwned, Serialize}; + use memmap::{Mmap, MmapOptions}; type LSize = u32; const LEN_SIZE: usize = std::mem::size_of::(); -const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard(); -use crate::util::BincodeVecWriter; use crate::util::ErrorToString; pub struct WriterOpts { @@ -28,19 +28,19 @@ impl Default for WriterOpts { compress_lvl: 1, data_buf_size: 500 * 1024 * 1024, out_buf_size: 200 * 1024 * 1024, - current_buf_size: 100 * 1024, + current_buf_size: 20 * 1024, } } } pub struct Writer where - T: bincode::Encode, + T: Serialize, { out: io::BufWriter, data_buf: Cursor>, cur_buf_raw: Cursor>, - cur_buf_item: BincodeVecWriter, + cur_buf_item: Vec, table: Vec, compress_lvl: i32, _t: PhantomData>, @@ -48,7 +48,7 @@ where impl Writer where - T: bincode::Encode, + T: Serialize, { pub fn new>(path: P, opts: WriterOpts) -> Result { let out = fs::File::create(path).str_err()?; @@ -58,8 +58,7 @@ where let cur_buf_raw: Vec = Vec::with_capacity(opts.current_buf_size); let cur_buf_raw = Cursor::new(cur_buf_raw); - let cur_buf_item: Vec = Vec::with_capacity(opts.current_buf_size); - let cur_buf_item = BincodeVecWriter::new(cur_buf_item); + let cur_buf_item: Vec = vec![0; opts.current_buf_size]; let compress_lvl = opts.compress_lvl; @@ -83,19 +82,20 @@ where pub fn push_by_ref(&mut self, item: &T) -> Result<(), String> { let pos: LSize = self.data_buf.position() as LSize; - bincode::encode_into_writer(item, &mut self.cur_buf_item, BINCODE_CFG).str_err()?; + let cur_item_data = postcard::to_slice(item, self.cur_buf_item.as_mut_slice()).str_err()?; let mut zencoder = zstd::stream::raw::Encoder::new(self.compress_lvl).str_err()?; zencoder - .set_pledged_src_size(Some(self.cur_buf_item.len() as u64)) + .set_pledged_src_size(Some(cur_item_data.len() as u64)) .str_err()?; self.cur_buf_raw.set_position(0); let mut cur_buf_z = zstd::stream::zio::Writer::new(&mut self.cur_buf_raw, zencoder); - cur_buf_z.write_all(&self.cur_buf_item).str_err()?; + cur_buf_z.write_all(cur_item_data).str_err()?; cur_buf_z.finish().str_err()?; cur_buf_z.flush().str_err()?; - self.cur_buf_item.clear(); + + cur_item_data.fill(0); self.table.push(pos); let (cur_buf_raw, _) = cur_buf_z.into_inner(); @@ -150,7 +150,7 @@ where pub struct Reader where - T: bincode::Decode, + T: DeserializeOwned, { mmap: Mmap, count: usize, @@ -160,7 +160,7 @@ where impl Reader where - T: bincode::Decode, + T: DeserializeOwned, { pub fn new>(path: P, _buf_size: usize) -> Result { let file = fs::File::open(path).str_err()?; @@ -215,9 +215,9 @@ where let data = zstd::decode_all(reader).str_err()?; // decode item - let item: (T, usize) = bincode::decode_from_slice(&data, BINCODE_CFG).str_err()?; + let (item, _): (T, _) = postcard::take_from_bytes(&data).str_err()?; - Ok(item.0) + Ok(item) } pub fn iter(&self) -> ReaderIter<'_, T> { @@ -227,7 +227,7 @@ where pub struct ReaderIter<'a, T> where - T: bincode::Decode, + T: DeserializeOwned, { reader: &'a Reader, index: Option, @@ -235,7 +235,7 @@ where impl<'a, T> ReaderIter<'a, T> where - T: bincode::Decode, + T: DeserializeOwned, { fn new(reader: &'a Reader) -> Self { ReaderIter { @@ -247,7 +247,7 @@ where impl<'a, T> Iterator for ReaderIter<'a, T> where - T: bincode::Decode, + T: DeserializeOwned, { type Item = T; @@ -300,7 +300,7 @@ where impl<'a, T> ExactSizeIterator for ReaderIter<'a, T> where - T: bincode::Decode, + T: DeserializeOwned, { fn len(&self) -> usize { self.reader.len() @@ -309,7 +309,7 @@ where pub struct ReaderIntoIter where - T: bincode::Decode, + T: DeserializeOwned, { reader: Reader, index: Option, @@ -317,7 +317,7 @@ where impl ReaderIntoIter where - T: bincode::Decode, + T: DeserializeOwned, { fn new(reader: Reader) -> Self { Self { @@ -329,7 +329,7 @@ where impl Iterator for ReaderIntoIter where - T: bincode::Decode, + T: DeserializeOwned, { type Item = T; @@ -382,7 +382,7 @@ where impl ExactSizeIterator for ReaderIntoIter where - T: bincode::Decode, + T: DeserializeOwned, { fn len(&self) -> usize { self.reader.len() @@ -391,7 +391,7 @@ where impl IntoIterator for Reader where - T: bincode::Decode, + T: DeserializeOwned, { type Item = T; type IntoIter = ReaderIntoIter; @@ -404,9 +404,10 @@ where #[cfg(test)] mod test { use super::*; + use serde::Deserialize; use tempfile::tempdir; - #[derive(bincode::Encode, bincode::Decode, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] + #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] struct TestData { num: u64, test: String, @@ -427,7 +428,7 @@ mod test { compress_lvl: 1, data_buf_size: 10 * 1024 * 1024, out_buf_size: 10 * 1024 * 1024, - current_buf_size: 4096, + current_buf_size: 20 * 1024, }; let mut writer: Writer = Writer::new(&tmpfile, opts).expect("new writer"); @@ -454,7 +455,7 @@ mod test { compress_lvl: 1, data_buf_size: 10 * 1024 * 1024, out_buf_size: 10 * 1024 * 1024, - current_buf_size: 4096, + current_buf_size: 20 * 1024, }; let mut writer: Writer = Writer::new(&tmpfile, opts).expect("new writer"); @@ -480,7 +481,7 @@ mod test { compress_lvl: 1, data_buf_size: 10 * 1024 * 1024, out_buf_size: 10 * 1024 * 1024, - current_buf_size: 4096, + current_buf_size: 20 * 1024, }; let mut writer: Writer = Writer::new(&tmpfile, opts).expect("new writer"); @@ -509,7 +510,7 @@ mod test { compress_lvl: 1, data_buf_size: 10 * 1024 * 1024, out_buf_size: 10 * 1024 * 1024, - current_buf_size: 4096, + current_buf_size: 20 * 1024, }; let mut writer: Writer = Writer::new(&tmpfile, opts).expect("new writer");