questions: remove "skip_serializing_if"

to fix ser/de to postcard err "Hit the end of buffer, expected more data" more info: https://github.com/apache/arrow-rs/issues/3082
apps: don't use bincode
2023-08-23 23:50:31 +03:00 · 2023-08-23 22:40:54 +03:00 · 2023-08-23 22:33:59 +03:00 · 2023-08-23 22:33:36 +03:00 · 2023-08-23 21:40:07 +03:00 · 2023-08-23 21:39:23 +03:00
10 changed files with 265 additions and 237 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -207,25 +207,6 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a32fd6af2b5827bce66c29053ba0e7c42b9dcab01835835058558c10851a46b"

-[[package]]
-name = "bincode"
-version = "2.0.0-rc.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f11ea1a0346b94ef188834a65c068a03aec181c94896d481d7a0a40d85b0ce95"
-dependencies = [
- "bincode_derive",
- "serde",
-]
-
-[[package]]
-name = "bincode_derive"
-version = "2.0.0-rc.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e30759b3b99a1b802a7a3aa21c85c3ded5c28e1c83170d82d70f08bbf7f3e4c"
-dependencies = [
- "virtue",
-]
-
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@ -311,7 +292,6 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 name = "chgk_ledb"
 version = "1.1.0"
 dependencies = [
- "bincode",
 "chgk_ledb_lib",
 "clap 4.3.16",
 "criterion 0.4.0",
@ -329,7 +309,6 @@ version = "1.1.0"
 dependencies = [
 "async-compression 0.4.1",
 "async_zip",
- "bincode",
 "chgk_ledb_lib",
 "clap 4.3.16",
 "criterion 0.5.1",
@ -351,7 +330,6 @@ dependencies = [
 "async-compression 0.4.1",
 "async-stream",
 "async_zip",
- "bincode",
 "fmmap",
 "futures",
 "futures-core",
@ -359,6 +337,7 @@ dependencies = [
 "insta",
 "memmap",
 "pin-project",
+ "postcard",
 "serde",
 "serde_derive",
 "serde_json",
@ -467,6 +446,12 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"

+[[package]]
+name = "cobs"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15"
+
 [[package]]
 name = "colorchoice"
 version = "1.0.0"
@ -1248,6 +1233,16 @@ dependencies = [
 "plotters-backend",
 ]

+[[package]]
+name = "postcard"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9ee729232311d3cd113749948b689627618133b1c5012b77342c1950b25eaeb"
+dependencies = [
+ "cobs",
+ "serde",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@ -1701,12 +1696,6 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"

-[[package]]
-name = "virtue"
-version = "0.0.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9dcc60c0624df774c82a0ef104151231d37da4962957d691c011c852b2473314"
-
 [[package]]
 name = "walkdir"
 version = "2.3.3"
--- a/app/Cargo.toml
+++ b/app/Cargo.toml
@ -23,6 +23,5 @@ clap = { version = "4.2.7", features = ["derive"] }
 [dev-dependencies]
 criterion = "0.4.0"
 tempfile = "3.3"
-bincode = "^2.0.0-rc.2"
 serde="1.0"
 serde_derive="1.0"
--- a/app/benches/db_bench.rs
+++ b/app/benches/db_bench.rs
@ -1,6 +1,5 @@
 #[macro_use]
 extern crate criterion;
-extern crate bincode;
 extern crate serde;
 extern crate serde_derive;
 extern crate serde_json;
@ -16,18 +15,7 @@ use tempfile::{tempdir, NamedTempFile};

 use serde_derive::{Deserialize, Serialize};

-#[derive(
-    bincode::Encode,
-    bincode::Decode,
-    Clone,
-    Debug,
-    PartialEq,
-    Eq,
-    PartialOrd,
-    Ord,
-    Serialize,
-    Deserialize,
-)]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
 struct TestData {
    num1: u64,
    num2: u64,
--- a/app_async/Cargo.toml
+++ b/app_async/Cargo.toml
@ -33,7 +33,6 @@ futures = "0.3"
 [dev-dependencies]
 criterion = { version = "0.5.1", features = ["async_tokio"]}
 tempfile = "3.3"
-bincode = "^2.0.0-rc.2"
 serde="1.0"
 serde_derive="1.0"
 futures = "0.3"
--- a/app_async/benches/async_bench.rs
+++ b/app_async/benches/async_bench.rs
@ -1,6 +1,5 @@
 #[macro_use]
 extern crate criterion;
-extern crate bincode;
 extern crate serde;
 extern crate serde_derive;
 extern crate tempfile;
@ -16,18 +15,7 @@ use tempfile::{tempdir, NamedTempFile};

 use serde_derive::{Deserialize, Serialize};

-#[derive(
-    bincode::Encode,
-    bincode::Decode,
-    Clone,
-    Debug,
-    PartialEq,
-    Eq,
-    PartialOrd,
-    Ord,
-    Serialize,
-    Deserialize,
-)]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
 struct TestData {
    num1: u64,
    num2: u64,
--- a/lib/Cargo.toml
+++ b/lib/Cargo.toml
@ -12,40 +12,16 @@ description = "Библиотека для доступа к файлу базы
 [features]
 default = []
 sync = ["zstd", "memmap"]
-async = [
-    "futures",
-    "futures-core",
-    "futures-util",
-    "fmmap",
-    "tokio",
-    "async-compression",
-    "async-stream",
-    "pin-project",
-]
+async = ["futures", "futures-core", "futures-util", "fmmap", "tokio", "async-compression", "async-stream", "pin-project"]
 source = ["zip"]
-source_async = [
-    "async_zip",
-    "tokio",
-    "futures",
-    "futures-core",
-    "futures-util",
-    "async-stream",
-]
+source_async = ["async_zip", "tokio", "futures", "futures-core", "futures-util", "async-stream"]
 convert = ["zip"]
-convert_async = [
-    "futures",
-    "futures-core",
-    "futures-util",
-    "async-stream",
-    "async_zip",
-    "tokio",
-]
+convert_async = ["futures", "futures-core", "futures-util", "async-stream", "async_zip", "tokio"]

 [dependencies]
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
-bincode = "^2.0.0-rc.2"
 zip = { version = "0.6", optional = true }
 async_zip = { version = "0.0.15" , features = [
    "zstd",
@ -70,6 +46,7 @@ async-stream = { version = "0.3", optional = true }
 zstd = { version = "^0.12", default-features = false, optional = true }
 memmap = { version = "0.7.0", optional = true }
 pin-project = { version = "1.1.3", optional = true }
+postcard = { version = "1.0.6", default-features = false }

 [dev-dependencies]
 insta = { version = "1.31.0", features = ["yaml"] }
--- a/lib/src/async_db.rs
+++ b/lib/src/async_db.rs
@ -24,9 +24,9 @@ use fmmap::tokio::{AsyncMmapFile, AsyncMmapFileExt, AsyncOptions};

 type LSize = u32;
 const LEN_SIZE: usize = std::mem::size_of::<LSize>();
-const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard();

-use crate::util::BincodeVecWriter;
+use serde::{de::DeserializeOwned, Serialize};
+
 use crate::util::ErrorToString;

 pub struct WriterOpts {
@ -42,18 +42,18 @@ impl Default for WriterOpts {
            compress_lvl: Level::Default,
            data_buf_size: 500 * 1024 * 1024,
            out_buf_size: 200 * 1024 * 1024,
-            current_buf_size: 100 * 1024,
+            current_buf_size: 1024 * 1024,
        }
    }
 }

 pub struct Writer<T>
 where
-    T: bincode::Encode,
+    T: Serialize,
 {
    out: io::BufWriter<fs::File>,
    data_buf: Vec<u8>,
-    cur_buf_item: BincodeVecWriter,
+    cur_buf_item: Vec<u8>,
    table: Vec<LSize>,
    compress_lvl: Level,
    _t: PhantomData<Arc<T>>,
@ -61,14 +61,13 @@ where

 impl<T> Writer<T>
 where
-    T: bincode::Encode,
+    T: Serialize,
 {
    pub async fn new<P: AsRef<Path>>(path: P, opts: WriterOpts) -> Result<Self, String> {
        let out = fs::File::create(path).await.str_err()?;
        let out = io::BufWriter::with_capacity(opts.out_buf_size, out);
        let data_buf: Vec<u8> = Vec::with_capacity(opts.data_buf_size);
-        let cur_buf_item: Vec<u8> = Vec::with_capacity(opts.current_buf_size);
-        let cur_buf_item = BincodeVecWriter::new(cur_buf_item);
+        let cur_buf_item: Vec<u8> = vec![0; opts.current_buf_size];

        let compress_lvl = opts.compress_lvl;

@ -88,16 +87,16 @@ where
        self.push_by_ref(&item).await
    }

+    #[allow(clippy::useless_asref)]
    pub async fn push_by_ref(&mut self, item: &T) -> Result<(), String> {
        let pos: LSize = self.data_buf.len() as LSize;
+        let cur_item_data = postcard::to_slice(item, self.cur_buf_item.as_mut_slice()).str_err()?;

-        bincode::encode_into_writer(item, &mut self.cur_buf_item, BINCODE_CFG).str_err()?;
-
-        let mut zencoder = ZstdEncoder::with_quality(&self.cur_buf_item[..], self.compress_lvl);
+        let mut zencoder = ZstdEncoder::with_quality(cur_item_data.as_ref(), self.compress_lvl);
        io::copy(&mut zencoder, &mut self.data_buf)
            .await
            .str_err()?;
-        self.cur_buf_item.clear();
+        cur_item_data.fill(0);

        self.table.push(pos);

@ -159,7 +158,7 @@ use pin_project::pin_project;
 /// only work when ..push.poll() returns Ready immediately
 pub struct WriterSink<'a, T>
 where
-    T: bincode::Encode,
+    T: Serialize,
 {
    #[pin]
    writer: &'a mut Writer<T>,
@ -168,7 +167,7 @@ where

 impl<'a, T> Sink<T> for WriterSink<'a, T>
 where
-    T: bincode::Encode,
+    T: Serialize,
 {
    type Error = String;

@ -213,7 +212,7 @@ where

 pub struct Reader<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    mmap: AsyncMmapFile,
    count: usize,
@ -223,7 +222,7 @@ where

 impl<T> Reader<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    pub async fn new<P: AsRef<Path>>(path: P) -> Result<Self, String> {
        let mmap = AsyncOptions::new()
@ -292,10 +291,10 @@ where
        decoder.read_to_end(data_buf).await.str_err()?;

        // decode item
-        let item: (T, usize) = bincode::decode_from_slice(data_buf, BINCODE_CFG).str_err()?;
+        let (item, _): (T, _) = postcard::take_from_bytes(data_buf).str_err()?;

        data_buf.clear();
-        Ok(item.0)
+        Ok(item)
    }

    /// get item at index
@ -311,7 +310,7 @@ where

 pub struct ReaderStream<'a, T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    reader: &'a Reader<T>,
    index: Option<usize>,
@ -319,7 +318,7 @@ where

 impl<'a, T> ReaderStream<'a, T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    fn new(reader: &'a Reader<T>) -> Self {
        ReaderStream {
@ -331,7 +330,7 @@ where

 impl<'a, T> Stream for ReaderStream<'a, T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    type Item = T;

@ -375,7 +374,7 @@ where

 pub struct BufReader<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    inner: Reader<T>,
    buf: Vec<u8>,
@ -383,7 +382,7 @@ where

 impl<T> BufReader<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    pub async fn new<P: AsRef<Path>>(path: P, buf_size: usize) -> Result<Self, String> {
        match Reader::<T>::new(path).await {
@ -410,7 +409,7 @@ where

 impl<T> From<Reader<T>> for BufReader<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    fn from(inner: Reader<T>) -> Self {
        Self {
@ -422,7 +421,7 @@ where

 impl<T> From<BufReader<T>> for Reader<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    fn from(value: BufReader<T>) -> Self {
        value.into_inner()
@ -431,7 +430,7 @@ where

 impl<T> Deref for BufReader<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    type Target = Reader<T>;
    fn deref(&self) -> &Self::Target {
@ -441,7 +440,7 @@ where

 pub struct BufReaderStream<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    reader: BufReader<T>,
    index: Option<usize>,
@ -449,7 +448,7 @@ where

 impl<T> BufReaderStream<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    fn new(reader: BufReader<T>) -> Self {
        BufReaderStream {
@ -473,7 +472,7 @@ where

 impl<T> Stream for BufReaderStream<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    type Item = T;

@ -516,9 +515,10 @@ where
 mod test {
    use super::*;
    use core::fmt::Debug;
+    use serde_derive::Deserialize;
    use tempfile::tempdir;

-    #[derive(bincode::Encode, bincode::Decode, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+    #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
    struct TestData {
        num: u64,
        test: String,
--- a/lib/src/db.rs
+++ b/lib/src/db.rs
@ -6,13 +6,13 @@ use std::{
    sync::Arc,
 };

+use serde::{de::DeserializeOwned, Serialize};
+
 use memmap::{Mmap, MmapOptions};

 type LSize = u32;
 const LEN_SIZE: usize = std::mem::size_of::<LSize>();
-const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard();

-use crate::util::BincodeVecWriter;
 use crate::util::ErrorToString;

 pub struct WriterOpts {
@ -28,19 +28,19 @@ impl Default for WriterOpts {
            compress_lvl: 1,
            data_buf_size: 500 * 1024 * 1024,
            out_buf_size: 200 * 1024 * 1024,
-            current_buf_size: 100 * 1024,
+            current_buf_size: 20 * 1024,
        }
    }
 }

 pub struct Writer<T>
 where
-    T: bincode::Encode,
+    T: Serialize,
 {
    out: io::BufWriter<fs::File>,
    data_buf: Cursor<Vec<u8>>,
    cur_buf_raw: Cursor<Vec<u8>>,
-    cur_buf_item: BincodeVecWriter,
+    cur_buf_item: Vec<u8>,
    table: Vec<LSize>,
    compress_lvl: i32,
    _t: PhantomData<Arc<T>>,
@ -48,7 +48,7 @@ where

 impl<T> Writer<T>
 where
-    T: bincode::Encode,
+    T: Serialize,
 {
    pub fn new<P: AsRef<Path>>(path: P, opts: WriterOpts) -> Result<Self, String> {
        let out = fs::File::create(path).str_err()?;
@ -58,8 +58,7 @@ where

        let cur_buf_raw: Vec<u8> = Vec::with_capacity(opts.current_buf_size);
        let cur_buf_raw = Cursor::new(cur_buf_raw);
-        let cur_buf_item: Vec<u8> = Vec::with_capacity(opts.current_buf_size);
-        let cur_buf_item = BincodeVecWriter::new(cur_buf_item);
+        let cur_buf_item: Vec<u8> = vec![0; opts.current_buf_size];

        let compress_lvl = opts.compress_lvl;

@ -83,19 +82,20 @@ where
    pub fn push_by_ref(&mut self, item: &T) -> Result<(), String> {
        let pos: LSize = self.data_buf.position() as LSize;

-        bincode::encode_into_writer(item, &mut self.cur_buf_item, BINCODE_CFG).str_err()?;
+        let cur_item_data = postcard::to_slice(item, self.cur_buf_item.as_mut_slice()).str_err()?;

        let mut zencoder = zstd::stream::raw::Encoder::new(self.compress_lvl).str_err()?;
        zencoder
-            .set_pledged_src_size(Some(self.cur_buf_item.len() as u64))
+            .set_pledged_src_size(Some(cur_item_data.len() as u64))
            .str_err()?;

        self.cur_buf_raw.set_position(0);
        let mut cur_buf_z = zstd::stream::zio::Writer::new(&mut self.cur_buf_raw, zencoder);
-        cur_buf_z.write_all(&self.cur_buf_item).str_err()?;
+        cur_buf_z.write_all(cur_item_data).str_err()?;
        cur_buf_z.finish().str_err()?;
        cur_buf_z.flush().str_err()?;
-        self.cur_buf_item.clear();
+
+        cur_item_data.fill(0);

        self.table.push(pos);
        let (cur_buf_raw, _) = cur_buf_z.into_inner();
@ -150,7 +150,7 @@ where

 pub struct Reader<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    mmap: Mmap,
    count: usize,
@ -160,7 +160,7 @@ where

 impl<T> Reader<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    pub fn new<P: AsRef<Path>>(path: P, _buf_size: usize) -> Result<Self, String> {
        let file = fs::File::open(path).str_err()?;
@ -215,9 +215,9 @@ where
        let data = zstd::decode_all(reader).str_err()?;

        // decode item
-        let item: (T, usize) = bincode::decode_from_slice(&data, BINCODE_CFG).str_err()?;
+        let (item, _): (T, _) = postcard::take_from_bytes(&data).str_err()?;

-        Ok(item.0)
+        Ok(item)
    }

    pub fn iter(&self) -> ReaderIter<'_, T> {
@ -227,7 +227,7 @@ where

 pub struct ReaderIter<'a, T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    reader: &'a Reader<T>,
    index: Option<usize>,
@ -235,7 +235,7 @@ where

 impl<'a, T> ReaderIter<'a, T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    fn new(reader: &'a Reader<T>) -> Self {
        ReaderIter {
@ -247,7 +247,7 @@ where

 impl<'a, T> Iterator for ReaderIter<'a, T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    type Item = T;

@ -300,7 +300,7 @@ where

 impl<'a, T> ExactSizeIterator for ReaderIter<'a, T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    fn len(&self) -> usize {
        self.reader.len()
@ -309,7 +309,7 @@ where

 pub struct ReaderIntoIter<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    reader: Reader<T>,
    index: Option<usize>,
@ -317,7 +317,7 @@ where

 impl<T> ReaderIntoIter<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    fn new(reader: Reader<T>) -> Self {
        Self {
@ -329,7 +329,7 @@ where

 impl<T> Iterator for ReaderIntoIter<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    type Item = T;

@ -382,7 +382,7 @@ where

 impl<T> ExactSizeIterator for ReaderIntoIter<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    fn len(&self) -> usize {
        self.reader.len()
@ -391,7 +391,7 @@ where

 impl<T> IntoIterator for Reader<T>
 where
-    T: bincode::Decode,
+    T: DeserializeOwned,
 {
    type Item = T;
    type IntoIter = ReaderIntoIter<Self::Item>;
@ -404,9 +404,10 @@ where
 #[cfg(test)]
 mod test {
    use super::*;
+    use serde::Deserialize;
    use tempfile::tempdir;

-    #[derive(bincode::Encode, bincode::Decode, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+    #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
    struct TestData {
        num: u64,
        test: String,
@ -427,7 +428,7 @@ mod test {
            compress_lvl: 1,
            data_buf_size: 10 * 1024 * 1024,
            out_buf_size: 10 * 1024 * 1024,
-            current_buf_size: 4096,
+            current_buf_size: 20 * 1024,
        };
        let mut writer: Writer<TestData> = Writer::new(&tmpfile, opts).expect("new writer");

@ -454,7 +455,7 @@ mod test {
            compress_lvl: 1,
            data_buf_size: 10 * 1024 * 1024,
            out_buf_size: 10 * 1024 * 1024,
-            current_buf_size: 4096,
+            current_buf_size: 20 * 1024,
        };
        let mut writer: Writer<TestData> = Writer::new(&tmpfile, opts).expect("new writer");

@ -480,7 +481,7 @@ mod test {
            compress_lvl: 1,
            data_buf_size: 10 * 1024 * 1024,
            out_buf_size: 10 * 1024 * 1024,
-            current_buf_size: 4096,
+            current_buf_size: 20 * 1024,
        };
        let mut writer: Writer<TestData> = Writer::new(&tmpfile, opts).expect("new writer");

@ -509,7 +510,7 @@ mod test {
            compress_lvl: 1,
            data_buf_size: 10 * 1024 * 1024,
            out_buf_size: 10 * 1024 * 1024,
-            current_buf_size: 4096,
+            current_buf_size: 20 * 1024,
        };
        let mut writer: Writer<TestData> = Writer::new(&tmpfile, opts).expect("new writer");

--- a/lib/src/questions.rs
+++ b/lib/src/questions.rs
@ -1,88 +1,72 @@
 use serde_derive::{Deserialize, Serialize};

-#[derive(
-    Debug, Default, Clone, Serialize, Deserialize, bincode::Decode, bincode::Encode, PartialEq,
-)]
+#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq)]
 pub struct BatchInfo {
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub filename: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub description: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub author: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub comment: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub url: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub date: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub processed_by: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub redacted_by: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub copyright: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub theme: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub kind: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub source: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub rating: String,
 }

-#[derive(
-    Debug, Default, Clone, Serialize, Deserialize, bincode::Decode, bincode::Encode, PartialEq,
-)]
+#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq)]
 pub struct Question {
-    #[serde(default, skip_serializing_if = "u32_is_zero")]
+    #[serde(default)]
    pub num: u32,
    pub id: String,
-
    pub description: String,
    pub answer: String,
-
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub author: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub comment: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub comment1: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub tour: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub url: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub date: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub processed_by: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub redacted_by: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub copyright: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub theme: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub kind: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub source: String,
-    #[serde(default, skip_serializing_if = "String::is_empty")]
+    #[serde(default)]
    pub rating: String,
-    #[serde(default, skip_serializing_if = "BatchInfo::is_default")]
+    #[serde(default)]
    pub batch_info: BatchInfo,
 }

-fn u32_is_zero(num: &u32) -> bool {
-    *num == 0
-}
-
-impl BatchInfo {
-    pub fn is_default(&self) -> bool {
-        *self == BatchInfo::default()
-    }
-}
-
 #[cfg(any(feature = "convert", feature = "convert_async"))]
 pub mod convert_common {
    use super::{BatchInfo, Question};
@ -174,20 +158,68 @@ pub mod convert {
            let converted: Vec<_> = source.convert().collect();
            assert_yaml_snapshot!(converted, @r#"
            ---
-            - id: Вопрос 1
+            - num: 0
+              id: Вопрос 1
              description: Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2
              answer: "42"
+              author: ""
+              comment: ""
+              comment1: ""
+              tour: ""
+              url: ""
+              date: ""
+              processed_by: ""
+              redacted_by: ""
+              copyright: ""
+              theme: ""
+              kind: ""
+              source: ""
+              rating: ""
              batch_info:
                filename: test.json
                description: Тестовый
+                author: ""
+                comment: ""
+                url: ""
                date: 00-000-2000
-            - id: Вопрос 2
+                processed_by: ""
+                redacted_by: ""
+                copyright: ""
+                theme: ""
+                kind: ""
+                source: ""
+                rating: ""
+            - num: 0
+              id: Вопрос 2
              description: Зимой и летом одним цветом
              answer: ёлка
+              author: ""
+              comment: ""
+              comment1: ""
+              tour: ""
+              url: ""
+              date: ""
+              processed_by: ""
+              redacted_by: ""
+              copyright: ""
+              theme: ""
+              kind: ""
+              source: ""
+              rating: ""
              batch_info:
                filename: test.json
                description: Тестовый
+                author: ""
+                comment: ""
+                url: ""
                date: 00-000-2000
+                processed_by: ""
+                redacted_by: ""
+                copyright: ""
+                theme: ""
+                kind: ""
+                source: ""
+                rating: ""
              
            "#);
        }
@ -289,20 +321,68 @@ pub mod convert_async {
            let converted: Vec<_> = converter.collect().await;
            assert_yaml_snapshot!(converted, @r#"
            ---
-            - id: Вопрос 1
+            - num: 0
+              id: Вопрос 1
              description: Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2
              answer: "42"
+              author: ""
+              comment: ""
+              comment1: ""
+              tour: ""
+              url: ""
+              date: ""
+              processed_by: ""
+              redacted_by: ""
+              copyright: ""
+              theme: ""
+              kind: ""
+              source: ""
+              rating: ""
              batch_info:
                filename: test.json
                description: Тестовый
+                author: ""
+                comment: ""
+                url: ""
                date: 00-000-2000
-            - id: Вопрос 2
+                processed_by: ""
+                redacted_by: ""
+                copyright: ""
+                theme: ""
+                kind: ""
+                source: ""
+                rating: ""
+            - num: 0
+              id: Вопрос 2
              description: Зимой и летом одним цветом
              answer: ёлка
+              author: ""
+              comment: ""
+              comment1: ""
+              tour: ""
+              url: ""
+              date: ""
+              processed_by: ""
+              redacted_by: ""
+              copyright: ""
+              theme: ""
+              kind: ""
+              source: ""
+              rating: ""
              batch_info:
                filename: test.json
                description: Тестовый
+                author: ""
+                comment: ""
+                url: ""
                date: 00-000-2000
+                processed_by: ""
+                redacted_by: ""
+                copyright: ""
+                theme: ""
+                kind: ""
+                source: ""
+                rating: ""
              
            "#);
        }
@ -362,13 +442,38 @@ mod test {
    fn test_question_ser() {
        assert_yaml_snapshot!(sample_question(), @r#"
            ---
+            num: 0
            id: Вопрос 1
            description: Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2
            answer: "42"
+            author: ""
+            comment: ""
+            comment1: ""
+            tour: ""
+            url: ""
+            date: ""
+            processed_by: ""
+            redacted_by: ""
+            copyright: ""
+            theme: ""
+            kind: ""
+            source: ""
+            rating: ""
            batch_info:
+              filename: ""
              description: Тестовый
+              author: ""
+              comment: ""
+              url: ""
              date: 00-000-2000
-              
+              processed_by: ""
+              redacted_by: ""
+              copyright: ""
+              theme: ""
+              kind: ""
+              source: ""
+              rating: ""
+            
            "#);
    }
    #[test]
@ -386,12 +491,37 @@ mod test {

        assert_yaml_snapshot!(question_from_json.unwrap(), @r#"
        ---
+        num: 0
        id: Вопрос 1
        description: Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2
        answer: "42"
+        author: ""
+        comment: ""
+        comment1: ""
+        tour: ""
+        url: ""
+        date: ""
+        processed_by: ""
+        redacted_by: ""
+        copyright: ""
+        theme: ""
+        kind: ""
+        source: ""
+        rating: ""
        batch_info:
+          filename: ""
          description: Тестовый
+          author: ""
+          comment: ""
+          url: ""
          date: 00-000-2000
+          processed_by: ""
+          redacted_by: ""
+          copyright: ""
+          theme: ""
+          kind: ""
+          source: ""
+          rating: ""
          
        "#);
    }
--- a/lib/src/util.rs
+++ b/lib/src/util.rs
@ -12,46 +12,3 @@ where
        self.map_err(|e| e.to_string())
    }
 }
-
-#[cfg(any(feature = "sync", feature = "async"))]
-mod bincode_utils {
-    use std::ops::{Deref, DerefMut};
-
-    use bincode::enc::write::Writer;
-    use bincode::error::EncodeError;
-
-    /// struct that allows [`Vec<u8>`] to implement [bincode::enc::write::Writer] trait
-    pub struct BincodeVecWriter {
-        vec: Vec<u8>,
-    }
-
-    impl BincodeVecWriter {
-        pub fn new(vec: Vec<u8>) -> BincodeVecWriter {
-            BincodeVecWriter { vec }
-        }
-    }
-
-    impl Deref for BincodeVecWriter {
-        type Target = Vec<u8>;
-
-        fn deref(&self) -> &Self::Target {
-            &self.vec
-        }
-    }
-
-    impl DerefMut for BincodeVecWriter {
-        fn deref_mut(&mut self) -> &mut Self::Target {
-            &mut self.vec
-        }
-    }
-
-    impl Writer for BincodeVecWriter {
-        fn write(&mut self, bytes: &[u8]) -> Result<(), EncodeError> {
-            self.vec.extend_from_slice(bytes);
-            Ok(())
-        }
-    }
-}
-
-#[cfg(any(feature = "sync", feature = "async"))]
-pub use bincode_utils::BincodeVecWriter;
Author	SHA1	Message	Date
Dmitry Belyaev	e4001ee69f	questions: remove "skip_serializing_if" continuous-integration/drone/push Build is passing Details to fix ser/de to postcard err "Hit the end of buffer, expected more data" more info: https://github.com/apache/arrow-rs/issues/3082	2023-08-23 23:50:31 +03:00
Dmitry Belyaev	85a879e9ec	apps: don't use bincode	2023-08-23 22:40:54 +03:00
Dmitry Belyaev	fad2d1d63d	async db: postcard	2023-08-23 22:33:59 +03:00
Dmitry Belyaev	cb781ccf76	postcard no features	2023-08-23 22:33:36 +03:00
Dmitry Belyaev	7e6b513179	sync db: postcard	2023-08-23 21:40:07 +03:00
Dmitry Belyaev	1d4005abdb	postcard instead of bincode	2023-08-23 21:39:23 +03:00