Compare commits
18 Commits
master
...
postcard_c
| Author | SHA1 | Date | |
|---|---|---|---|
| 6424041198 | |||
| 5f5b299eb3 | |||
| 9934825a93 | |||
6a21543890
|
|||
|
49a214283c
|
|||
|
cb5bd7a628
|
|||
|
f3197603f6
|
|||
|
069a6e22fe
|
|||
| f1aed09319 | |||
|
af9f5d8f24
|
|||
|
67e7d4daef
|
|||
|
bd67e3ee85
|
|||
| e4001ee69f | |||
| 85a879e9ec | |||
| fad2d1d63d | |||
| cb781ccf76 | |||
| 7e6b513179 | |||
| 1d4005abdb |
59
Cargo.lock
generated
59
Cargo.lock
generated
@ -207,25 +207,6 @@ version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a32fd6af2b5827bce66c29053ba0e7c42b9dcab01835835058558c10851a46b"
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "2.0.0-rc.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f11ea1a0346b94ef188834a65c068a03aec181c94896d481d7a0a40d85b0ce95"
|
||||
dependencies = [
|
||||
"bincode_derive",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bincode_derive"
|
||||
version = "2.0.0-rc.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e30759b3b99a1b802a7a3aa21c85c3ded5c28e1c83170d82d70f08bbf7f3e4c"
|
||||
dependencies = [
|
||||
"virtue",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
@ -234,9 +215,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.3.3"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
|
||||
checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
@ -247,6 +228,12 @@ dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bstringify"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd769563b4ea2953e2825c9e6b7470a5f55f67e0be00030bf3e390a2a6071f64"
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.13.0"
|
||||
@ -311,7 +298,6 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
name = "chgk_ledb"
|
||||
version = "1.1.0"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"chgk_ledb_lib",
|
||||
"clap 4.3.16",
|
||||
"criterion 0.4.0",
|
||||
@ -329,7 +315,6 @@ version = "1.1.0"
|
||||
dependencies = [
|
||||
"async-compression 0.4.1",
|
||||
"async_zip",
|
||||
"bincode",
|
||||
"chgk_ledb_lib",
|
||||
"clap 4.3.16",
|
||||
"criterion 0.5.1",
|
||||
@ -351,7 +336,8 @@ dependencies = [
|
||||
"async-compression 0.4.1",
|
||||
"async-stream",
|
||||
"async_zip",
|
||||
"bincode",
|
||||
"bitflags 2.4.0",
|
||||
"bstringify",
|
||||
"fmmap",
|
||||
"futures",
|
||||
"futures-core",
|
||||
@ -359,6 +345,7 @@ dependencies = [
|
||||
"insta",
|
||||
"memmap",
|
||||
"pin-project",
|
||||
"postcard",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
@ -467,6 +454,12 @@ version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
|
||||
|
||||
[[package]]
|
||||
name = "cobs"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.0"
|
||||
@ -1248,6 +1241,16 @@ dependencies = [
|
||||
"plotters-backend",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "postcard"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9ee729232311d3cd113749948b689627618133b1c5012b77342c1950b25eaeb"
|
||||
dependencies = [
|
||||
"cobs",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.17"
|
||||
@ -1408,7 +1411,7 @@ version = "0.38.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5"
|
||||
dependencies = [
|
||||
"bitflags 2.3.3",
|
||||
"bitflags 2.4.0",
|
||||
"errno 0.3.1",
|
||||
"libc",
|
||||
"linux-raw-sys 0.4.3",
|
||||
@ -1701,12 +1704,6 @@ version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
|
||||
|
||||
[[package]]
|
||||
name = "virtue"
|
||||
version = "0.0.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9dcc60c0624df774c82a0ef104151231d37da4962957d691c011c852b2473314"
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.3.3"
|
||||
|
||||
@ -27,6 +27,5 @@ clap = { version = "4.2.7", features = ["derive"] }
|
||||
[dev-dependencies]
|
||||
criterion = "0.4.0"
|
||||
tempfile = "3.3"
|
||||
bincode = "^2.0.0-rc.2"
|
||||
serde="1.0"
|
||||
serde_derive="1.0"
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
#[macro_use]
|
||||
extern crate criterion;
|
||||
extern crate bincode;
|
||||
extern crate serde;
|
||||
extern crate serde_derive;
|
||||
extern crate serde_json;
|
||||
@ -16,18 +15,7 @@ use tempfile::{tempdir, NamedTempFile};
|
||||
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
|
||||
#[derive(
|
||||
bincode::Encode,
|
||||
bincode::Decode,
|
||||
Clone,
|
||||
Debug,
|
||||
PartialEq,
|
||||
Eq,
|
||||
PartialOrd,
|
||||
Ord,
|
||||
Serialize,
|
||||
Deserialize,
|
||||
)]
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
struct TestData {
|
||||
num1: u64,
|
||||
num2: u64,
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
#[macro_use]
|
||||
extern crate criterion;
|
||||
extern crate bincode;
|
||||
extern crate serde;
|
||||
extern crate serde_derive;
|
||||
extern crate serde_json;
|
||||
|
||||
@ -37,7 +37,6 @@ futures = "0.3"
|
||||
[dev-dependencies]
|
||||
criterion = { version = "0.5.1", features = ["async_tokio"]}
|
||||
tempfile = "3.3"
|
||||
bincode = "^2.0.0-rc.2"
|
||||
serde="1.0"
|
||||
serde_derive="1.0"
|
||||
futures = "0.3"
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
#[macro_use]
|
||||
extern crate criterion;
|
||||
extern crate bincode;
|
||||
extern crate serde;
|
||||
extern crate serde_derive;
|
||||
extern crate tempfile;
|
||||
@ -16,18 +15,7 @@ use tempfile::{tempdir, NamedTempFile};
|
||||
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
|
||||
#[derive(
|
||||
bincode::Encode,
|
||||
bincode::Decode,
|
||||
Clone,
|
||||
Debug,
|
||||
PartialEq,
|
||||
Eq,
|
||||
PartialOrd,
|
||||
Ord,
|
||||
Serialize,
|
||||
Deserialize,
|
||||
)]
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
struct TestData {
|
||||
num1: u64,
|
||||
num2: u64,
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
#[macro_use]
|
||||
extern crate criterion;
|
||||
extern crate bincode;
|
||||
extern crate serde;
|
||||
extern crate serde_derive;
|
||||
extern crate tempfile;
|
||||
|
||||
@ -12,40 +12,16 @@ description = "Библиотека для доступа к файлу базы
|
||||
[features]
|
||||
default = []
|
||||
sync = ["zstd", "memmap"]
|
||||
async = [
|
||||
"futures",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"fmmap",
|
||||
"tokio",
|
||||
"async-compression",
|
||||
"async-stream",
|
||||
"pin-project",
|
||||
]
|
||||
async = ["futures", "futures-core", "futures-util", "fmmap", "tokio", "async-compression", "async-stream", "pin-project"]
|
||||
source = ["zip"]
|
||||
source_async = [
|
||||
"async_zip",
|
||||
"tokio",
|
||||
"futures",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"async-stream",
|
||||
]
|
||||
source_async = ["async_zip", "tokio", "futures", "futures-core", "futures-util", "async-stream"]
|
||||
convert = ["zip"]
|
||||
convert_async = [
|
||||
"futures",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"async-stream",
|
||||
"async_zip",
|
||||
"tokio",
|
||||
]
|
||||
convert_async = ["futures", "futures-core", "futures-util", "async-stream", "async_zip", "tokio"]
|
||||
|
||||
[dependencies]
|
||||
serde = "1.0"
|
||||
serde_derive = "1.0"
|
||||
serde_json = "1.0"
|
||||
bincode = "^2.0.0-rc.2"
|
||||
zip = { version = "0.6", optional = true }
|
||||
async_zip = { version = "0.0.15" , features = [
|
||||
"zstd",
|
||||
@ -70,6 +46,9 @@ async-stream = { version = "0.3", optional = true }
|
||||
zstd = { version = "^0.12", default-features = false, optional = true }
|
||||
memmap = { version = "0.7.0", optional = true }
|
||||
pin-project = { version = "1.1.3", optional = true }
|
||||
postcard = { version = "1.0.6", default-features = false }
|
||||
bitflags = "2.4.0"
|
||||
bstringify = "0.1.2"
|
||||
|
||||
[dev-dependencies]
|
||||
insta = { version = "1.31.0", features = ["yaml"] }
|
||||
|
||||
@ -24,9 +24,9 @@ use fmmap::tokio::{AsyncMmapFile, AsyncMmapFileExt, AsyncOptions};
|
||||
|
||||
type LSize = u32;
|
||||
const LEN_SIZE: usize = std::mem::size_of::<LSize>();
|
||||
const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard();
|
||||
|
||||
use crate::util::BincodeVecWriter;
|
||||
use serde::{de::DeserializeOwned, Serialize};
|
||||
|
||||
use crate::util::ErrorToString;
|
||||
|
||||
pub struct WriterOpts {
|
||||
@ -42,18 +42,18 @@ impl Default for WriterOpts {
|
||||
compress_lvl: Level::Default,
|
||||
data_buf_size: 500 * 1024 * 1024,
|
||||
out_buf_size: 200 * 1024 * 1024,
|
||||
current_buf_size: 100 * 1024,
|
||||
current_buf_size: 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Writer<T>
|
||||
where
|
||||
T: bincode::Encode,
|
||||
T: Serialize,
|
||||
{
|
||||
out: io::BufWriter<fs::File>,
|
||||
data_buf: Vec<u8>,
|
||||
cur_buf_item: BincodeVecWriter,
|
||||
cur_buf_item: Vec<u8>,
|
||||
table: Vec<LSize>,
|
||||
compress_lvl: Level,
|
||||
_t: PhantomData<Arc<T>>,
|
||||
@ -61,14 +61,13 @@ where
|
||||
|
||||
impl<T> Writer<T>
|
||||
where
|
||||
T: bincode::Encode,
|
||||
T: Serialize,
|
||||
{
|
||||
pub async fn new<P: AsRef<Path>>(path: P, opts: WriterOpts) -> Result<Self, String> {
|
||||
let out = fs::File::create(path).await.str_err()?;
|
||||
let out = io::BufWriter::with_capacity(opts.out_buf_size, out);
|
||||
let data_buf: Vec<u8> = Vec::with_capacity(opts.data_buf_size);
|
||||
let cur_buf_item: Vec<u8> = Vec::with_capacity(opts.current_buf_size);
|
||||
let cur_buf_item = BincodeVecWriter::new(cur_buf_item);
|
||||
let cur_buf_item: Vec<u8> = vec![0; opts.current_buf_size];
|
||||
|
||||
let compress_lvl = opts.compress_lvl;
|
||||
|
||||
@ -88,16 +87,16 @@ where
|
||||
self.push_by_ref(&item).await
|
||||
}
|
||||
|
||||
#[allow(clippy::useless_asref)]
|
||||
pub async fn push_by_ref(&mut self, item: &T) -> Result<(), String> {
|
||||
let pos: LSize = self.data_buf.len() as LSize;
|
||||
let cur_item_data = postcard::to_slice(item, self.cur_buf_item.as_mut_slice()).str_err()?;
|
||||
|
||||
bincode::encode_into_writer(item, &mut self.cur_buf_item, BINCODE_CFG).str_err()?;
|
||||
|
||||
let mut zencoder = ZstdEncoder::with_quality(&self.cur_buf_item[..], self.compress_lvl);
|
||||
let mut zencoder = ZstdEncoder::with_quality(cur_item_data.as_ref(), self.compress_lvl);
|
||||
io::copy(&mut zencoder, &mut self.data_buf)
|
||||
.await
|
||||
.str_err()?;
|
||||
self.cur_buf_item.clear();
|
||||
cur_item_data.fill(0);
|
||||
|
||||
self.table.push(pos);
|
||||
|
||||
@ -159,7 +158,7 @@ use pin_project::pin_project;
|
||||
/// only work when ..push.poll() returns Ready immediately
|
||||
pub struct WriterSink<'a, T>
|
||||
where
|
||||
T: bincode::Encode,
|
||||
T: Serialize,
|
||||
{
|
||||
#[pin]
|
||||
writer: &'a mut Writer<T>,
|
||||
@ -168,7 +167,7 @@ where
|
||||
|
||||
impl<'a, T> Sink<T> for WriterSink<'a, T>
|
||||
where
|
||||
T: bincode::Encode,
|
||||
T: Serialize,
|
||||
{
|
||||
type Error = String;
|
||||
|
||||
@ -213,7 +212,7 @@ where
|
||||
|
||||
pub struct Reader<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
mmap: AsyncMmapFile,
|
||||
count: usize,
|
||||
@ -223,7 +222,7 @@ where
|
||||
|
||||
impl<T> Reader<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
pub async fn new<P: AsRef<Path>>(path: P) -> Result<Self, String> {
|
||||
let mmap = AsyncOptions::new()
|
||||
@ -292,10 +291,10 @@ where
|
||||
decoder.read_to_end(data_buf).await.str_err()?;
|
||||
|
||||
// decode item
|
||||
let item: (T, usize) = bincode::decode_from_slice(data_buf, BINCODE_CFG).str_err()?;
|
||||
let (item, _): (T, _) = postcard::take_from_bytes(data_buf).str_err()?;
|
||||
|
||||
data_buf.clear();
|
||||
Ok(item.0)
|
||||
Ok(item)
|
||||
}
|
||||
|
||||
/// get item at index
|
||||
@ -311,7 +310,7 @@ where
|
||||
|
||||
pub struct ReaderStream<'a, T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
reader: &'a Reader<T>,
|
||||
index: Option<usize>,
|
||||
@ -319,7 +318,7 @@ where
|
||||
|
||||
impl<'a, T> ReaderStream<'a, T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
fn new(reader: &'a Reader<T>) -> Self {
|
||||
ReaderStream {
|
||||
@ -331,7 +330,7 @@ where
|
||||
|
||||
impl<'a, T> Stream for ReaderStream<'a, T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
type Item = T;
|
||||
|
||||
@ -375,7 +374,7 @@ where
|
||||
|
||||
pub struct BufReader<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
inner: Reader<T>,
|
||||
buf: Vec<u8>,
|
||||
@ -383,7 +382,7 @@ where
|
||||
|
||||
impl<T> BufReader<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
pub async fn new<P: AsRef<Path>>(path: P, buf_size: usize) -> Result<Self, String> {
|
||||
match Reader::<T>::new(path).await {
|
||||
@ -410,7 +409,7 @@ where
|
||||
|
||||
impl<T> From<Reader<T>> for BufReader<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
fn from(inner: Reader<T>) -> Self {
|
||||
Self {
|
||||
@ -422,7 +421,7 @@ where
|
||||
|
||||
impl<T> From<BufReader<T>> for Reader<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
fn from(value: BufReader<T>) -> Self {
|
||||
value.into_inner()
|
||||
@ -431,7 +430,7 @@ where
|
||||
|
||||
impl<T> Deref for BufReader<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
type Target = Reader<T>;
|
||||
fn deref(&self) -> &Self::Target {
|
||||
@ -441,7 +440,7 @@ where
|
||||
|
||||
pub struct BufReaderStream<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
reader: BufReader<T>,
|
||||
index: Option<usize>,
|
||||
@ -449,7 +448,7 @@ where
|
||||
|
||||
impl<T> BufReaderStream<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
fn new(reader: BufReader<T>) -> Self {
|
||||
BufReaderStream {
|
||||
@ -473,7 +472,7 @@ where
|
||||
|
||||
impl<T> Stream for BufReaderStream<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
type Item = T;
|
||||
|
||||
@ -516,9 +515,10 @@ where
|
||||
mod test {
|
||||
use super::*;
|
||||
use core::fmt::Debug;
|
||||
use serde_derive::Deserialize;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[derive(bincode::Encode, bincode::Decode, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
struct TestData {
|
||||
num: u64,
|
||||
test: String,
|
||||
|
||||
@ -6,13 +6,13 @@ use std::{
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use serde::{de::DeserializeOwned, Serialize};
|
||||
|
||||
use memmap::{Mmap, MmapOptions};
|
||||
|
||||
type LSize = u32;
|
||||
const LEN_SIZE: usize = std::mem::size_of::<LSize>();
|
||||
const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard();
|
||||
|
||||
use crate::util::BincodeVecWriter;
|
||||
use crate::util::ErrorToString;
|
||||
|
||||
pub struct WriterOpts {
|
||||
@ -28,19 +28,19 @@ impl Default for WriterOpts {
|
||||
compress_lvl: 1,
|
||||
data_buf_size: 500 * 1024 * 1024,
|
||||
out_buf_size: 200 * 1024 * 1024,
|
||||
current_buf_size: 100 * 1024,
|
||||
current_buf_size: 20 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Writer<T>
|
||||
where
|
||||
T: bincode::Encode,
|
||||
T: Serialize,
|
||||
{
|
||||
out: io::BufWriter<fs::File>,
|
||||
data_buf: Cursor<Vec<u8>>,
|
||||
cur_buf_raw: Cursor<Vec<u8>>,
|
||||
cur_buf_item: BincodeVecWriter,
|
||||
cur_buf_item: Vec<u8>,
|
||||
table: Vec<LSize>,
|
||||
compress_lvl: i32,
|
||||
_t: PhantomData<Arc<T>>,
|
||||
@ -48,7 +48,7 @@ where
|
||||
|
||||
impl<T> Writer<T>
|
||||
where
|
||||
T: bincode::Encode,
|
||||
T: Serialize,
|
||||
{
|
||||
pub fn new<P: AsRef<Path>>(path: P, opts: WriterOpts) -> Result<Self, String> {
|
||||
let out = fs::File::create(path).str_err()?;
|
||||
@ -58,8 +58,7 @@ where
|
||||
|
||||
let cur_buf_raw: Vec<u8> = Vec::with_capacity(opts.current_buf_size);
|
||||
let cur_buf_raw = Cursor::new(cur_buf_raw);
|
||||
let cur_buf_item: Vec<u8> = Vec::with_capacity(opts.current_buf_size);
|
||||
let cur_buf_item = BincodeVecWriter::new(cur_buf_item);
|
||||
let cur_buf_item: Vec<u8> = vec![0; opts.current_buf_size];
|
||||
|
||||
let compress_lvl = opts.compress_lvl;
|
||||
|
||||
@ -83,19 +82,20 @@ where
|
||||
pub fn push_by_ref(&mut self, item: &T) -> Result<(), String> {
|
||||
let pos: LSize = self.data_buf.position() as LSize;
|
||||
|
||||
bincode::encode_into_writer(item, &mut self.cur_buf_item, BINCODE_CFG).str_err()?;
|
||||
let cur_item_data = postcard::to_slice(item, self.cur_buf_item.as_mut_slice()).str_err()?;
|
||||
|
||||
let mut zencoder = zstd::stream::raw::Encoder::new(self.compress_lvl).str_err()?;
|
||||
zencoder
|
||||
.set_pledged_src_size(Some(self.cur_buf_item.len() as u64))
|
||||
.set_pledged_src_size(Some(cur_item_data.len() as u64))
|
||||
.str_err()?;
|
||||
|
||||
self.cur_buf_raw.set_position(0);
|
||||
let mut cur_buf_z = zstd::stream::zio::Writer::new(&mut self.cur_buf_raw, zencoder);
|
||||
cur_buf_z.write_all(&self.cur_buf_item).str_err()?;
|
||||
cur_buf_z.write_all(cur_item_data).str_err()?;
|
||||
cur_buf_z.finish().str_err()?;
|
||||
cur_buf_z.flush().str_err()?;
|
||||
self.cur_buf_item.clear();
|
||||
|
||||
cur_item_data.fill(0);
|
||||
|
||||
self.table.push(pos);
|
||||
let (cur_buf_raw, _) = cur_buf_z.into_inner();
|
||||
@ -150,7 +150,7 @@ where
|
||||
|
||||
pub struct Reader<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
mmap: Mmap,
|
||||
count: usize,
|
||||
@ -160,7 +160,7 @@ where
|
||||
|
||||
impl<T> Reader<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
pub fn new<P: AsRef<Path>>(path: P, _buf_size: usize) -> Result<Self, String> {
|
||||
let file = fs::File::open(path).str_err()?;
|
||||
@ -215,9 +215,9 @@ where
|
||||
let data = zstd::decode_all(reader).str_err()?;
|
||||
|
||||
// decode item
|
||||
let item: (T, usize) = bincode::decode_from_slice(&data, BINCODE_CFG).str_err()?;
|
||||
let (item, _): (T, _) = postcard::take_from_bytes(&data).str_err()?;
|
||||
|
||||
Ok(item.0)
|
||||
Ok(item)
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> ReaderIter<'_, T> {
|
||||
@ -227,7 +227,7 @@ where
|
||||
|
||||
pub struct ReaderIter<'a, T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
reader: &'a Reader<T>,
|
||||
index: Option<usize>,
|
||||
@ -235,7 +235,7 @@ where
|
||||
|
||||
impl<'a, T> ReaderIter<'a, T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
fn new(reader: &'a Reader<T>) -> Self {
|
||||
ReaderIter {
|
||||
@ -247,7 +247,7 @@ where
|
||||
|
||||
impl<'a, T> Iterator for ReaderIter<'a, T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
type Item = T;
|
||||
|
||||
@ -300,7 +300,7 @@ where
|
||||
|
||||
impl<'a, T> ExactSizeIterator for ReaderIter<'a, T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
fn len(&self) -> usize {
|
||||
self.reader.len()
|
||||
@ -309,7 +309,7 @@ where
|
||||
|
||||
pub struct ReaderIntoIter<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
reader: Reader<T>,
|
||||
index: Option<usize>,
|
||||
@ -317,7 +317,7 @@ where
|
||||
|
||||
impl<T> ReaderIntoIter<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
fn new(reader: Reader<T>) -> Self {
|
||||
Self {
|
||||
@ -329,7 +329,7 @@ where
|
||||
|
||||
impl<T> Iterator for ReaderIntoIter<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
type Item = T;
|
||||
|
||||
@ -382,7 +382,7 @@ where
|
||||
|
||||
impl<T> ExactSizeIterator for ReaderIntoIter<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
fn len(&self) -> usize {
|
||||
self.reader.len()
|
||||
@ -391,7 +391,7 @@ where
|
||||
|
||||
impl<T> IntoIterator for Reader<T>
|
||||
where
|
||||
T: bincode::Decode,
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
type Item = T;
|
||||
type IntoIter = ReaderIntoIter<Self::Item>;
|
||||
@ -404,9 +404,10 @@ where
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use serde::Deserialize;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[derive(bincode::Encode, bincode::Decode, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
struct TestData {
|
||||
num: u64,
|
||||
test: String,
|
||||
@ -427,7 +428,7 @@ mod test {
|
||||
compress_lvl: 1,
|
||||
data_buf_size: 10 * 1024 * 1024,
|
||||
out_buf_size: 10 * 1024 * 1024,
|
||||
current_buf_size: 4096,
|
||||
current_buf_size: 20 * 1024,
|
||||
};
|
||||
let mut writer: Writer<TestData> = Writer::new(&tmpfile, opts).expect("new writer");
|
||||
|
||||
@ -454,7 +455,7 @@ mod test {
|
||||
compress_lvl: 1,
|
||||
data_buf_size: 10 * 1024 * 1024,
|
||||
out_buf_size: 10 * 1024 * 1024,
|
||||
current_buf_size: 4096,
|
||||
current_buf_size: 20 * 1024,
|
||||
};
|
||||
let mut writer: Writer<TestData> = Writer::new(&tmpfile, opts).expect("new writer");
|
||||
|
||||
@ -480,7 +481,7 @@ mod test {
|
||||
compress_lvl: 1,
|
||||
data_buf_size: 10 * 1024 * 1024,
|
||||
out_buf_size: 10 * 1024 * 1024,
|
||||
current_buf_size: 4096,
|
||||
current_buf_size: 20 * 1024,
|
||||
};
|
||||
let mut writer: Writer<TestData> = Writer::new(&tmpfile, opts).expect("new writer");
|
||||
|
||||
@ -509,7 +510,7 @@ mod test {
|
||||
compress_lvl: 1,
|
||||
data_buf_size: 10 * 1024 * 1024,
|
||||
out_buf_size: 10 * 1024 * 1024,
|
||||
current_buf_size: 4096,
|
||||
current_buf_size: 20 * 1024,
|
||||
};
|
||||
let mut writer: Writer<TestData> = Writer::new(&tmpfile, opts).expect("new writer");
|
||||
|
||||
|
||||
@ -1,88 +1,800 @@
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use std::{
|
||||
fmt::{self, Formatter},
|
||||
marker::PhantomData,
|
||||
};
|
||||
|
||||
#[derive(
|
||||
Debug, Default, Clone, Serialize, Deserialize, bincode::Decode, bincode::Encode, PartialEq,
|
||||
)]
|
||||
use serde::{
|
||||
de::{MapAccess, SeqAccess, Visitor},
|
||||
Deserialize, Deserializer,
|
||||
};
|
||||
|
||||
use bstringify::bstringify;
|
||||
use serde::ser::SerializeStruct;
|
||||
|
||||
use bitflags::bitflags;
|
||||
|
||||
bitflags! {
|
||||
#[repr(transparent)]
|
||||
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct BatchFlags: u16 {
|
||||
const FILENAME = 1;
|
||||
const DESCRIPTION = 1 << 1;
|
||||
const AUTHOR = 1 << 2;
|
||||
const COMMENT = 1 << 3;
|
||||
const URL = 1 << 4;
|
||||
const DATE = 1 << 5;
|
||||
const PROCESSED = 1 << 6;
|
||||
const REDACTED = 1 << 7;
|
||||
const COPYRIGHT = 1 << 8;
|
||||
const THEME = 1 << 9;
|
||||
const KIND = 1 << 10;
|
||||
const SOURCE = 1 << 11;
|
||||
const RATING = 1 << 12;
|
||||
}
|
||||
}
|
||||
|
||||
bitflags! {
|
||||
#[repr(transparent)]
|
||||
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct QuestionFlags: u16 {
|
||||
const NUM = 1;
|
||||
const AUTHOR = 1 << 1;
|
||||
const COMMENT = 1 << 2;
|
||||
const COMMENT1 = 1 << 3;
|
||||
const TOUR = 1 << 4;
|
||||
const URL = 1 << 5;
|
||||
const DATE = 1 << 6;
|
||||
const PROCESSED_BY = 1 << 7;
|
||||
const REDACTED_BY = 1 << 8;
|
||||
const COPYRIGHT = 1 << 9;
|
||||
const THEME = 1 << 10;
|
||||
const KIND = 1 << 11;
|
||||
const SOURCE = 1 << 12;
|
||||
const RATING = 1 << 13;
|
||||
const BATCH_INFO = 1 << 14;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq)]
|
||||
pub struct BatchInfo {
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub filename: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub description: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub author: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub comment: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub url: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub date: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub processed_by: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub redacted_by: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub copyright: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub theme: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub kind: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub source: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub rating: String,
|
||||
}
|
||||
|
||||
#[derive(
|
||||
Debug, Default, Clone, Serialize, Deserialize, bincode::Decode, bincode::Encode, PartialEq,
|
||||
)]
|
||||
#[derive(Debug, Default, Clone, PartialEq)]
|
||||
pub struct Question {
|
||||
#[serde(default, skip_serializing_if = "u32_is_zero")]
|
||||
pub num: u32,
|
||||
pub id: String,
|
||||
|
||||
// required fields
|
||||
pub id: String,
|
||||
pub description: String,
|
||||
pub answer: String,
|
||||
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub author: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub comment: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub comment1: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub tour: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub url: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub date: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub processed_by: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub redacted_by: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub copyright: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub theme: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub kind: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub source: String,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub rating: String,
|
||||
#[serde(default, skip_serializing_if = "BatchInfo::is_default")]
|
||||
pub batch_info: BatchInfo,
|
||||
}
|
||||
|
||||
fn u32_is_zero(num: &u32) -> bool {
|
||||
*num == 0
|
||||
}
|
||||
|
||||
impl BatchInfo {
|
||||
pub fn is_default(&self) -> bool {
|
||||
*self == BatchInfo::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// fill flags, if string field is not empty set bit flag
|
||||
///
|
||||
/// Example:
|
||||
/// ```ignore
|
||||
/// flags_from_strings!((self, flags, BatchFlags) <- { filename: FILENAME, description: DESCRIPTION, ... });
|
||||
/// ```
|
||||
macro_rules! flags_from_strings {
|
||||
(($self:ident, $flags:ident, $FlagsType:ident) <- {$($field:ident:$flag:ident),+} ) => {$(
|
||||
if !$self.$field.is_empty() {
|
||||
$flags |= $FlagsType::$flag;
|
||||
}
|
||||
)+}
|
||||
}
|
||||
|
||||
/// serialize fields, if bit flag is not set then skip
|
||||
///
|
||||
/// Example:
|
||||
/// ```ignore
|
||||
/// serialize_fields!((self, flags, state, BatchFlags) <- { filename: FILENAME, description: DESCRIPTION, ... });
|
||||
/// ```
|
||||
macro_rules! serialize_fields {
|
||||
(($self:ident, $flags:ident, $state:ident, $FlagsType:ident) <- {$($field:ident:$flag:ident),+} ) => {$(
|
||||
if $flags.intersects($FlagsType::$flag) {
|
||||
$state.serialize_field(std::stringify!($field), &$self.$field)?;
|
||||
} else {
|
||||
$state.skip_field(std::stringify!($field))?;
|
||||
}
|
||||
)+}
|
||||
}
|
||||
|
||||
impl serde::Serialize for BatchInfo {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
let is_human_readable = serializer.is_human_readable();
|
||||
let mut flags: BatchFlags = Default::default();
|
||||
let len = 13;
|
||||
|
||||
flags_from_strings!((self, flags, BatchFlags) <- {
|
||||
filename: FILENAME, description: DESCRIPTION, author: AUTHOR, comment: COMMENT, url: URL, date: DATE, processed_by: PROCESSED,
|
||||
redacted_by: REDACTED, copyright: COPYRIGHT, theme: THEME, kind: KIND, source: SOURCE, rating: RATING
|
||||
});
|
||||
|
||||
let mut state = serializer.serialize_struct("BatchInfo", len)?;
|
||||
|
||||
if is_human_readable {
|
||||
state.skip_field("_flags")?;
|
||||
} else {
|
||||
state.serialize_field("_flags", &flags.bits())?;
|
||||
}
|
||||
|
||||
serialize_fields!((self, flags, state, BatchFlags) <- {
|
||||
filename: FILENAME, description: DESCRIPTION, author: AUTHOR, comment: COMMENT, url: URL, date: DATE, processed_by: PROCESSED,
|
||||
redacted_by: REDACTED, copyright: COPYRIGHT, theme: THEME, kind: KIND, source: SOURCE, rating: RATING
|
||||
});
|
||||
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
|
||||
impl serde::Serialize for Question {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
let is_human_readable = serializer.is_human_readable();
|
||||
let mut flags: QuestionFlags = Default::default();
|
||||
let len = 19;
|
||||
|
||||
if self.num != 0 {
|
||||
flags |= QuestionFlags::NUM;
|
||||
}
|
||||
|
||||
if !self.batch_info.is_default() {
|
||||
flags |= QuestionFlags::BATCH_INFO;
|
||||
}
|
||||
|
||||
flags_from_strings!((self, flags, QuestionFlags) <- {
|
||||
author: AUTHOR, comment: COMMENT, comment1: COMMENT1, tour: TOUR, url: URL, date: DATE,
|
||||
processed_by: PROCESSED_BY, redacted_by: REDACTED_BY, copyright: COPYRIGHT, theme: THEME,
|
||||
kind: KIND, source: SOURCE, rating: RATING
|
||||
});
|
||||
|
||||
let mut state = serializer.serialize_struct("Question", len)?;
|
||||
|
||||
if is_human_readable {
|
||||
state.skip_field("_flags")?;
|
||||
} else {
|
||||
state.serialize_field("_flags", &flags.bits())?;
|
||||
}
|
||||
|
||||
if flags.intersects(QuestionFlags::NUM) {
|
||||
state.serialize_field("num", &self.num)?;
|
||||
} else {
|
||||
state.skip_field("num")?;
|
||||
}
|
||||
|
||||
state.serialize_field("id", &self.id)?;
|
||||
state.serialize_field("description", &self.description)?;
|
||||
state.serialize_field("answer", &self.answer)?;
|
||||
|
||||
serialize_fields!((self, flags, state, QuestionFlags) <- {
|
||||
author: AUTHOR, comment: COMMENT, comment1: COMMENT1, tour: TOUR, url: URL, date: DATE,
|
||||
processed_by: PROCESSED_BY, redacted_by: REDACTED_BY, copyright: COPYRIGHT, theme: THEME,
|
||||
kind: KIND, source: SOURCE, rating: RATING, batch_info: BATCH_INFO
|
||||
});
|
||||
|
||||
if flags.intersects(QuestionFlags::BATCH_INFO) {
|
||||
state.serialize_field("batch_info", &self.batch_info)?;
|
||||
} else {
|
||||
state.skip_field("batch_info")?;
|
||||
}
|
||||
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
|
||||
/// match field name on _FieldVisitor::visit_str or ::visit_bytes
|
||||
///
|
||||
/// Examples:
|
||||
///
|
||||
/// str:
|
||||
/// ```ignore
|
||||
/// fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
|
||||
/// where E: serde::de::Error, {
|
||||
/// match_visit_str_fields!(value => BatchField {filename, description, ...})
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// or bytes:
|
||||
/// ```ignore
|
||||
/// fn visit_bytes<E>(self, value: &[u8]) -> Result<Self::Value, E>
|
||||
/// where E: serde::de::Error, {
|
||||
/// match_visit_str_fields!([bytes] value => BatchField {filename, description, ...})
|
||||
/// }
|
||||
/// ```
|
||||
macro_rules! match_visit_str_fields {
|
||||
($value:ident => $FieldType:ty {$($field:ident),+}) => {
|
||||
match $value {
|
||||
$(
|
||||
std::stringify!($field) => Ok(<$FieldType>::$field),
|
||||
)+
|
||||
_ => Ok(<$FieldType>::__ignore),
|
||||
}
|
||||
};
|
||||
([bytes] $value:ident => $FieldType:ty {$($field:ident),+}) => {
|
||||
match $value {
|
||||
$(
|
||||
bstringify!($field) => Ok(<$FieldType>::$field),
|
||||
)+
|
||||
_ => Ok(<$FieldType>::__ignore),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// for seq access, deserialize string fields if bit flag is set, else read as default (empty)
|
||||
macro_rules! seq_read_strings {
|
||||
(($flags:ident, $seq:ident, $FlagsType:ty) <- {$($field:ident:$flag:ident),+} ) => {$(
|
||||
let $field: String = if $flags.intersects(<$FlagsType>::$flag) {
|
||||
SeqAccess::next_element::<String>(&mut $seq)?.unwrap_or_default()
|
||||
} else {
|
||||
Default::default()
|
||||
};
|
||||
)+}
|
||||
}
|
||||
|
||||
/// for map access, match and deserialize fields + check for duplicated fields
|
||||
///
|
||||
/// Examples:
|
||||
///
|
||||
/// variant for mixed value types:
|
||||
/// ```ignore
|
||||
/// match_map_fields!((map, key, QuestionField) <- {num: u32, id: String, ...});
|
||||
/// ```
|
||||
///
|
||||
/// variant for single value type:
|
||||
/// ```ignore
|
||||
/// match_map_fields!((map, key, BatchField, String) <- {filename, url, ...});
|
||||
/// ```
|
||||
macro_rules! match_map_fields {
|
||||
(($map:ident, $key:ident, $FieldType:ty) <- {$($field:ident:$FieldValueType:ty),+} ) => {
|
||||
match $key {
|
||||
$(
|
||||
<$FieldType>::$field => {
|
||||
if $field.is_some() {
|
||||
return Err(<V::Error as serde::de::Error>::duplicate_field(
|
||||
std::stringify!($field),
|
||||
));
|
||||
}
|
||||
$field = Some(MapAccess::next_value::<$FieldValueType>(&mut $map)?);
|
||||
},
|
||||
)+
|
||||
_ => {
|
||||
let _ = MapAccess::next_value::<serde::de::IgnoredAny>(&mut $map)?;
|
||||
}
|
||||
}
|
||||
};
|
||||
(($map:ident, $key:ident, $FieldType:ty, $FieldValueType:ty) <- {$($field:ident),+} ) => {
|
||||
match $key {
|
||||
$(
|
||||
<$FieldType>::$field => {
|
||||
if $field.is_some() {
|
||||
return Err(<V::Error as serde::de::Error>::duplicate_field(
|
||||
std::stringify!($field),
|
||||
));
|
||||
}
|
||||
$field = Some(MapAccess::next_value::<$FieldValueType>(&mut $map)?);
|
||||
},
|
||||
)+
|
||||
_ => {
|
||||
let _ = MapAccess::next_value::<serde::de::IgnoredAny>(&mut $map)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for BatchInfo {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
#[allow(non_camel_case_types)]
|
||||
enum BatchField {
|
||||
filename,
|
||||
description,
|
||||
author,
|
||||
comment,
|
||||
url,
|
||||
date,
|
||||
processed_by,
|
||||
redacted_by,
|
||||
copyright,
|
||||
theme,
|
||||
kind,
|
||||
source,
|
||||
rating,
|
||||
_flags,
|
||||
__ignore,
|
||||
}
|
||||
|
||||
struct BatchFieldVisitor;
|
||||
impl<'de> Visitor<'de> for BatchFieldVisitor {
|
||||
type Value = BatchField;
|
||||
fn expecting(&self, formatter: &mut Formatter) -> fmt::Result {
|
||||
Formatter::write_str(formatter, "field identifier")
|
||||
}
|
||||
fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
|
||||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
match value {
|
||||
0u64 => Ok(BatchField::filename),
|
||||
1u64 => Ok(BatchField::description),
|
||||
2u64 => Ok(BatchField::author),
|
||||
3u64 => Ok(BatchField::comment),
|
||||
4u64 => Ok(BatchField::url),
|
||||
5u64 => Ok(BatchField::date),
|
||||
6u64 => Ok(BatchField::processed_by),
|
||||
7u64 => Ok(BatchField::redacted_by),
|
||||
8u64 => Ok(BatchField::copyright),
|
||||
9u64 => Ok(BatchField::theme),
|
||||
10u64 => Ok(BatchField::kind),
|
||||
11u64 => Ok(BatchField::source),
|
||||
12u64 => Ok(BatchField::rating),
|
||||
13u64 => Ok(BatchField::_flags),
|
||||
_ => Ok(BatchField::__ignore),
|
||||
}
|
||||
}
|
||||
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
|
||||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
match_visit_str_fields!(value => BatchField {
|
||||
filename, description, author, comment, url, date, processed_by, redacted_by,
|
||||
copyright, theme, kind, source, rating, _flags, __ignore
|
||||
})
|
||||
}
|
||||
fn visit_bytes<E>(self, value: &[u8]) -> Result<Self::Value, E>
|
||||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
match_visit_str_fields!([bytes] value => BatchField {
|
||||
filename, description, author, comment, url, date, processed_by, redacted_by,
|
||||
copyright, theme, kind, source, rating, _flags, __ignore
|
||||
})
|
||||
}
|
||||
}
|
||||
impl<'de> Deserialize<'de> for BatchField {
|
||||
#[inline]
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
Deserializer::deserialize_identifier(deserializer, BatchFieldVisitor)
|
||||
}
|
||||
}
|
||||
struct BatchInfoVisitor<'de> {
|
||||
marker: PhantomData<BatchInfo>,
|
||||
lifetime: PhantomData<&'de ()>,
|
||||
}
|
||||
impl<'de> Visitor<'de> for BatchInfoVisitor<'de> {
|
||||
type Value = BatchInfo;
|
||||
fn expecting(&self, formatter: &mut Formatter) -> fmt::Result {
|
||||
Formatter::write_str(formatter, "struct BatchInfo")
|
||||
}
|
||||
#[inline]
|
||||
fn visit_seq<V>(self, mut seq: V) -> Result<Self::Value, V::Error>
|
||||
where
|
||||
V: SeqAccess<'de>,
|
||||
{
|
||||
let flags = BatchFlags::from_bits(
|
||||
SeqAccess::next_element::<u16>(&mut seq)?.unwrap_or_default(),
|
||||
)
|
||||
.unwrap_or_default();
|
||||
|
||||
seq_read_strings!((flags, seq, BatchFlags) <- {
|
||||
filename: FILENAME, description: DESCRIPTION, author: AUTHOR, comment: COMMENT, url: URL, date: DATE, processed_by: PROCESSED,
|
||||
redacted_by: REDACTED, copyright: COPYRIGHT, theme: THEME, kind: KIND, source: SOURCE, rating: RATING
|
||||
});
|
||||
|
||||
Ok(BatchInfo {
|
||||
filename,
|
||||
description,
|
||||
author,
|
||||
comment,
|
||||
url,
|
||||
date,
|
||||
processed_by,
|
||||
redacted_by,
|
||||
copyright,
|
||||
theme,
|
||||
kind,
|
||||
source,
|
||||
rating,
|
||||
})
|
||||
}
|
||||
#[inline]
|
||||
fn visit_map<V>(self, mut map: V) -> Result<Self::Value, V::Error>
|
||||
where
|
||||
V: MapAccess<'de>,
|
||||
{
|
||||
let mut filename: Option<String> = None;
|
||||
let mut description: Option<String> = None;
|
||||
let mut author: Option<String> = None;
|
||||
let mut comment: Option<String> = None;
|
||||
let mut url: Option<String> = None;
|
||||
let mut date: Option<String> = None;
|
||||
let mut processed_by: Option<String> = None;
|
||||
let mut redacted_by: Option<String> = None;
|
||||
let mut copyright: Option<String> = None;
|
||||
let mut theme: Option<String> = None;
|
||||
let mut kind: Option<String> = None;
|
||||
let mut source: Option<String> = None;
|
||||
let mut rating: Option<String> = None;
|
||||
|
||||
while let Some(key) = MapAccess::next_key::<BatchField>(&mut map)? {
|
||||
match_map_fields!((map, key, BatchField, String) <- {
|
||||
filename, description, author, comment, url, date, processed_by,
|
||||
redacted_by, copyright, theme, kind, source, rating
|
||||
});
|
||||
}
|
||||
|
||||
let filename = filename.unwrap_or_default();
|
||||
let description = description.unwrap_or_default();
|
||||
let author = author.unwrap_or_default();
|
||||
let comment = comment.unwrap_or_default();
|
||||
let url = url.unwrap_or_default();
|
||||
let date = date.unwrap_or_default();
|
||||
let processed_by = processed_by.unwrap_or_default();
|
||||
let redacted_by = redacted_by.unwrap_or_default();
|
||||
let copyright = copyright.unwrap_or_default();
|
||||
let theme = theme.unwrap_or_default();
|
||||
let kind = kind.unwrap_or_default();
|
||||
let source = source.unwrap_or_default();
|
||||
let rating = rating.unwrap_or_default();
|
||||
|
||||
Ok(BatchInfo {
|
||||
filename,
|
||||
description,
|
||||
author,
|
||||
comment,
|
||||
url,
|
||||
date,
|
||||
processed_by,
|
||||
redacted_by,
|
||||
copyright,
|
||||
theme,
|
||||
kind,
|
||||
source,
|
||||
rating,
|
||||
})
|
||||
}
|
||||
}
|
||||
const FIELDS: &[&str] = &[
|
||||
"filename",
|
||||
"description",
|
||||
"author",
|
||||
"comment",
|
||||
"url",
|
||||
"date",
|
||||
"processed_by",
|
||||
"redacted_by",
|
||||
"copyright",
|
||||
"theme",
|
||||
"kind",
|
||||
"source",
|
||||
"rating",
|
||||
];
|
||||
Deserializer::deserialize_struct(
|
||||
deserializer,
|
||||
"BatchInfo",
|
||||
FIELDS,
|
||||
BatchInfoVisitor {
|
||||
marker: PhantomData::<BatchInfo>,
|
||||
lifetime: PhantomData,
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for Question {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
#[allow(non_camel_case_types)]
|
||||
enum QuestionField {
|
||||
num,
|
||||
id,
|
||||
description,
|
||||
answer,
|
||||
author,
|
||||
comment,
|
||||
comment1,
|
||||
tour,
|
||||
url,
|
||||
date,
|
||||
processed_by,
|
||||
redacted_by,
|
||||
copyright,
|
||||
theme,
|
||||
kind,
|
||||
source,
|
||||
rating,
|
||||
batch_info,
|
||||
_flags,
|
||||
__ignore,
|
||||
}
|
||||
|
||||
struct QuestionFieldVisitor;
|
||||
impl<'de> Visitor<'de> for QuestionFieldVisitor {
|
||||
type Value = QuestionField;
|
||||
fn expecting(&self, formatter: &mut Formatter) -> fmt::Result {
|
||||
Formatter::write_str(formatter, "field identifier")
|
||||
}
|
||||
fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
|
||||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
match value {
|
||||
0u64 => Ok(QuestionField::num),
|
||||
1u64 => Ok(QuestionField::id),
|
||||
2u64 => Ok(QuestionField::description),
|
||||
3u64 => Ok(QuestionField::answer),
|
||||
4u64 => Ok(QuestionField::author),
|
||||
5u64 => Ok(QuestionField::comment),
|
||||
6u64 => Ok(QuestionField::comment1),
|
||||
7u64 => Ok(QuestionField::tour),
|
||||
8u64 => Ok(QuestionField::url),
|
||||
9u64 => Ok(QuestionField::date),
|
||||
10u64 => Ok(QuestionField::processed_by),
|
||||
11u64 => Ok(QuestionField::redacted_by),
|
||||
12u64 => Ok(QuestionField::copyright),
|
||||
13u64 => Ok(QuestionField::theme),
|
||||
14u64 => Ok(QuestionField::kind),
|
||||
15u64 => Ok(QuestionField::source),
|
||||
16u64 => Ok(QuestionField::rating),
|
||||
17u64 => Ok(QuestionField::batch_info),
|
||||
18u64 => Ok(QuestionField::_flags),
|
||||
_ => Ok(QuestionField::__ignore),
|
||||
}
|
||||
}
|
||||
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
|
||||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
match_visit_str_fields!(value => QuestionField {
|
||||
num, id, description, answer, author, comment, comment1, tour, url, date, processed_by,
|
||||
redacted_by, copyright, theme, kind, source, rating, batch_info, _flags, __ignore
|
||||
})
|
||||
}
|
||||
fn visit_bytes<E>(self, value: &[u8]) -> Result<Self::Value, E>
|
||||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
match_visit_str_fields!([bytes] value => QuestionField {
|
||||
num, id, description, answer, author, comment, comment1, tour, url, date, processed_by,
|
||||
redacted_by, copyright, theme, kind, source, rating, batch_info, _flags, __ignore
|
||||
})
|
||||
}
|
||||
}
|
||||
impl<'de> Deserialize<'de> for QuestionField {
|
||||
#[inline]
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
Deserializer::deserialize_identifier(deserializer, QuestionFieldVisitor)
|
||||
}
|
||||
}
|
||||
#[doc(hidden)]
|
||||
struct QuestionVisitor<'de> {
|
||||
marker: PhantomData<Question>,
|
||||
lifetime: PhantomData<&'de ()>,
|
||||
}
|
||||
impl<'de> Visitor<'de> for QuestionVisitor<'de> {
|
||||
type Value = Question;
|
||||
fn expecting(&self, formatter: &mut Formatter) -> fmt::Result {
|
||||
Formatter::write_str(formatter, "struct Question")
|
||||
}
|
||||
#[inline]
|
||||
fn visit_seq<V>(self, mut seq: V) -> Result<Self::Value, V::Error>
|
||||
where
|
||||
V: SeqAccess<'de>,
|
||||
{
|
||||
let _flags = QuestionFlags::from_bits(
|
||||
SeqAccess::next_element::<u16>(&mut seq)?.unwrap_or_default(),
|
||||
)
|
||||
.unwrap_or_default();
|
||||
|
||||
let num: u32 = if _flags.intersects(QuestionFlags::NUM) {
|
||||
SeqAccess::next_element::<u32>(&mut seq)?.unwrap_or_default()
|
||||
} else {
|
||||
Default::default()
|
||||
};
|
||||
|
||||
let id = match SeqAccess::next_element::<String>(&mut seq)? {
|
||||
Some(value) => value,
|
||||
_ => return Err(<V::Error as serde::de::Error>::missing_field("id")),
|
||||
};
|
||||
let description = match SeqAccess::next_element::<String>(&mut seq)? {
|
||||
Some(value) => value,
|
||||
_ => return Err(<V::Error as serde::de::Error>::missing_field("description")),
|
||||
};
|
||||
let answer = match SeqAccess::next_element::<String>(&mut seq)? {
|
||||
Some(value) => value,
|
||||
_ => return Err(<V::Error as serde::de::Error>::missing_field("answer")),
|
||||
};
|
||||
|
||||
seq_read_strings!((_flags, seq, QuestionFlags) <- {
|
||||
author: AUTHOR, comment: COMMENT, comment1: COMMENT1, tour: TOUR, url: URL, date: DATE,
|
||||
processed_by: PROCESSED_BY, redacted_by: REDACTED_BY, copyright: COPYRIGHT, theme: THEME,
|
||||
kind: KIND, source: SOURCE, rating: RATING
|
||||
});
|
||||
|
||||
let batch_info: BatchInfo = if _flags.intersects(QuestionFlags::BATCH_INFO) {
|
||||
SeqAccess::next_element::<BatchInfo>(&mut seq)?.unwrap_or_default()
|
||||
} else {
|
||||
Default::default()
|
||||
};
|
||||
|
||||
Ok(Question {
|
||||
num,
|
||||
id,
|
||||
description,
|
||||
answer,
|
||||
author,
|
||||
comment,
|
||||
comment1,
|
||||
tour,
|
||||
url,
|
||||
date,
|
||||
processed_by,
|
||||
redacted_by,
|
||||
copyright,
|
||||
theme,
|
||||
kind,
|
||||
source,
|
||||
rating,
|
||||
batch_info,
|
||||
})
|
||||
}
|
||||
#[inline]
|
||||
fn visit_map<V>(self, mut map: V) -> Result<Self::Value, V::Error>
|
||||
where
|
||||
V: MapAccess<'de>,
|
||||
{
|
||||
let mut num: Option<u32> = None;
|
||||
let mut id: Option<String> = None;
|
||||
let mut description: Option<String> = None;
|
||||
let mut answer: Option<String> = None;
|
||||
let mut author: Option<String> = None;
|
||||
let mut comment: Option<String> = None;
|
||||
let mut comment1: Option<String> = None;
|
||||
let mut tour: Option<String> = None;
|
||||
let mut url: Option<String> = None;
|
||||
let mut date: Option<String> = None;
|
||||
let mut processed_by: Option<String> = None;
|
||||
let mut redacted_by: Option<String> = None;
|
||||
let mut copyright: Option<String> = None;
|
||||
let mut theme: Option<String> = None;
|
||||
let mut kind: Option<String> = None;
|
||||
let mut source: Option<String> = None;
|
||||
let mut rating: Option<String> = None;
|
||||
let mut batch_info: Option<BatchInfo> = None;
|
||||
|
||||
while let Some(key) = MapAccess::next_key::<QuestionField>(&mut map)? {
|
||||
match_map_fields!((map, key, QuestionField) <- {
|
||||
num: u32, id: String, description: String, answer: String, author: String, comment: String,
|
||||
comment1: String, tour: String, url: String, date: String, processed_by: String, redacted_by: String,
|
||||
copyright: String, theme: String, kind: String, source: String, rating: String, batch_info: BatchInfo
|
||||
});
|
||||
}
|
||||
|
||||
let id = id.ok_or(<V::Error as serde::de::Error>::missing_field("id"))?;
|
||||
let description = description
|
||||
.ok_or(<V::Error as serde::de::Error>::missing_field("description"))?;
|
||||
let answer =
|
||||
answer.ok_or(<V::Error as serde::de::Error>::missing_field("answer"))?;
|
||||
|
||||
let num = num.unwrap_or_default();
|
||||
let author = author.unwrap_or_default();
|
||||
let comment = comment.unwrap_or_default();
|
||||
let comment1 = comment1.unwrap_or_default();
|
||||
let tour = tour.unwrap_or_default();
|
||||
let url = url.unwrap_or_default();
|
||||
let date = date.unwrap_or_default();
|
||||
let processed_by = processed_by.unwrap_or_default();
|
||||
let redacted_by = redacted_by.unwrap_or_default();
|
||||
let copyright = copyright.unwrap_or_default();
|
||||
let theme = theme.unwrap_or_default();
|
||||
let kind = kind.unwrap_or_default();
|
||||
let source = source.unwrap_or_default();
|
||||
let rating = rating.unwrap_or_default();
|
||||
let batch_info = batch_info.unwrap_or_default();
|
||||
|
||||
Ok(Question {
|
||||
num,
|
||||
id,
|
||||
description,
|
||||
answer,
|
||||
author,
|
||||
comment,
|
||||
comment1,
|
||||
tour,
|
||||
url,
|
||||
date,
|
||||
processed_by,
|
||||
redacted_by,
|
||||
copyright,
|
||||
theme,
|
||||
kind,
|
||||
source,
|
||||
rating,
|
||||
batch_info,
|
||||
})
|
||||
}
|
||||
}
|
||||
#[doc(hidden)]
|
||||
const FIELDS: &[&str] = &[
|
||||
"num",
|
||||
"id",
|
||||
"description",
|
||||
"answer",
|
||||
"author",
|
||||
"comment",
|
||||
"comment1",
|
||||
"tour",
|
||||
"url",
|
||||
"date",
|
||||
"processed_by",
|
||||
"redacted_by",
|
||||
"copyright",
|
||||
"theme",
|
||||
"kind",
|
||||
"source",
|
||||
"rating",
|
||||
"batch_info",
|
||||
];
|
||||
Deserializer::deserialize_struct(
|
||||
deserializer,
|
||||
"Question",
|
||||
FIELDS,
|
||||
QuestionVisitor {
|
||||
marker: PhantomData::<Question>,
|
||||
lifetime: PhantomData,
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "convert", feature = "convert_async"))]
|
||||
pub mod convert_common {
|
||||
use super::{BatchInfo, Question};
|
||||
@ -372,6 +1084,18 @@ mod test {
|
||||
"#);
|
||||
}
|
||||
#[test]
|
||||
fn test_question_serde_postcard() {
|
||||
use postcard::{from_bytes, to_slice};
|
||||
|
||||
let original = sample_question();
|
||||
let mut buff: Vec<u8> = vec![0; 135];
|
||||
|
||||
to_slice(&original, buff.as_mut_slice()).expect("to_slice");
|
||||
let actual: Question = from_bytes(buff.as_slice()).expect("from_bytes");
|
||||
|
||||
assert_eq!(actual, original);
|
||||
}
|
||||
#[test]
|
||||
fn test_question_de() {
|
||||
let question_from_json: Result<Question, _> = serde_json::from_value(json!({
|
||||
"id": "Вопрос 1",
|
||||
|
||||
@ -12,46 +12,3 @@ where
|
||||
self.map_err(|e| e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "sync", feature = "async"))]
|
||||
mod bincode_utils {
|
||||
use std::ops::{Deref, DerefMut};
|
||||
|
||||
use bincode::enc::write::Writer;
|
||||
use bincode::error::EncodeError;
|
||||
|
||||
/// struct that allows [`Vec<u8>`] to implement [bincode::enc::write::Writer] trait
|
||||
pub struct BincodeVecWriter {
|
||||
vec: Vec<u8>,
|
||||
}
|
||||
|
||||
impl BincodeVecWriter {
|
||||
pub fn new(vec: Vec<u8>) -> BincodeVecWriter {
|
||||
BincodeVecWriter { vec }
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for BincodeVecWriter {
|
||||
type Target = Vec<u8>;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.vec
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for BincodeVecWriter {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.vec
|
||||
}
|
||||
}
|
||||
|
||||
impl Writer for BincodeVecWriter {
|
||||
fn write(&mut self, bytes: &[u8]) -> Result<(), EncodeError> {
|
||||
self.vec.extend_from_slice(bytes);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "sync", feature = "async"))]
|
||||
pub use bincode_utils::BincodeVecWriter;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user