test fmt 1

src/main.rs

@@ -9,6 +9,7 @@ extern crate zip;
 use clap::{Parser, Subcommand};
 use rand::seq::IteratorRandom;
 
+use std::io::{Read, Write};
 use std::path::PathBuf;
 use std::time::Instant;
 use std::{fs, io, sync::mpsc, thread};
@@ -38,6 +39,11 @@ enum Command {
         #[clap(value_parser, default_value = "0")]
         num: usize,
     },
+    Write2,
+    Print2 {
+        #[clap(value_parser, default_value = "0")]
+        id: u32,
+    },
 }
 
 #[derive(Parser, Debug)]
@@ -210,6 +216,11 @@ fn main() {
             let get_question = Box::new(|| read_from_zip(*file_num, *num));
             Box::new(|| print_question_from(get_question))
         }
+        Command::Write2 => Box::new(write_db2),
+        Command::Print2 { id } => {
+            let get_question = Box::new(|| read_from_db2(*id));
+            Box::new(|| print_question_from(get_question))
+        }
     };
 
     if args.measure {
@@ -218,3 +229,101 @@ fn main() {
 
     action();
 }
+
+#[derive(bincode::Decode, bincode::Encode)]
+struct VEntry {
+    pos: u32,
+    len: u16,
+}
+
+fn read_from_db2(id: u32) -> Option<Question> {
+    use std::io::Cursor;
+    const LEN_SIZE: usize = std::mem::size_of::<u64>();
+    let cfg = bincode::config::standard().with_fixed_int_encoding();
+
+    let input = fs::File::open("test.bin").expect("open input");
+    let mut input = std::io::BufReader::with_capacity(100 * 1024 * 1024, input);
+
+    let mut len_data: [u8; LEN_SIZE] = [0; LEN_SIZE];
+    input.read_exact(&mut len_data).expect("read len");
+    let len = u64::from_le_bytes(len_data) as usize;
+
+    //println!("read len done");
+
+    let mut zdata = vec![0u8; len];
+    input.read_exact(&mut zdata[..len]).expect("read ztab");
+    let tab_data = zstd::decode_all(Cursor::new(zdata)).expect("zstd decode table");
+    let tab: (Vec<VEntry>, usize) =
+        bincode::decode_from_slice(&tab_data, cfg).expect("bincode decode tab");
+    let tab = tab.0;
+
+    //println!("read tab done");
+
+    let choosen = tab.get(id as usize).expect("get by id");
+    input.seek_relative(choosen.pos as i64).expect("q seek");
+    let reader = input.take(choosen.len as u64);
+    let data = zstd::decode_all(reader).expect("zstd decode data");
+
+    let question: (Question, usize) =
+        bincode::decode_from_slice(&data, cfg).expect("bincode decode q");
+    let question = question.0;
+
+    //println!("read done");
+
+    Some(question)
+}
+fn write_db2() {
+    let (tx, rx) = mpsc::channel::<Question>();
+    [
+        thread::spawn(move || reader_task(tx)),
+        thread::spawn(move || db_writer2_task(rx)),
+    ]
+    .into_iter()
+    .for_each(|handle| handle.join().expect("thread panic"));
+    println!("all done");
+}
+fn db_writer2_task(rx: mpsc::Receiver<Question>) {
+    const LEN_SIZE: usize = std::mem::size_of::<u64>();
+
+    let cfg = bincode::config::standard().with_fixed_int_encoding();
+    let mut table: Vec<VEntry> = vec![];
+
+    let buf_data: Vec<u8> = Vec::with_capacity(500 * 1024 * 1024);
+    use std::io::Cursor;
+    let mut buf = Cursor::new(buf_data);
+    let mut pos: u32 = 0;
+
+    rx.into_iter().for_each(|q| {
+        let data = bincode::encode_to_vec(q, cfg).expect("bincode q encode");
+        let data = zstd::encode_all(Cursor::new(data), 3).expect("zstd q encode");
+        let len = buf.write(&data).expect("write question");
+        table.push(VEntry {
+            pos,
+            len: len.try_into().expect("len try_into len"),
+        });
+        pos += len as u32;
+    });
+
+    buf.set_position(0);
+
+    println!("zbuf done");
+
+    let tab_data = bincode::encode_to_vec(&table, cfg).expect("encode table");
+    let zdata = zstd::encode_all(Cursor::new(tab_data), 3).expect("zstd enc table");
+    let zlen = zdata.len() as u64;
+
+    println!("z tab done");
+
+    let out = fs::File::create("test.bin").expect("out create");
+    let mut out = std::io::BufWriter::with_capacity(500 * 1024 * 1024, out);
+    let len_writed = out.write(&zlen.to_le_bytes()).expect("write zlen");
+    assert_eq!(len_writed, LEN_SIZE);
+    let ztab_writed = out.write(&zdata).expect("write tab zdata");
+    assert_eq!(ztab_writed, zdata.len());
+    drop(zdata);
+    std::io::copy(&mut buf, &mut out).expect("copy z buf");
+    drop(buf);
+    out.flush().expect("out flush");
+
+    println!("write done");
+}

src/questions.rs

@@ -12,7 +12,9 @@ macro_rules! make {
     ),+ ,..$Target::default()}}
 }
 
-#[derive(Debug, Default, Clone, Serialize, Deserialize, Document)]
+#[derive(
+    Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode,
+)]
 pub struct BatchInfo {
     #[document(primary)]
     #[serde(default)]
@@ -43,7 +45,9 @@ pub struct BatchInfo {
     pub rating: String,
 }
 
-#[derive(Debug, Default, Clone, Serialize, Deserialize, Document)]
+#[derive(
+    Debug, Default, Clone, Serialize, Deserialize, Document, bincode::Decode, bincode::Encode,
+)]
 pub struct Question {
     #[document(primary)]
     #[serde(default)]