644 lines
20 KiB
Rust
644 lines
20 KiB
Rust
use serde_derive::{Deserialize, Serialize};
|
||
|
||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq)]
|
||
pub struct SourceQuestion {
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "u32_is_zero")]
|
||
pub num: u32,
|
||
pub id: String,
|
||
|
||
#[serde(alias = "Вопрос")]
|
||
pub description: String,
|
||
#[serde(alias = "Ответ")]
|
||
pub answer: String,
|
||
|
||
#[serde(alias = "Автор")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub author: String,
|
||
#[serde(alias = "Комментарий")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub comment: String,
|
||
#[serde(alias = "Комментарии")]
|
||
#[serde(alias = "Инфо")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub comment1: String,
|
||
#[serde(alias = "Тур")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub tour: String,
|
||
#[serde(alias = "Ссылка")]
|
||
#[serde(alias = "URL")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub url: String,
|
||
#[serde(alias = "Дата")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub date: String,
|
||
#[serde(alias = "Обработан")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub processed_by: String,
|
||
#[serde(alias = "Редактор")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub redacted_by: String,
|
||
#[serde(alias = "Копирайт")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub copyright: String,
|
||
#[serde(alias = "Тема")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub theme: String,
|
||
#[serde(alias = "Вид")]
|
||
#[serde(alias = "Тип")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub kind: String,
|
||
#[serde(alias = "Источник")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub source: String,
|
||
#[serde(alias = "Рейтинг")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub rating: String,
|
||
}
|
||
|
||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq)]
|
||
pub struct SourceQuestionsBatch {
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub filename: String,
|
||
#[serde(alias = "Пакет")]
|
||
#[serde(alias = "Чемпионат")]
|
||
pub description: String,
|
||
#[serde(alias = "Автор")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub author: String,
|
||
#[serde(alias = "Комментарий")]
|
||
#[serde(alias = "Комментарии")]
|
||
#[serde(alias = "Инфо")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub comment: String,
|
||
#[serde(alias = "Ссылка")]
|
||
#[serde(alias = "URL")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub url: String,
|
||
#[serde(alias = "Дата")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub date: String,
|
||
#[serde(alias = "Обработан")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub processed_by: String,
|
||
#[serde(alias = "Редактор")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub redacted_by: String,
|
||
#[serde(alias = "Копирайт")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub copyright: String,
|
||
#[serde(alias = "Тема")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub theme: String,
|
||
#[serde(alias = "Вид")]
|
||
#[serde(alias = "Тип")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub kind: String,
|
||
#[serde(alias = "Источник")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub source: String,
|
||
#[serde(alias = "Рейтинг")]
|
||
#[serde(default)]
|
||
#[serde(skip_serializing_if = "String::is_empty")]
|
||
pub rating: String,
|
||
#[serde(alias = "Вопросы")]
|
||
pub questions: Vec<SourceQuestion>,
|
||
}
|
||
|
||
fn u32_is_zero(num: &u32) -> bool {
|
||
*num == 0
|
||
}
|
||
|
||
#[cfg(any(feature = "convert", feature = "source"))]
|
||
pub mod reader_sync {
|
||
use std::io::{Read, Seek};
|
||
use zip::ZipArchive;
|
||
|
||
use super::SourceQuestionsBatch;
|
||
|
||
pub struct SourceQuestionsZipReader<R>
|
||
where
|
||
R: Read + Seek,
|
||
{
|
||
zipfile: ZipArchive<R>,
|
||
index: Option<usize>,
|
||
}
|
||
|
||
impl<R> SourceQuestionsZipReader<R>
|
||
where
|
||
R: Read + Seek,
|
||
{
|
||
fn new(zipfile: ZipArchive<R>) -> Self {
|
||
SourceQuestionsZipReader {
|
||
zipfile,
|
||
index: None,
|
||
}
|
||
}
|
||
}
|
||
|
||
impl<R> Iterator for SourceQuestionsZipReader<R>
|
||
where
|
||
R: Read + Seek,
|
||
{
|
||
type Item = (String, Result<SourceQuestionsBatch, serde_json::Error>);
|
||
|
||
fn next(&mut self) -> Option<Self::Item> {
|
||
if self.index.is_none() && !self.zipfile.is_empty() {
|
||
self.index = Some(0);
|
||
}
|
||
|
||
match self.index {
|
||
Some(i) if i < self.zipfile.len() => {
|
||
self.index = Some(i + 1);
|
||
|
||
self.nth(i)
|
||
}
|
||
_ => None,
|
||
}
|
||
}
|
||
|
||
fn nth(&mut self, n: usize) -> Option<Self::Item> {
|
||
if self.zipfile.len() <= n {
|
||
return None;
|
||
}
|
||
self.index = Some(n + 1);
|
||
|
||
let file = self.zipfile.by_index(n).unwrap();
|
||
let name = file.mangled_name();
|
||
let name_str = name.to_str().unwrap();
|
||
|
||
let data: Result<SourceQuestionsBatch, _> = serde_json::from_reader(file);
|
||
|
||
Some((String::from(name_str), data))
|
||
}
|
||
|
||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||
let len = self.zipfile.len();
|
||
let index = self.index.unwrap_or(0);
|
||
let rem = if len > index + 1 {
|
||
len - (index + 1)
|
||
} else {
|
||
0
|
||
};
|
||
(rem, Some(rem))
|
||
}
|
||
|
||
fn count(self) -> usize
|
||
where
|
||
Self: Sized,
|
||
{
|
||
self.zipfile.len()
|
||
}
|
||
}
|
||
|
||
impl<R> ExactSizeIterator for SourceQuestionsZipReader<R>
|
||
where
|
||
R: Read + Seek,
|
||
{
|
||
fn len(&self) -> usize {
|
||
self.zipfile.len()
|
||
}
|
||
}
|
||
|
||
pub trait ReadSourceQuestionsBatches<R>
|
||
where
|
||
R: Read + Seek,
|
||
{
|
||
fn source_questions(self) -> SourceQuestionsZipReader<R>;
|
||
}
|
||
|
||
impl<R> ReadSourceQuestionsBatches<R> for ZipArchive<R>
|
||
where
|
||
R: Read + Seek,
|
||
{
|
||
fn source_questions(self) -> SourceQuestionsZipReader<R> {
|
||
SourceQuestionsZipReader::new(self)
|
||
}
|
||
}
|
||
#[cfg(test)]
|
||
mod test {
|
||
use crate::source::SourceQuestion;
|
||
|
||
use super::*;
|
||
use std::fs;
|
||
use std::{io::Write, iter, path::Path};
|
||
use tempfile::tempdir;
|
||
|
||
use super::SourceQuestionsBatch;
|
||
|
||
fn sample_batch() -> SourceQuestionsBatch {
|
||
SourceQuestionsBatch {
|
||
description: "Тестовый".into(),
|
||
date: "00-000-2000".into(),
|
||
questions: vec![
|
||
SourceQuestion {
|
||
id: "Вопрос 1".into(),
|
||
description: "Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2".into(),
|
||
answer: "42".into(),
|
||
..Default::default()
|
||
},
|
||
SourceQuestion {
|
||
id: "Вопрос 2".into(),
|
||
description: "Зимой и летом одним цветом".into(),
|
||
answer: "ёлка".into(),
|
||
..Default::default()
|
||
},
|
||
],
|
||
..Default::default()
|
||
}
|
||
}
|
||
|
||
fn write_sample_zip<P>(path: P)
|
||
where
|
||
P: AsRef<Path>,
|
||
{
|
||
let batch = sample_batch();
|
||
let z_file = fs::File::create(path).expect("crerate zip file");
|
||
let mut zip_file = zip::ZipWriter::new(z_file);
|
||
let options =
|
||
zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Zstd);
|
||
zip_file
|
||
.start_file("test.json", options)
|
||
.expect("zip start file");
|
||
zip_file
|
||
.write(serde_json::to_vec(&batch).unwrap().as_slice())
|
||
.expect("write entry");
|
||
zip_file.finish().expect("finish zip file");
|
||
}
|
||
|
||
#[test]
|
||
fn test_source_questions_get() {
|
||
let expected_batch = sample_batch();
|
||
let dir = tempdir().expect("tempdir");
|
||
|
||
// write sample
|
||
let tmpfile_zip = dir.path().join("test.zip");
|
||
write_sample_zip(&tmpfile_zip);
|
||
|
||
let z_file = fs::File::open(tmpfile_zip).expect("open zip file");
|
||
let zip_file = zip::ZipArchive::new(z_file).expect("open zip file reader");
|
||
|
||
let mut source = zip_file.source_questions();
|
||
assert_eq!(source.len(), 1);
|
||
|
||
let actual = source.nth(0).expect("get batch");
|
||
assert_eq!(actual.0, "test.json");
|
||
assert_eq!(actual.1.expect("parse batch"), expected_batch);
|
||
}
|
||
|
||
#[test]
|
||
fn test_source_questions_iter() {
|
||
let expected_batch = sample_batch();
|
||
let dir = tempdir().expect("tempdir");
|
||
|
||
// write sample
|
||
let tmpfile_zip = dir.path().join("test.zip");
|
||
write_sample_zip(&tmpfile_zip);
|
||
|
||
let z_file = fs::File::open(tmpfile_zip).expect("open zip file");
|
||
let zip_file = zip::ZipArchive::new(z_file).expect("open zip file reader");
|
||
|
||
let source = zip_file.source_questions();
|
||
assert_eq!(source.len(), 1);
|
||
|
||
let expected_iter = iter::once((String::from("test.json"), Ok(expected_batch)));
|
||
|
||
assert!(source
|
||
.map(|x| (x.0, x.1.map_err(|e| e.to_string())))
|
||
.eq(expected_iter));
|
||
}
|
||
}
|
||
}
|
||
|
||
#[cfg(any(feature = "convert", feature = "source"))]
|
||
pub use reader_sync::{ReadSourceQuestionsBatches, SourceQuestionsZipReader};
|
||
|
||
#[cfg(any(feature = "convert_async", feature = "source_async"))]
|
||
pub mod reader_async {
|
||
use async_stream::stream;
|
||
use async_zip::tokio::read::seek::ZipFileReader;
|
||
use futures_core::stream::Stream;
|
||
use futures_util::AsyncReadExt;
|
||
|
||
use tokio::io::{AsyncRead, AsyncSeek};
|
||
|
||
use super::SourceQuestionsBatch;
|
||
|
||
pub struct SourceQuestionsZipReaderAsync<R>
|
||
where
|
||
R: AsyncRead + AsyncSeek + Unpin,
|
||
{
|
||
zipfile: ZipFileReader<R>,
|
||
index: Option<usize>,
|
||
}
|
||
|
||
impl<R> SourceQuestionsZipReaderAsync<R>
|
||
where
|
||
R: AsyncRead + AsyncSeek + Unpin,
|
||
{
|
||
fn new(zipfile: ZipFileReader<R>) -> Self {
|
||
SourceQuestionsZipReaderAsync {
|
||
zipfile,
|
||
index: None,
|
||
}
|
||
}
|
||
|
||
pub fn len(&self) -> usize {
|
||
self.zipfile.file().entries().len()
|
||
}
|
||
|
||
pub fn is_empty(&self) -> bool {
|
||
self.len() == 0
|
||
}
|
||
|
||
pub async fn get(
|
||
&mut self,
|
||
index: usize,
|
||
) -> Result<(String, Result<SourceQuestionsBatch, serde_json::Error>), String>
|
||
where
|
||
R: AsyncRead + AsyncSeek + Unpin,
|
||
{
|
||
let len = self.len();
|
||
if index >= len {
|
||
return Err(format!("get index={index}, when len={len}"));
|
||
}
|
||
|
||
let reader = self.zipfile.reader_with_entry(index).await;
|
||
if let Err(error) = reader {
|
||
return Err(format!("reader_with_entry: {error:?}"));
|
||
}
|
||
let mut reader = reader.unwrap();
|
||
|
||
let filename = reader.entry().filename().clone().into_string().unwrap();
|
||
let mut data: Vec<u8> = Vec::new();
|
||
let readed = reader.read_to_end(&mut data).await;
|
||
if let Err(error) = readed {
|
||
return Err(format!("read_to_end: {error:?}"));
|
||
}
|
||
let parsed: Result<SourceQuestionsBatch, _> = serde_json::from_slice(&data);
|
||
Ok((filename, parsed))
|
||
}
|
||
pub async fn get_next(
|
||
&mut self,
|
||
) -> Option<Result<(String, Result<SourceQuestionsBatch, serde_json::Error>), String>>
|
||
where
|
||
R: AsyncRead + AsyncSeek + Unpin,
|
||
{
|
||
if self.index.is_none() && !self.is_empty() {
|
||
self.index = Some(0);
|
||
}
|
||
|
||
if self.index.unwrap() >= self.len() {
|
||
return None;
|
||
}
|
||
|
||
let item = self.get(self.index.unwrap()).await;
|
||
self.index = Some(self.index.unwrap() + 1);
|
||
|
||
Some(item)
|
||
}
|
||
pub fn stream(
|
||
&mut self,
|
||
) -> impl Stream<Item = (String, Result<SourceQuestionsBatch, serde_json::Error>)> + '_
|
||
{
|
||
stream! {
|
||
while let Some(Ok(item)) = self.get_next().await {
|
||
yield item
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
pub trait ReadSourceQuestionsBatchesAsync<R>
|
||
where
|
||
R: AsyncRead + AsyncSeek + Unpin,
|
||
{
|
||
fn source_questions(self) -> SourceQuestionsZipReaderAsync<R>;
|
||
}
|
||
|
||
impl<R> ReadSourceQuestionsBatchesAsync<R> for ZipFileReader<R>
|
||
where
|
||
R: AsyncRead + AsyncSeek + Unpin,
|
||
{
|
||
fn source_questions(self) -> SourceQuestionsZipReaderAsync<R> {
|
||
SourceQuestionsZipReaderAsync::new(self)
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod test {
|
||
use crate::source::SourceQuestion;
|
||
|
||
use super::*;
|
||
use async_zip::{base::write::ZipFileWriter, ZipEntryBuilder};
|
||
use core::fmt::Debug;
|
||
use futures_util::StreamExt;
|
||
use std::path::Path;
|
||
use tempfile::tempdir;
|
||
use tokio::fs;
|
||
|
||
fn sample_batch() -> SourceQuestionsBatch {
|
||
SourceQuestionsBatch {
|
||
description: "Тестовый".into(),
|
||
date: "00-000-2000".into(),
|
||
questions: vec![
|
||
SourceQuestion {
|
||
id: "Вопрос 1".into(),
|
||
description: "Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2".into(),
|
||
answer: "42".into(),
|
||
..Default::default()
|
||
},
|
||
SourceQuestion {
|
||
id: "Вопрос 2".into(),
|
||
description: "Зимой и летом одним цветом".into(),
|
||
answer: "ёлка".into(),
|
||
..Default::default()
|
||
},
|
||
],
|
||
..Default::default()
|
||
}
|
||
}
|
||
|
||
async fn write_sample_zip<P>(path: P)
|
||
where
|
||
P: AsRef<Path>,
|
||
{
|
||
let batch = sample_batch();
|
||
let z_file = fs::File::create(path).await.expect("crerate zip file");
|
||
let mut zip_file = ZipFileWriter::with_tokio(z_file);
|
||
let entry =
|
||
ZipEntryBuilder::new("test.json".into(), async_zip::Compression::Zstd).build();
|
||
zip_file
|
||
.write_entry_whole(entry, serde_json::to_vec(&batch).unwrap().as_slice())
|
||
.await
|
||
.expect("write entry");
|
||
zip_file.close().await.expect("close zip");
|
||
}
|
||
|
||
async fn assert_data_rref_eq<T>((x, y): (T, &T))
|
||
where
|
||
T: PartialEq + Debug,
|
||
{
|
||
assert_eq!(x, *y);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_source_questions_stream() {
|
||
let expected_batch = sample_batch();
|
||
let dir = tempdir().expect("tempdir");
|
||
|
||
// write sample
|
||
let tmpfile_zip = dir.path().join("test.zip");
|
||
write_sample_zip(&tmpfile_zip).await;
|
||
|
||
let mut z_file = fs::File::open(tmpfile_zip).await.expect("open zip file");
|
||
let zip_file = ZipFileReader::with_tokio(&mut z_file)
|
||
.await
|
||
.expect("open zip file reader");
|
||
|
||
let expected_count = expected_batch.questions.len();
|
||
let expected_stream = futures::stream::iter(expected_batch.questions.iter());
|
||
let mut actual_source = zip_file.source_questions();
|
||
let actual_stream = actual_source.stream();
|
||
let mut actual_count: usize = 0;
|
||
|
||
actual_stream
|
||
.flat_map(|x| futures::stream::iter(x.1.expect("parse batch").questions))
|
||
.zip(expected_stream)
|
||
.map(|x| {
|
||
actual_count += 1;
|
||
x
|
||
})
|
||
.for_each(assert_data_rref_eq::<SourceQuestion>)
|
||
.await;
|
||
assert_eq!(actual_count, expected_count);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_source_questions_get() {
|
||
let expected_batch = sample_batch();
|
||
let dir = tempdir().expect("tempdir");
|
||
|
||
// write sample
|
||
let tmpfile_zip = dir.path().join("test.zip");
|
||
write_sample_zip(&tmpfile_zip).await;
|
||
|
||
let mut z_file = fs::File::open(tmpfile_zip).await.expect("open zip file");
|
||
let zip_file = ZipFileReader::with_tokio(&mut z_file)
|
||
.await
|
||
.expect("open zip file reader");
|
||
|
||
let mut source = zip_file.source_questions();
|
||
assert_eq!(source.len(), 1);
|
||
|
||
let actual = source.get(0).await.expect("get batch");
|
||
assert_eq!(actual.0, "test.json");
|
||
assert_eq!(actual.1.expect("parse batch"), expected_batch);
|
||
}
|
||
}
|
||
}
|
||
#[cfg(any(feature = "convert_async", feature = "source_async"))]
|
||
pub use reader_async::{ReadSourceQuestionsBatchesAsync, SourceQuestionsZipReaderAsync};
|
||
|
||
#[cfg(test)]
|
||
mod test {
|
||
use super::*;
|
||
use insta::assert_yaml_snapshot;
|
||
use serde_json::json;
|
||
|
||
#[test]
|
||
fn test_batch_ser() {
|
||
let batch = SourceQuestionsBatch {
|
||
description: "Тестовый".into(),
|
||
date: "00-000-2000".into(),
|
||
questions: vec![
|
||
SourceQuestion {
|
||
id: "Вопрос 1".into(),
|
||
description: "Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2".into(),
|
||
answer: "42".into(),
|
||
..Default::default()
|
||
},
|
||
SourceQuestion {
|
||
id: "Вопрос 2".into(),
|
||
description: "Зимой и летом одним цветом".into(),
|
||
answer: "ёлка".into(),
|
||
..Default::default()
|
||
},
|
||
],
|
||
..Default::default()
|
||
};
|
||
|
||
assert_yaml_snapshot!(batch, @r#"
|
||
---
|
||
description: Тестовый
|
||
date: 00-000-2000
|
||
questions:
|
||
- id: Вопрос 1
|
||
description: Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2
|
||
answer: "42"
|
||
- id: Вопрос 2
|
||
description: Зимой и летом одним цветом
|
||
answer: ёлка
|
||
|
||
"#);
|
||
}
|
||
#[test]
|
||
fn test_batch_de() {
|
||
let batch_from_json: Result<SourceQuestionsBatch, _> = serde_json::from_value(json!({
|
||
"Чемпионат": "Тестовый",
|
||
"Дата": "00-000-2000",
|
||
"Вопросы": [
|
||
{
|
||
"id": "Вопрос 1",
|
||
"Вопрос": "Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2",
|
||
"Ответ": "42",
|
||
},
|
||
{
|
||
"id": "Вопрос 2",
|
||
"Вопрос": "Зимой и летом одним цветом",
|
||
"Ответ": "ёлка",
|
||
},
|
||
]
|
||
}));
|
||
assert!(batch_from_json.is_ok());
|
||
|
||
assert_yaml_snapshot!(batch_from_json.unwrap(), @r#"
|
||
---
|
||
description: Тестовый
|
||
date: 00-000-2000
|
||
questions:
|
||
- id: Вопрос 1
|
||
description: Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2
|
||
answer: "42"
|
||
- id: Вопрос 2
|
||
description: Зимой и летом одним цветом
|
||
answer: ёлка
|
||
|
||
"#);
|
||
}
|
||
}
|