576 lines
18 KiB
Rust
576 lines
18 KiB
Rust
use serde_derive::{Deserialize, Serialize};
|
|
|
|
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq)]
|
|
pub struct SourceQuestion {
|
|
#[serde(default, skip_serializing_if = "u32_is_zero")]
|
|
pub num: u32,
|
|
#[serde(default)]
|
|
pub id: String,
|
|
|
|
#[serde(alias = "Вопрос")]
|
|
pub description: String,
|
|
#[serde(alias = "Ответ")]
|
|
pub answer: String,
|
|
|
|
#[serde(alias = "Автор", default, skip_serializing_if = "String::is_empty")]
|
|
pub author: String,
|
|
#[serde(
|
|
default,
|
|
alias = "Комментарий",
|
|
skip_serializing_if = "String::is_empty"
|
|
)]
|
|
pub comment: String,
|
|
#[serde(
|
|
default,
|
|
alias = "Комментарии",
|
|
alias = "Инфо",
|
|
skip_serializing_if = "String::is_empty"
|
|
)]
|
|
pub comment1: String,
|
|
#[serde(default, alias = "Тур", skip_serializing_if = "String::is_empty")]
|
|
pub tour: String,
|
|
#[serde(
|
|
default,
|
|
alias = "Ссылка",
|
|
alias = "URL",
|
|
skip_serializing_if = "String::is_empty"
|
|
)]
|
|
pub url: String,
|
|
#[serde(default, alias = "Дата", skip_serializing_if = "String::is_empty")]
|
|
pub date: String,
|
|
#[serde(default, alias = "Обработан", skip_serializing_if = "String::is_empty")]
|
|
pub processed_by: String,
|
|
#[serde(default, alias = "Редактор", skip_serializing_if = "String::is_empty")]
|
|
pub redacted_by: String,
|
|
#[serde(default, alias = "Копирайт", skip_serializing_if = "String::is_empty")]
|
|
pub copyright: String,
|
|
#[serde(default, alias = "Тема", skip_serializing_if = "String::is_empty")]
|
|
pub theme: String,
|
|
#[serde(
|
|
default,
|
|
alias = "Вид",
|
|
alias = "Тип",
|
|
skip_serializing_if = "String::is_empty"
|
|
)]
|
|
pub kind: String,
|
|
#[serde(default, alias = "Источник", skip_serializing_if = "String::is_empty")]
|
|
pub source: String,
|
|
#[serde(default, alias = "Рейтинг", skip_serializing_if = "String::is_empty")]
|
|
pub rating: String,
|
|
}
|
|
|
|
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq)]
|
|
pub struct SourceQuestionsBatch {
|
|
#[serde(default, skip_serializing_if = "String::is_empty")]
|
|
pub filename: String,
|
|
#[serde(alias = "Пакет", alias = "Чемпионат")]
|
|
pub description: String,
|
|
#[serde(default, alias = "Автор", skip_serializing_if = "String::is_empty")]
|
|
pub author: String,
|
|
#[serde(
|
|
default,
|
|
alias = "Комментарий",
|
|
alias = "Комментарии",
|
|
alias = "Инфо",
|
|
skip_serializing_if = "String::is_empty"
|
|
)]
|
|
pub comment: String,
|
|
#[serde(
|
|
default,
|
|
alias = "Ссылка",
|
|
alias = "URL",
|
|
skip_serializing_if = "String::is_empty"
|
|
)]
|
|
pub url: String,
|
|
#[serde(default, alias = "Дата", skip_serializing_if = "String::is_empty")]
|
|
pub date: String,
|
|
#[serde(default, alias = "Обработан", skip_serializing_if = "String::is_empty")]
|
|
pub processed_by: String,
|
|
#[serde(default, alias = "Редактор", skip_serializing_if = "String::is_empty")]
|
|
pub redacted_by: String,
|
|
#[serde(default, alias = "Копирайт", skip_serializing_if = "String::is_empty")]
|
|
pub copyright: String,
|
|
#[serde(default, alias = "Тема", skip_serializing_if = "String::is_empty")]
|
|
pub theme: String,
|
|
#[serde(
|
|
default,
|
|
alias = "Вид",
|
|
alias = "Тип",
|
|
skip_serializing_if = "String::is_empty"
|
|
)]
|
|
pub kind: String,
|
|
#[serde(default, alias = "Источник", skip_serializing_if = "String::is_empty")]
|
|
pub source: String,
|
|
#[serde(default, alias = "Рейтинг", skip_serializing_if = "String::is_empty")]
|
|
pub rating: String,
|
|
#[serde(alias = "Вопросы")]
|
|
pub questions: Vec<SourceQuestion>,
|
|
}
|
|
|
|
fn u32_is_zero(num: &u32) -> bool {
|
|
*num == 0
|
|
}
|
|
|
|
#[cfg(any(feature = "convert", feature = "source"))]
|
|
pub mod reader_sync {
|
|
use std::io::{Read, Seek};
|
|
use zip::ZipArchive;
|
|
|
|
use super::SourceQuestionsBatch;
|
|
|
|
pub struct SourceQuestionsZipReader<R>
|
|
where
|
|
R: Read + Seek,
|
|
{
|
|
zipfile: ZipArchive<R>,
|
|
index: Option<usize>,
|
|
}
|
|
|
|
impl<R> SourceQuestionsZipReader<R>
|
|
where
|
|
R: Read + Seek,
|
|
{
|
|
fn new(zipfile: ZipArchive<R>) -> Self {
|
|
SourceQuestionsZipReader {
|
|
zipfile,
|
|
index: None,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<R> Iterator for SourceQuestionsZipReader<R>
|
|
where
|
|
R: Read + Seek,
|
|
{
|
|
type Item = (String, Result<SourceQuestionsBatch, serde_json::Error>);
|
|
|
|
fn next(&mut self) -> Option<Self::Item> {
|
|
if self.index.is_none() && !self.zipfile.is_empty() {
|
|
self.index = Some(0);
|
|
}
|
|
|
|
match self.index {
|
|
Some(i) if i < self.zipfile.len() => {
|
|
self.index = Some(i + 1);
|
|
|
|
self.nth(i)
|
|
}
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
fn nth(&mut self, n: usize) -> Option<Self::Item> {
|
|
if self.zipfile.len() <= n {
|
|
return None;
|
|
}
|
|
self.index = Some(n + 1);
|
|
|
|
let file = self.zipfile.by_index(n).unwrap();
|
|
let name = file.mangled_name();
|
|
let name_str = name.to_str().unwrap();
|
|
|
|
let data: Result<SourceQuestionsBatch, _> = serde_json::from_reader(file);
|
|
|
|
Some((String::from(name_str), data))
|
|
}
|
|
|
|
fn size_hint(&self) -> (usize, Option<usize>) {
|
|
let len = self.zipfile.len();
|
|
let index = self.index.unwrap_or(0);
|
|
let rem = len.saturating_sub(index + 1);
|
|
(rem, Some(rem))
|
|
}
|
|
|
|
fn count(self) -> usize
|
|
where
|
|
Self: Sized,
|
|
{
|
|
self.zipfile.len()
|
|
}
|
|
}
|
|
|
|
impl<R> ExactSizeIterator for SourceQuestionsZipReader<R>
|
|
where
|
|
R: Read + Seek,
|
|
{
|
|
fn len(&self) -> usize {
|
|
self.zipfile.len()
|
|
}
|
|
}
|
|
|
|
pub trait ReadSourceQuestionsBatches<R>
|
|
where
|
|
R: Read + Seek,
|
|
{
|
|
fn source_questions(self) -> SourceQuestionsZipReader<R>;
|
|
}
|
|
|
|
impl<R> ReadSourceQuestionsBatches<R> for ZipArchive<R>
|
|
where
|
|
R: Read + Seek,
|
|
{
|
|
fn source_questions(self) -> SourceQuestionsZipReader<R> {
|
|
SourceQuestionsZipReader::new(self)
|
|
}
|
|
}
|
|
#[cfg(test)]
|
|
mod test {
|
|
use super::super::test::sample_batch;
|
|
use super::*;
|
|
use std::fs;
|
|
use std::{io::Write, iter, path::Path};
|
|
use tempfile::tempdir;
|
|
|
|
fn write_sample_zip<P>(path: P)
|
|
where
|
|
P: AsRef<Path>,
|
|
{
|
|
let batch = sample_batch();
|
|
let z_file = fs::File::create(path).expect("crerate zip file");
|
|
let mut zip_file = zip::ZipWriter::new(z_file);
|
|
let options =
|
|
zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Zstd);
|
|
zip_file
|
|
.start_file("test.json", options)
|
|
.expect("zip start file");
|
|
let data = serde_json::to_vec(&batch).unwrap();
|
|
let amount = zip_file.write(data.as_slice()).expect("write entry");
|
|
assert_eq!(amount, data.len());
|
|
zip_file.finish().expect("finish zip file");
|
|
}
|
|
|
|
#[test]
|
|
fn test_source_questions_get() {
|
|
let expected_batch = sample_batch();
|
|
let dir = tempdir().expect("tempdir");
|
|
|
|
// write sample
|
|
let tmpfile_zip = dir.path().join("test.zip");
|
|
write_sample_zip(&tmpfile_zip);
|
|
|
|
let z_file = fs::File::open(tmpfile_zip).expect("open zip file");
|
|
let zip_file = zip::ZipArchive::new(z_file).expect("open zip file reader");
|
|
|
|
let mut source = zip_file.source_questions();
|
|
assert_eq!(source.len(), 1);
|
|
|
|
let actual = source.next().expect("get batch");
|
|
assert_eq!(actual.0, "test.json");
|
|
assert_eq!(actual.1.expect("parse batch"), expected_batch);
|
|
}
|
|
|
|
#[test]
|
|
fn test_source_questions_iter() {
|
|
let expected_batch = sample_batch();
|
|
let dir = tempdir().expect("tempdir");
|
|
|
|
// write sample
|
|
let tmpfile_zip = dir.path().join("test.zip");
|
|
write_sample_zip(&tmpfile_zip);
|
|
|
|
let z_file = fs::File::open(tmpfile_zip).expect("open zip file");
|
|
let zip_file = zip::ZipArchive::new(z_file).expect("open zip file reader");
|
|
|
|
let source = zip_file.source_questions();
|
|
assert_eq!(source.len(), 1);
|
|
|
|
let expected_iter = iter::once((String::from("test.json"), Ok(expected_batch)));
|
|
|
|
assert!(source
|
|
.map(|x| (x.0, x.1.map_err(|e| e.to_string())))
|
|
.eq(expected_iter));
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(any(feature = "convert", feature = "source"))]
|
|
pub use reader_sync::{ReadSourceQuestionsBatches, SourceQuestionsZipReader};
|
|
|
|
#[cfg(any(feature = "convert_async", feature = "source_async"))]
|
|
pub mod reader_async {
|
|
use async_stream::stream;
|
|
use async_zip::tokio::read::seek::ZipFileReader;
|
|
use futures_core::stream::Stream;
|
|
use futures_util::AsyncReadExt;
|
|
|
|
use tokio::io::{AsyncRead, AsyncSeek};
|
|
|
|
use super::SourceQuestionsBatch;
|
|
|
|
pub struct SourceQuestionsZipReaderAsync<R>
|
|
where
|
|
R: AsyncRead + AsyncSeek + Unpin,
|
|
{
|
|
zipfile: ZipFileReader<R>,
|
|
index: Option<usize>,
|
|
}
|
|
|
|
impl<R> SourceQuestionsZipReaderAsync<R>
|
|
where
|
|
R: AsyncRead + AsyncSeek + Unpin,
|
|
{
|
|
pub fn new(zipfile: ZipFileReader<R>) -> Self {
|
|
SourceQuestionsZipReaderAsync {
|
|
zipfile,
|
|
index: None,
|
|
}
|
|
}
|
|
|
|
pub fn len(&self) -> usize {
|
|
self.zipfile.file().entries().len()
|
|
}
|
|
|
|
pub fn is_empty(&self) -> bool {
|
|
self.len() == 0
|
|
}
|
|
|
|
pub async fn get(
|
|
&mut self,
|
|
index: usize,
|
|
) -> Result<(String, Result<SourceQuestionsBatch, serde_json::Error>), String>
|
|
where
|
|
R: AsyncRead + AsyncSeek + Unpin,
|
|
{
|
|
let len = self.len();
|
|
if index >= len {
|
|
return Err(format!("get index={index}, when len={len}"));
|
|
}
|
|
|
|
let reader = self.zipfile.reader_with_entry(index).await;
|
|
if let Err(error) = reader {
|
|
return Err(format!("reader_with_entry: {error:?}"));
|
|
}
|
|
let mut reader = reader.unwrap();
|
|
|
|
let filename = reader.entry().filename().clone().into_string().unwrap();
|
|
let mut data: Vec<u8> = Vec::new();
|
|
let readed = reader.read_to_end(&mut data).await;
|
|
if let Err(error) = readed {
|
|
return Err(format!("read_to_end: {error:?}"));
|
|
}
|
|
let parsed: Result<SourceQuestionsBatch, _> = serde_json::from_slice(&data);
|
|
Ok((filename, parsed))
|
|
}
|
|
pub async fn get_next(
|
|
&mut self,
|
|
) -> Option<Result<(String, Result<SourceQuestionsBatch, serde_json::Error>), String>>
|
|
where
|
|
R: AsyncRead + AsyncSeek + Unpin,
|
|
{
|
|
if self.index.is_none() && !self.is_empty() {
|
|
self.index = Some(0);
|
|
}
|
|
|
|
if self.index.unwrap() >= self.len() {
|
|
return None;
|
|
}
|
|
|
|
let item = self.get(self.index.unwrap()).await;
|
|
self.index = Some(self.index.unwrap() + 1);
|
|
|
|
Some(item)
|
|
}
|
|
pub fn stream(
|
|
&mut self,
|
|
) -> impl Stream<Item = (String, Result<SourceQuestionsBatch, serde_json::Error>)> + '_
|
|
{
|
|
stream! {
|
|
while let Some(Ok(item)) = self.get_next().await {
|
|
yield item
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
pub trait ReadSourceQuestionsBatchesAsync<R>
|
|
where
|
|
R: AsyncRead + AsyncSeek + Unpin,
|
|
{
|
|
fn source_questions(self) -> SourceQuestionsZipReaderAsync<R>;
|
|
}
|
|
|
|
impl<R> ReadSourceQuestionsBatchesAsync<R> for ZipFileReader<R>
|
|
where
|
|
R: AsyncRead + AsyncSeek + Unpin,
|
|
{
|
|
fn source_questions(self) -> SourceQuestionsZipReaderAsync<R> {
|
|
SourceQuestionsZipReaderAsync::new(self)
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod test {
|
|
use crate::source::SourceQuestion;
|
|
|
|
use super::super::test::sample_batch;
|
|
use super::*;
|
|
use async_zip::{base::write::ZipFileWriter, ZipEntryBuilder};
|
|
use core::fmt::Debug;
|
|
use futures_util::StreamExt;
|
|
use std::path::Path;
|
|
use tempfile::tempdir;
|
|
use tokio::fs;
|
|
|
|
async fn write_sample_zip<P>(path: P)
|
|
where
|
|
P: AsRef<Path>,
|
|
{
|
|
let batch = sample_batch();
|
|
let z_file = fs::File::create(path).await.expect("crerate zip file");
|
|
let mut zip_file = ZipFileWriter::with_tokio(z_file);
|
|
let entry =
|
|
ZipEntryBuilder::new("test.json".into(), async_zip::Compression::Zstd).build();
|
|
zip_file
|
|
.write_entry_whole(entry, serde_json::to_vec(&batch).unwrap().as_slice())
|
|
.await
|
|
.expect("write entry");
|
|
zip_file.close().await.expect("close zip");
|
|
}
|
|
|
|
async fn assert_data_rref_eq<T>((x, y): (T, &T))
|
|
where
|
|
T: PartialEq + Debug,
|
|
{
|
|
assert_eq!(x, *y);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_source_questions_stream() {
|
|
let expected_batch = sample_batch();
|
|
let dir = tempdir().expect("tempdir");
|
|
|
|
// write sample
|
|
let tmpfile_zip = dir.path().join("test.zip");
|
|
write_sample_zip(&tmpfile_zip).await;
|
|
|
|
let mut z_file = fs::File::open(tmpfile_zip).await.expect("open zip file");
|
|
let zip_file = ZipFileReader::with_tokio(&mut z_file)
|
|
.await
|
|
.expect("open zip file reader");
|
|
|
|
let expected_count = expected_batch.questions.len();
|
|
let expected_stream = futures::stream::iter(expected_batch.questions.iter());
|
|
let mut actual_source = zip_file.source_questions();
|
|
let actual_stream = actual_source.stream();
|
|
let mut actual_count: usize = 0;
|
|
|
|
actual_stream
|
|
.flat_map(|x| futures::stream::iter(x.1.expect("parse batch").questions))
|
|
.zip(expected_stream)
|
|
.map(|x| {
|
|
actual_count += 1;
|
|
x
|
|
})
|
|
.for_each(assert_data_rref_eq::<SourceQuestion>)
|
|
.await;
|
|
assert_eq!(actual_count, expected_count);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_source_questions_get() {
|
|
let expected_batch = sample_batch();
|
|
let dir = tempdir().expect("tempdir");
|
|
|
|
// write sample
|
|
let tmpfile_zip = dir.path().join("test.zip");
|
|
write_sample_zip(&tmpfile_zip).await;
|
|
|
|
let mut z_file = fs::File::open(tmpfile_zip).await.expect("open zip file");
|
|
let zip_file = ZipFileReader::with_tokio(&mut z_file)
|
|
.await
|
|
.expect("open zip file reader");
|
|
|
|
let mut source = zip_file.source_questions();
|
|
assert_eq!(source.len(), 1);
|
|
|
|
let actual = source.get(0).await.expect("get batch");
|
|
assert_eq!(actual.0, "test.json");
|
|
assert_eq!(actual.1.expect("parse batch"), expected_batch);
|
|
}
|
|
}
|
|
}
|
|
#[cfg(any(feature = "convert_async", feature = "source_async"))]
|
|
pub use reader_async::{ReadSourceQuestionsBatchesAsync, SourceQuestionsZipReaderAsync};
|
|
|
|
#[cfg(test)]
|
|
mod test {
|
|
use super::*;
|
|
use insta::assert_yaml_snapshot;
|
|
use serde_json::json;
|
|
|
|
pub fn sample_batch() -> SourceQuestionsBatch {
|
|
SourceQuestionsBatch {
|
|
description: "Тестовый".into(),
|
|
date: "00-000-2000".into(),
|
|
questions: vec![
|
|
SourceQuestion {
|
|
id: "Вопрос 1".into(),
|
|
description: "Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2".into(),
|
|
answer: "42".into(),
|
|
..Default::default()
|
|
},
|
|
SourceQuestion {
|
|
id: "Вопрос 2".into(),
|
|
description: "Зимой и летом одним цветом".into(),
|
|
answer: "ёлка".into(),
|
|
..Default::default()
|
|
},
|
|
],
|
|
..Default::default()
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_batch_ser() {
|
|
let batch = sample_batch();
|
|
|
|
assert_yaml_snapshot!(batch, @r#"
|
|
---
|
|
description: Тестовый
|
|
date: 00-000-2000
|
|
questions:
|
|
- id: Вопрос 1
|
|
description: Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2
|
|
answer: "42"
|
|
- id: Вопрос 2
|
|
description: Зимой и летом одним цветом
|
|
answer: ёлка
|
|
|
|
"#);
|
|
}
|
|
#[test]
|
|
fn test_batch_de() {
|
|
let batch_from_json: Result<SourceQuestionsBatch, _> = serde_json::from_value(json!({
|
|
"Чемпионат": "Тестовый",
|
|
"Дата": "00-000-2000",
|
|
"Вопросы": [
|
|
{
|
|
"id": "Вопрос 1",
|
|
"Вопрос": "Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2",
|
|
"Ответ": "42",
|
|
},
|
|
{
|
|
"id": "Вопрос 2",
|
|
"Вопрос": "Зимой и летом одним цветом",
|
|
"Ответ": "ёлка",
|
|
},
|
|
]
|
|
}));
|
|
assert!(batch_from_json.is_ok());
|
|
|
|
assert_yaml_snapshot!(batch_from_json.unwrap(), @r#"
|
|
---
|
|
description: Тестовый
|
|
date: 00-000-2000
|
|
questions:
|
|
- id: Вопрос 1
|
|
description: Сколько будет (2 * 2 * 2 + 2) * 2 * 2 + 2
|
|
answer: "42"
|
|
- id: Вопрос 2
|
|
description: Зимой и летом одним цветом
|
|
answer: ёлка
|
|
|
|
"#);
|
|
}
|
|
}
|