Compare commits
No commits in common. "851058245d1ea71ebc72b560a6b6fb3d3a46baf3" and "c0193e4f74d0b2476fb1aa1d04ddbf23f5930c0b" have entirely different histories.
851058245d
...
c0193e4f74
@ -7,11 +7,8 @@ steps:
|
|||||||
image: rust:1-alpine
|
image: rust:1-alpine
|
||||||
commands:
|
commands:
|
||||||
- apk add --no-cache musl-dev
|
- apk add --no-cache musl-dev
|
||||||
- cargo build -p chgk_ledb
|
- cargo build --verbose --all-features --all
|
||||||
- cargo build -p chgk_ledb_lib --features sync,convert,source
|
- cargo test --verbose --all-features --all
|
||||||
- cargo build -p chgk_ledb_lib --features async,convert_async,source_async
|
|
||||||
- cargo test -p chgk_ledb_lib --features sync,convert,source
|
|
||||||
- cargo test -p chgk_ledb_lib --features async,convert_async,source_async
|
|
||||||
environment:
|
environment:
|
||||||
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
|
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
|
||||||
|
|
||||||
|
85
Cargo.lock
generated
85
Cargo.lock
generated
@ -92,20 +92,6 @@ dependencies = [
|
|||||||
"windows-sys 0.48.0",
|
"windows-sys 0.48.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "async-compression"
|
|
||||||
version = "0.4.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "62b74f44609f0f91493e3082d3734d98497e094777144380ea4db9f9905dd5b6"
|
|
||||||
dependencies = [
|
|
||||||
"futures-core",
|
|
||||||
"futures-io",
|
|
||||||
"memchr",
|
|
||||||
"pin-project-lite",
|
|
||||||
"zstd 0.12.4",
|
|
||||||
"zstd-safe 6.0.6",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "async-compression"
|
name = "async-compression"
|
||||||
version = "0.4.1"
|
version = "0.4.1"
|
||||||
@ -152,20 +138,6 @@ dependencies = [
|
|||||||
"syn 2.0.26",
|
"syn 2.0.26",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "async_zip"
|
|
||||||
version = "0.0.15"
|
|
||||||
source = "git+https://github.com/Majored/rs-async-zip?rev=ff0d985#ff0d985ef54cf00d73c497dbca0beea7541e37dc"
|
|
||||||
dependencies = [
|
|
||||||
"async-compression 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
|
||||||
"crc32fast",
|
|
||||||
"futures-util",
|
|
||||||
"pin-project",
|
|
||||||
"thiserror",
|
|
||||||
"tokio",
|
|
||||||
"tokio-util",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "atty"
|
name = "atty"
|
||||||
version = "0.2.14"
|
version = "0.2.14"
|
||||||
@ -324,9 +296,8 @@ dependencies = [
|
|||||||
name = "chgk_ledb_lib"
|
name = "chgk_ledb_lib"
|
||||||
version = "1.1.0"
|
version = "1.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-compression 0.4.1 (git+https://github.com/Nullus157/async-compression?rev=4fd4c42)",
|
"async-compression",
|
||||||
"async-stream",
|
"async-stream",
|
||||||
"async_zip",
|
|
||||||
"bincode",
|
"bincode",
|
||||||
"fmmap",
|
"fmmap",
|
||||||
"futures",
|
"futures",
|
||||||
@ -1090,26 +1061,6 @@ dependencies = [
|
|||||||
"sha2",
|
"sha2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pin-project"
|
|
||||||
version = "1.1.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
|
|
||||||
dependencies = [
|
|
||||||
"pin-project-internal",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pin-project-internal"
|
|
||||||
version = "1.1.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"syn 2.0.26",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pin-project-lite"
|
name = "pin-project-lite"
|
||||||
version = "0.2.10"
|
version = "0.2.10"
|
||||||
@ -1483,26 +1434,6 @@ version = "0.16.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
|
checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "thiserror"
|
|
||||||
version = "1.0.44"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90"
|
|
||||||
dependencies = [
|
|
||||||
"thiserror-impl",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "thiserror-impl"
|
|
||||||
version = "1.0.44"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"syn 2.0.26",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "time"
|
name = "time"
|
||||||
version = "0.3.23"
|
version = "0.3.23"
|
||||||
@ -1553,20 +1484,6 @@ dependencies = [
|
|||||||
"syn 2.0.26",
|
"syn 2.0.26",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tokio-util"
|
|
||||||
version = "0.7.8"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
|
|
||||||
dependencies = [
|
|
||||||
"bytes",
|
|
||||||
"futures-core",
|
|
||||||
"futures-io",
|
|
||||||
"futures-sink",
|
|
||||||
"pin-project-lite",
|
|
||||||
"tokio",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "typenum"
|
name = "typenum"
|
||||||
version = "1.16.0"
|
version = "1.16.0"
|
||||||
|
@ -14,7 +14,7 @@ name = "db_bench"
|
|||||||
harness = false
|
harness = false
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
chgk_ledb_lib = {path = "../lib", features = ["sync", "source", "convert"]}
|
chgk_ledb_lib = {path = "../lib", features = ["sync", "source"]}
|
||||||
serde_json="1.0"
|
serde_json="1.0"
|
||||||
zip="0.6"
|
zip="0.6"
|
||||||
rand="0.8"
|
rand="0.8"
|
||||||
|
@ -10,28 +10,10 @@ description = "Библиотека для доступа к файлу базы
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = []
|
default = ["async"]
|
||||||
sync = ["zstd", "memmap"]
|
sync = ["zstd", "memmap"]
|
||||||
async = [
|
async = ["futures", "futures-core", "futures-util", "fmmap", "tokio", "async-compression", "async-stream"]
|
||||||
"futures",
|
|
||||||
"futures-core",
|
|
||||||
"futures-util",
|
|
||||||
"fmmap",
|
|
||||||
"tokio",
|
|
||||||
"async-compression",
|
|
||||||
"async-stream",
|
|
||||||
]
|
|
||||||
source = ["zip"]
|
source = ["zip"]
|
||||||
source_async = [
|
|
||||||
"async_zip",
|
|
||||||
"tokio",
|
|
||||||
"futures",
|
|
||||||
"futures-core",
|
|
||||||
"futures-util",
|
|
||||||
"async-stream",
|
|
||||||
]
|
|
||||||
convert = []
|
|
||||||
convert_async = ["futures", "futures-core", "futures-util", "async-stream"]
|
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
serde="1.0"
|
serde="1.0"
|
||||||
@ -39,25 +21,12 @@ serde_derive = "1.0"
|
|||||||
serde_json="1.0"
|
serde_json="1.0"
|
||||||
bincode = "^2.0.0-rc.2"
|
bincode = "^2.0.0-rc.2"
|
||||||
zip= {version = "0.6", optional = true}
|
zip= {version = "0.6", optional = true}
|
||||||
async_zip = { git = "https://github.com/Majored/rs-async-zip", rev = "ff0d985", features = [
|
|
||||||
"zstd",
|
|
||||||
"tokio",
|
|
||||||
"tokio-fs",
|
|
||||||
], optional = true }
|
|
||||||
fmmap = { version = "0.3", features = ["tokio-async"] , optional = true}
|
fmmap = { version = "0.3", features = ["tokio-async"] , optional = true}
|
||||||
tokio = { version = "1", features = [
|
tokio = { version = "1", features = ["fs","io-util","rt", "macros"] , optional = true}
|
||||||
"fs",
|
|
||||||
"io-util",
|
|
||||||
"rt",
|
|
||||||
"macros",
|
|
||||||
], optional = true }
|
|
||||||
futures-core = {version = "0.3", optional = true}
|
futures-core = {version = "0.3", optional = true}
|
||||||
futures = {version = "0.3", optional = true}
|
futures = {version = "0.3", optional = true}
|
||||||
futures-util = {version = "0.3", optional = true}
|
futures-util = {version = "0.3", optional = true}
|
||||||
async-compression = { git = "https://github.com/Nullus157/async-compression", rev = "4fd4c42", default-features = false, features = [
|
async-compression = {git="https://github.com/Nullus157/async-compression", rev="4fd4c42", default-features = false, features = ["zstdmt", "tokio"], optional = true}
|
||||||
"zstd",
|
|
||||||
"tokio",
|
|
||||||
], optional = true }
|
|
||||||
async-stream = {version = "0.3", optional = true}
|
async-stream = {version = "0.3", optional = true}
|
||||||
zstd = {version = "^0.12", default-features = false, optional = true}
|
zstd = {version = "^0.12", default-features = false, optional = true}
|
||||||
memmap = {version = "0.7.0", optional = true}
|
memmap = {version = "0.7.0", optional = true}
|
||||||
|
@ -22,7 +22,20 @@ type LSize = u32;
|
|||||||
const LEN_SIZE: usize = std::mem::size_of::<LSize>();
|
const LEN_SIZE: usize = std::mem::size_of::<LSize>();
|
||||||
const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard();
|
const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard();
|
||||||
|
|
||||||
use crate::util::ErrorToString;
|
trait ErrorToString {
|
||||||
|
type Output;
|
||||||
|
fn str_err(self) -> std::result::Result<Self::Output, String>;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T, E> ErrorToString for std::result::Result<T, E>
|
||||||
|
where
|
||||||
|
E: std::error::Error,
|
||||||
|
{
|
||||||
|
type Output = T;
|
||||||
|
fn str_err(self) -> std::result::Result<Self::Output, String> {
|
||||||
|
self.map_err(|e| e.to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub struct WriterOpts {
|
pub struct WriterOpts {
|
||||||
pub compress_lvl: Level,
|
pub compress_lvl: Level,
|
||||||
|
@ -12,7 +12,20 @@ type LSize = u32;
|
|||||||
const LEN_SIZE: usize = std::mem::size_of::<LSize>();
|
const LEN_SIZE: usize = std::mem::size_of::<LSize>();
|
||||||
const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard();
|
const BINCODE_CFG: bincode::config::Configuration = bincode::config::standard();
|
||||||
|
|
||||||
use crate::util::ErrorToString;
|
trait ErrorToString {
|
||||||
|
type Output;
|
||||||
|
fn str_err(self) -> std::result::Result<Self::Output, String>;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T, E> ErrorToString for std::result::Result<T, E>
|
||||||
|
where
|
||||||
|
E: std::error::Error,
|
||||||
|
{
|
||||||
|
type Output = T;
|
||||||
|
fn str_err(self) -> std::result::Result<Self::Output, String> {
|
||||||
|
self.map_err(|e| e.to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub struct WriterOpts {
|
pub struct WriterOpts {
|
||||||
pub compress_lvl: i32,
|
pub compress_lvl: i32,
|
||||||
|
@ -1,13 +1,7 @@
|
|||||||
#[cfg(all(feature = "convert", feature = "convert_async"))]
|
|
||||||
compile_error!(
|
|
||||||
"feature \"convert\" and feature \"convert_async\" cannot be enabled at the same time"
|
|
||||||
);
|
|
||||||
|
|
||||||
#[cfg(feature = "async")]
|
#[cfg(feature = "async")]
|
||||||
pub mod async_db;
|
pub mod async_db;
|
||||||
#[cfg(feature = "sync")]
|
#[cfg(feature = "sync")]
|
||||||
pub mod db;
|
pub mod db;
|
||||||
pub mod questions;
|
pub mod questions;
|
||||||
#[cfg(any(feature = "source", feature = "source_async"))]
|
#[cfg(feature = "source")]
|
||||||
pub mod source;
|
pub mod source;
|
||||||
pub mod util;
|
|
||||||
|
@ -69,7 +69,7 @@ pub struct Question {
|
|||||||
pub batch_info: BatchInfo,
|
pub batch_info: BatchInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "convert")]
|
#[cfg(feature = "source")]
|
||||||
pub mod convert {
|
pub mod convert {
|
||||||
use super::{BatchInfo, Question};
|
use super::{BatchInfo, Question};
|
||||||
use crate::source::{SourceQuestion, SourceQuestionsBatch};
|
use crate::source::{SourceQuestion, SourceQuestionsBatch};
|
||||||
@ -138,123 +138,6 @@ pub mod convert {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#[cfg(feature = "convert")]
|
|
||||||
|
#[cfg(feature = "source")]
|
||||||
pub use convert::QuestionsConverter;
|
pub use convert::QuestionsConverter;
|
||||||
|
|
||||||
#[cfg(feature = "convert_async")]
|
|
||||||
pub mod convert_async {
|
|
||||||
use async_stream::stream;
|
|
||||||
use futures_core::stream::Stream;
|
|
||||||
use futures_core::Future;
|
|
||||||
use futures_util::{pin_mut, StreamExt};
|
|
||||||
|
|
||||||
use std::pin::Pin;
|
|
||||||
use std::task::{Context, Poll};
|
|
||||||
|
|
||||||
use super::{BatchInfo, Question};
|
|
||||||
use crate::source::{SourceQuestion, SourceQuestionsBatch};
|
|
||||||
|
|
||||||
macro_rules! make {
|
|
||||||
($Target:ident; by {$($field:ident),+}; from $src:expr) => {$Target {$(
|
|
||||||
$field: $src.$field
|
|
||||||
),+}};
|
|
||||||
($Target:ident; with defaults and by {$($field:ident),+}; from $src:expr) => {$Target {$(
|
|
||||||
$field: $src.$field
|
|
||||||
),+ ,..$Target::default()}}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<SourceQuestion> for Question {
|
|
||||||
fn from(src: SourceQuestion) -> Self {
|
|
||||||
make! {Self; with defaults and by {
|
|
||||||
num, id, description, answer, author, comment, comment1, tour, url,
|
|
||||||
date, processed_by, redacted_by, copyright, theme, kind, source, rating
|
|
||||||
}; from src}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<SourceQuestionsBatch> for BatchInfo {
|
|
||||||
fn from(src: SourceQuestionsBatch) -> Self {
|
|
||||||
make! {Self; by {
|
|
||||||
filename, description, author, comment, url, date,
|
|
||||||
processed_by, redacted_by, copyright, theme, kind, source, rating
|
|
||||||
}; from src}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<SourceQuestionsBatch> for Vec<Question> {
|
|
||||||
fn from(src: SourceQuestionsBatch) -> Self {
|
|
||||||
let mut result: Vec<Question> = src
|
|
||||||
.questions
|
|
||||||
.iter()
|
|
||||||
.map(|item| item.clone().into())
|
|
||||||
.collect();
|
|
||||||
let batch_info = BatchInfo::from(src);
|
|
||||||
result.iter_mut().for_each(|question| {
|
|
||||||
question.batch_info = batch_info.clone();
|
|
||||||
});
|
|
||||||
|
|
||||||
result
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct QuestionsConverterAsync<T>
|
|
||||||
where
|
|
||||||
T: Stream<Item = (String, Result<SourceQuestionsBatch, serde_json::Error>)>
|
|
||||||
+ std::marker::Unpin,
|
|
||||||
{
|
|
||||||
inner: T,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T> From<T> for QuestionsConverterAsync<T>
|
|
||||||
where
|
|
||||||
T: Stream<Item = (String, Result<SourceQuestionsBatch, serde_json::Error>)>
|
|
||||||
+ std::marker::Unpin,
|
|
||||||
{
|
|
||||||
fn from(inner: T) -> Self {
|
|
||||||
Self { inner }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T> QuestionsConverterAsync<T>
|
|
||||||
where
|
|
||||||
T: Stream<Item = (String, Result<SourceQuestionsBatch, serde_json::Error>)>
|
|
||||||
+ std::marker::Unpin,
|
|
||||||
{
|
|
||||||
fn convert(&mut self) -> impl Stream<Item = Question> + '_ {
|
|
||||||
stream! {
|
|
||||||
while let Some((filename, Ok(batch))) = self.inner.next().await {
|
|
||||||
let mut batch = batch;
|
|
||||||
batch.filename = filename;
|
|
||||||
let questions: Vec<Question> = batch.into();
|
|
||||||
for question in questions {
|
|
||||||
yield question
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T> Stream for QuestionsConverterAsync<T>
|
|
||||||
where
|
|
||||||
T: Stream<Item = (String, Result<SourceQuestionsBatch, serde_json::Error>)>
|
|
||||||
+ std::marker::Unpin,
|
|
||||||
{
|
|
||||||
type Item = Question;
|
|
||||||
|
|
||||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
|
||||||
let convert = self.convert();
|
|
||||||
pin_mut!(convert);
|
|
||||||
match Pin::new(&mut convert.next()).poll(cx) {
|
|
||||||
Poll::Ready(Some(item)) => Poll::Ready(Some(item)),
|
|
||||||
Poll::Ready(None) => Poll::Ready(None),
|
|
||||||
Poll::Pending => Poll::Pending,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
|
||||||
self.inner.size_hint()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#[cfg(feature = "convert_async")]
|
|
||||||
pub use convert_async::QuestionsConverterAsync;
|
|
||||||
|
@ -1,4 +1,6 @@
|
|||||||
use serde_derive::{Deserialize, Serialize};
|
use serde_derive::{Deserialize, Serialize};
|
||||||
|
use std::io::{Read, Seek};
|
||||||
|
use zip::ZipArchive;
|
||||||
|
|
||||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||||
pub struct SourceQuestion {
|
pub struct SourceQuestion {
|
||||||
@ -103,13 +105,6 @@ pub struct SourceQuestionsBatch {
|
|||||||
pub questions: Vec<SourceQuestion>,
|
pub questions: Vec<SourceQuestion>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "source")]
|
|
||||||
pub mod reader_sync {
|
|
||||||
use std::io::{Read, Seek};
|
|
||||||
use zip::ZipArchive;
|
|
||||||
|
|
||||||
use super::SourceQuestionsBatch;
|
|
||||||
|
|
||||||
pub struct SourceQuestionsZipReader<R>
|
pub struct SourceQuestionsZipReader<R>
|
||||||
where
|
where
|
||||||
R: Read + Seek,
|
R: Read + Seek,
|
||||||
@ -209,123 +204,3 @@ pub mod reader_sync {
|
|||||||
SourceQuestionsZipReader::new(self)
|
SourceQuestionsZipReader::new(self)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "source")]
|
|
||||||
pub use reader_sync::{ReadSourceQuestionsBatches, SourceQuestionsZipReader};
|
|
||||||
|
|
||||||
#[cfg(feature = "source_async")]
|
|
||||||
pub mod reader_async {
|
|
||||||
use crate::util::ErrorToString;
|
|
||||||
|
|
||||||
use async_zip::tokio::read::seek::ZipFileReader;
|
|
||||||
use futures_core::stream::Stream;
|
|
||||||
use futures_core::Future;
|
|
||||||
use futures_util::{pin_mut, AsyncReadExt};
|
|
||||||
|
|
||||||
use std::pin::Pin;
|
|
||||||
use std::task::{Context, Poll};
|
|
||||||
|
|
||||||
use tokio::io::{AsyncRead, AsyncSeek};
|
|
||||||
|
|
||||||
use super::SourceQuestionsBatch;
|
|
||||||
|
|
||||||
pub struct SourceQuestionsZipReaderAsync<R>
|
|
||||||
where
|
|
||||||
R: AsyncRead + AsyncSeek + Unpin,
|
|
||||||
{
|
|
||||||
zipfile: ZipFileReader<R>,
|
|
||||||
index: Option<usize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<R> SourceQuestionsZipReaderAsync<R>
|
|
||||||
where
|
|
||||||
R: AsyncRead + AsyncSeek + Unpin,
|
|
||||||
{
|
|
||||||
fn new(zipfile: ZipFileReader<R>) -> Self {
|
|
||||||
SourceQuestionsZipReaderAsync {
|
|
||||||
zipfile,
|
|
||||||
index: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
async fn parse_zip_entry(
|
|
||||||
&mut self,
|
|
||||||
) -> Result<(String, Result<SourceQuestionsBatch, serde_json::Error>), String>
|
|
||||||
where
|
|
||||||
R: AsyncRead + AsyncSeek + Unpin,
|
|
||||||
{
|
|
||||||
let mut reader = self
|
|
||||||
.zipfile
|
|
||||||
.reader_with_entry(self.index.unwrap())
|
|
||||||
.await
|
|
||||||
.str_err()?;
|
|
||||||
let filename = reader.entry().filename().clone().into_string().str_err()?;
|
|
||||||
let mut data: Vec<u8> = Vec::new();
|
|
||||||
reader.read_to_end(&mut data).await.str_err()?;
|
|
||||||
let parsed: Result<SourceQuestionsBatch, _> = serde_json::from_slice(&data);
|
|
||||||
self.index = Some(self.index.unwrap() + 1);
|
|
||||||
|
|
||||||
Ok((filename, parsed))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<R> Stream for SourceQuestionsZipReaderAsync<R>
|
|
||||||
where
|
|
||||||
R: AsyncRead + AsyncSeek + Unpin,
|
|
||||||
{
|
|
||||||
type Item = (String, Result<SourceQuestionsBatch, serde_json::Error>);
|
|
||||||
|
|
||||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
|
||||||
let len = self.zipfile.file().entries().len();
|
|
||||||
if self.index.is_none() && len > 0 {
|
|
||||||
self.index = Some(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
let index = &mut self.index;
|
|
||||||
if index.unwrap() == len {
|
|
||||||
return Poll::Ready(None);
|
|
||||||
}
|
|
||||||
|
|
||||||
let future = self.parse_zip_entry();
|
|
||||||
pin_mut!(future);
|
|
||||||
match Pin::new(&mut future).poll(cx) {
|
|
||||||
Poll::Ready(Ok(item)) => Poll::Ready(Some(item)),
|
|
||||||
Poll::Ready(Err(_)) => Poll::Ready(None),
|
|
||||||
Poll::Pending => Poll::Pending,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
|
||||||
let len = self.zipfile.file().entries().len();
|
|
||||||
if self.index.is_none() {
|
|
||||||
return (len, Some(len));
|
|
||||||
}
|
|
||||||
|
|
||||||
let index = self.index.unwrap();
|
|
||||||
let rem = if len > index + 1 {
|
|
||||||
len - (index + 1)
|
|
||||||
} else {
|
|
||||||
0
|
|
||||||
};
|
|
||||||
(rem, Some(rem))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub trait ReadSourceQuestionsBatchesAsync<R>
|
|
||||||
where
|
|
||||||
R: AsyncRead + AsyncSeek + Unpin,
|
|
||||||
{
|
|
||||||
fn source_questions(self) -> SourceQuestionsZipReaderAsync<R>;
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<R> ReadSourceQuestionsBatchesAsync<R> for ZipFileReader<R>
|
|
||||||
where
|
|
||||||
R: AsyncRead + AsyncSeek + Unpin,
|
|
||||||
{
|
|
||||||
fn source_questions(self) -> SourceQuestionsZipReaderAsync<R> {
|
|
||||||
SourceQuestionsZipReaderAsync::new(self)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#[cfg(feature = "source_async")]
|
|
||||||
pub use reader_async::{ReadSourceQuestionsBatchesAsync, SourceQuestionsZipReaderAsync};
|
|
||||||
|
@ -1,14 +0,0 @@
|
|||||||
pub trait ErrorToString {
|
|
||||||
type Output;
|
|
||||||
fn str_err(self) -> std::result::Result<Self::Output, String>;
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T, E> ErrorToString for std::result::Result<T, E>
|
|
||||||
where
|
|
||||||
E: std::error::Error,
|
|
||||||
{
|
|
||||||
type Output = T;
|
|
||||||
fn str_err(self) -> std::result::Result<Self::Output, String> {
|
|
||||||
self.map_err(|e| e.to_string())
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
Reference in New Issue
Block a user