Compare commits

..

No commits in common. "8120a996a360fd739a10f7f7bb6c0fb5a08e378f" and "249ac3a4ef31b5e83276d2eb61ec07aab87a0e5b" have entirely different histories.

3 changed files with 12 additions and 172 deletions

View File

@ -79,8 +79,9 @@ fn db_read(c: &mut Criterion) {
let reader: Reader<TestData> = Reader::new(&tmpfile, 2048).expect("new reader"); let reader: Reader<TestData> = Reader::new(&tmpfile, 2048).expect("new reader");
reader reader
}, },
|reader| { |mut reader| {
for item in reader { let mut reader_iter = reader.iter();
while let Some(item) = reader_iter.next() {
drop(item); drop(item);
} }
}, },

View File

@ -232,7 +232,7 @@ fn main() {
} }
fn read_from_db2(id: u32) -> Option<Question> { fn read_from_db2(id: u32) -> Option<Question> {
let reader: db::Reader<Question> = let mut reader: db::Reader<Question> =
db::Reader::new(NEW_DB_FILENAME, 2048).expect("new db reader"); db::Reader::new(NEW_DB_FILENAME, 2048).expect("new db reader");
let mut questions = reader.iter(); let mut questions = reader.iter();

View File

@ -3,7 +3,6 @@ use std::{
io::{self, Cursor, Read, Write}, io::{self, Cursor, Read, Write},
marker::PhantomData, marker::PhantomData,
path::Path, path::Path,
sync::Arc
}; };
use memmap::{Mmap, MmapOptions}; use memmap::{Mmap, MmapOptions};
@ -158,7 +157,7 @@ where
mmap: Mmap, mmap: Mmap,
count: usize, count: usize,
first_pos: LSize, first_pos: LSize,
_t: Option<Arc<T>> // PhantomData replacement _t: PhantomData<*const T>,
} }
impl<T> Reader<T> impl<T> Reader<T>
@ -179,7 +178,7 @@ where
mmap, mmap,
count, count,
first_pos, first_pos,
_t: None _t: PhantomData,
}) })
} }
@ -191,7 +190,7 @@ where
0 == self.len() 0 == self.len()
} }
pub fn get(&self, index: usize) -> Result<T, String> { pub fn get(&mut self, index: usize) -> Result<T, String> {
if index >= self.len() { if index >= self.len() {
return Err("index out of range".into()); return Err("index out of range".into());
} }
@ -223,20 +222,16 @@ where
Ok(item.0) Ok(item.0)
} }
pub fn iter(&self) -> ReaderIter<'_, T> { pub fn iter(&mut self) -> ReaderIter<'_, T> {
ReaderIter::new(self) ReaderIter::new(self)
} }
pub fn into_iter(self) -> ReaderIntoIter<T> {
ReaderIntoIter::new(self)
}
} }
pub struct ReaderIter<'a, T> pub struct ReaderIter<'a, T>
where where
T: bincode::Decode, T: bincode::Decode,
{ {
reader: &'a Reader<T>, reader: &'a mut Reader<T>,
index: Option<usize>, index: Option<usize>,
} }
@ -244,7 +239,7 @@ impl<'a, T> ReaderIter<'a, T>
where where
T: bincode::Decode, T: bincode::Decode,
{ {
fn new(reader: &'a Reader<T>) -> Self { fn new(reader: &'a mut Reader<T>) -> Self {
ReaderIter { ReaderIter {
reader, reader,
index: None, index: None,
@ -314,100 +309,6 @@ where
} }
} }
pub struct ReaderIntoIter<T>
where
T: bincode::Decode,
{
reader: Reader<T>,
index: Option<usize>,
}
impl<T> ReaderIntoIter<T>
where
T: bincode::Decode,
{
fn new(reader: Reader<T>) -> Self {
Self {
reader,
index: None,
}
}
}
impl<T> Iterator for ReaderIntoIter<T>
where
T: bincode::Decode,
{
type Item = T;
fn next(&mut self) -> Option<Self::Item> {
if self.index.is_none() && !self.reader.is_empty() {
self.index = Some(0);
}
match self.index {
Some(i) if i < self.reader.len() => self.nth(i),
_ => None,
}
}
fn nth(&mut self, n: usize) -> Option<Self::Item> {
if self.reader.len() <= n {
return None;
}
self.index = Some(n + 1);
let item = self.reader.get(n);
match item {
Ok(item) => Some(item),
Err(_) => None,
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
let len = self.reader.len();
if self.index.is_none() {
return (len, Some(len));
}
let index = self.index.unwrap();
let rem = if len > index + 1 {
len - (index + 1)
} else {
0
};
(rem, Some(rem))
}
fn count(self) -> usize
where
Self: Sized,
{
self.reader.len()
}
}
impl<T> ExactSizeIterator for ReaderIntoIter<T>
where
T: bincode::Decode,
{
fn len(&self) -> usize {
self.reader.len()
}
}
impl<T> IntoIterator for Reader<T>
where
T: bincode::Decode,
{
type Item = T;
type IntoIter = ReaderIntoIter<Self::Item>;
fn into_iter(self) -> Self::IntoIter {
self.into_iter()
}
}
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*; use super::*;
@ -444,7 +345,7 @@ mod test {
writer.load(&mut items.clone().into_iter()).expect("load"); writer.load(&mut items.clone().into_iter()).expect("load");
writer.finish().expect("finish write"); writer.finish().expect("finish write");
let reader: Reader<TestData> = Reader::new(&tmpfile, 2048).expect("new reader"); let mut reader: Reader<TestData> = Reader::new(&tmpfile, 2048).expect("new reader");
assert_eq!(items.len(), reader.len()); assert_eq!(items.len(), reader.len());
for (idx, item) in items.iter().enumerate() { for (idx, item) in items.iter().enumerate() {
@ -471,73 +372,11 @@ mod test {
writer.load(&mut items.clone().into_iter()).expect("load"); writer.load(&mut items.clone().into_iter()).expect("load");
writer.finish().expect("finish write"); writer.finish().expect("finish write");
let reader: Reader<TestData> = Reader::new(&tmpfile, 2048).expect("new reader"); let mut reader: Reader<TestData> = Reader::new(&tmpfile, 2048).expect("new reader");
assert_eq!(items.len(), reader.len()); assert_eq!(items.len(), reader.len());
items.into_iter().zip(reader.iter()).for_each(|pair| { items.into_iter().zip(reader.iter()).for_each(|pair| {
assert_eq!(pair.0, pair.1); assert_eq!(pair.0, pair.1);
}); });
} }
#[test]
fn test_write_read_into_iter() {
let dir = tempdir().expect("tempdir");
let tmpfile = dir.path().join("test.tmp");
let opts = WriterOpts {
compress_lvl: 1,
data_buf_size: 10 * 1024 * 1024,
out_buf_size: 10 * 1024 * 1024,
current_buf_size: 4096,
};
let mut writer: Writer<TestData> = Writer::new(&tmpfile, opts).expect("new writer");
let items_iter = gen_data(10);
let items: Vec<TestData> = items_iter.collect();
writer.load(&mut items.clone().into_iter()).expect("load");
writer.finish().expect("finish write");
let reader: Reader<TestData> = Reader::new(&tmpfile, 2048).expect("new reader");
assert_eq!(items.len(), reader.len());
items.into_iter().zip(reader).for_each(|pair| {
assert_eq!(pair.0, pair.1);
});
}
/// sharing Reader instance between threads
#[test]
fn test_share_reader() {
use std::thread;
let dir = tempdir().expect("tempdir");
let tmpfile = dir.path().join("test.tmp");
let opts = WriterOpts {
compress_lvl: 1,
data_buf_size: 10 * 1024 * 1024,
out_buf_size: 10 * 1024 * 1024,
current_buf_size: 4096,
};
let mut writer: Writer<TestData> = Writer::new(&tmpfile, opts).expect("new writer");
let items_iter = gen_data(10);
let items: Vec<TestData> = items_iter.collect();
writer.load(&mut items.clone().into_iter()).expect("load");
writer.finish().expect("finish write");
let reader: Reader<TestData> = Reader::new(&tmpfile, 2048).expect("new reader");
assert_eq!(items.len(), reader.len());
let reader = Arc::new(reader);
for _ in 0..=3 {
let cur_items = items.clone();
let cur_reader = Arc::clone(&reader);
thread::spawn(move || {
cur_items.into_iter().zip(cur_reader.iter()).for_each(|pair| {
assert_eq!(pair.0, pair.1);
});
});
}
}
} }