trait SplitTo
This commit is contained in:
parent
ab9816ee97
commit
5cf4cea9b0
30
src/main.rs
30
src/main.rs
@ -216,14 +216,19 @@ fn parse_file(file: impl io::Read) -> Result<json::JsonValue, Box<dyn std::error
|
|||||||
Ok(ctx.data.clone())
|
Ok(ctx.data.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
// split slice to a vector of slices
|
// split vector to a vector of [num] slices
|
||||||
fn split_vec<'a, T>(src: &'a [T], num: usize) -> Vec<&'a [T]> {
|
trait SplitTo<T> {
|
||||||
let part_len = src.len() / num;
|
fn split_to<'a>(&'a self, num: usize) -> Vec<&'a [T]>;
|
||||||
let add_len = src.len() % num;
|
}
|
||||||
|
|
||||||
|
impl<T> SplitTo<T> for Vec<T> {
|
||||||
|
fn split_to<'a>(&'a self, num: usize) -> Vec<&'a [T]> {
|
||||||
|
let part_len = self.len() / num;
|
||||||
|
let add_len = self.len() % num;
|
||||||
let mut result = Vec::<&'a [T]>::with_capacity(num);
|
let mut result = Vec::<&'a [T]>::with_capacity(num);
|
||||||
|
|
||||||
if 0 == part_len {
|
if 0 == part_len {
|
||||||
result.push(src);
|
result.push(self);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
for i in 0..num {
|
for i in 0..num {
|
||||||
@ -233,12 +238,21 @@ fn split_vec<'a, T>(src: &'a [T], num: usize) -> Vec<&'a [T]> {
|
|||||||
part_len
|
part_len
|
||||||
};
|
};
|
||||||
let start = part_len * i;
|
let start = part_len * i;
|
||||||
result.push(&src[start..(start + size)]);
|
result.push(&self[start..(start + size)]);
|
||||||
}
|
}
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
fn process_files(files: &&[PathBuf]) {
|
fn process_files(files: &&[PathBuf]) {
|
||||||
|
if files.len() < 1 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let start_file = files[0].to_str().unwrap();
|
||||||
|
println!("-> start from \"{}\" ({} files)", start_file, files.len());
|
||||||
|
|
||||||
let zip_file = fs::File::open(BASE_FILENAME).unwrap();
|
let zip_file = fs::File::open(BASE_FILENAME).unwrap();
|
||||||
let zip_reader = io::BufReader::new(zip_file);
|
let zip_reader = io::BufReader::new(zip_file);
|
||||||
let mut archive = zip::ZipArchive::new(zip_reader).unwrap();
|
let mut archive = zip::ZipArchive::new(zip_reader).unwrap();
|
||||||
@ -259,6 +273,8 @@ fn process_files(files: &&[PathBuf]) {
|
|||||||
let mut outfile = fs::File::create(outfilename).unwrap();
|
let mut outfile = fs::File::create(outfilename).unwrap();
|
||||||
data.write_pretty(&mut outfile, 1).unwrap();
|
data.write_pretty(&mut outfile, 1).unwrap();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
println!("<- done {} files (from \"{}\")", files.len(), start_file);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
@ -294,7 +310,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||||||
println!("processing {} files with {} threads...", source_files.len(), rayon::current_num_threads());
|
println!("processing {} files with {} threads...", source_files.len(), rayon::current_num_threads());
|
||||||
|
|
||||||
// split vector and process its parts in parallel
|
// split vector and process its parts in parallel
|
||||||
split_vec(&source_files, rayon::current_num_threads())
|
source_files.split_to(rayon::current_num_threads())
|
||||||
.par_iter()
|
.par_iter()
|
||||||
.for_each(process_files);
|
.for_each(process_files);
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user