check for utf-8 encodings
This commit is contained in:
parent
0b98540e35
commit
c95c1fb785
33
src/main.rs
33
src/main.rs
@ -7,10 +7,10 @@ use async_zip::write::{EntryOptions, ZipFileWriter};
|
||||
use async_zip::Compression;
|
||||
use clap::{Parser, ValueEnum};
|
||||
use encoding::label::encoding_from_whatwg_label;
|
||||
use encoding::EncodingRef;
|
||||
use encoding::{all::UTF_8, Encoding, EncodingRef};
|
||||
use encoding::{DecoderTrap, EncoderTrap};
|
||||
use regex::Regex;
|
||||
use tokio::io::{AsyncReadExt, BufWriter, AsyncWriteExt};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt, BufWriter};
|
||||
use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
|
||||
use tokio::{fs, task};
|
||||
|
||||
@ -119,13 +119,30 @@ async fn transcoder_task(
|
||||
encoding_from: EncodingRef,
|
||||
encoding_to: EncodingRef,
|
||||
) {
|
||||
let is_encodings_same = encoding_from.name() == encoding_to.name();
|
||||
let is_src_encodig_native = UTF_8.name() == encoding_from.name();
|
||||
let is_dst_encodig_native = UTF_8.name() == encoding_to.name();
|
||||
|
||||
while let Some(FileData { name, data }) = rx.recv().await {
|
||||
let new_data = task::block_in_place(move || {
|
||||
let text = encoding_from.decode(&data, DecoderTrap::Ignore).unwrap();
|
||||
encoding_to
|
||||
.encode(text.as_str(), EncoderTrap::Ignore)
|
||||
.unwrap()
|
||||
});
|
||||
let new_data = if is_encodings_same {
|
||||
data
|
||||
} else {
|
||||
task::block_in_place(move || {
|
||||
let text = if is_src_encodig_native {
|
||||
String::from_utf8(data).unwrap()
|
||||
} else {
|
||||
encoding_from.decode(&data, DecoderTrap::Ignore).unwrap()
|
||||
};
|
||||
|
||||
if is_dst_encodig_native {
|
||||
text.into_bytes()
|
||||
} else {
|
||||
encoding_to
|
||||
.encode(text.as_str(), EncoderTrap::Ignore)
|
||||
.unwrap()
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
tx.send(FileData {
|
||||
name,
|
||||
|
Loading…
Reference in New Issue
Block a user