temp/ztb_recode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

async

Browse Source
This commit is contained in:
Dmitry Belyaev 2022-09-22 15:34:51 +03:00
parent 10e540c075
commit d48339b88b
Signed by: b4tman
GPG Key ID: 41A00BF15EA7E5F3
3 changed files with 454 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

540
Cargo.lock generated
View File

@ -8,18 +8,6 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "aes"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8"
dependencies = [
"cfg-if",
"cipher",
"cpufeatures",
"opaque-debug",
]
[[package]] [[package]]
name = "aho-corasick" name = "aho-corasick"
version = "0.7.19" version = "0.7.19"
@ -29,6 +17,53 @@ dependencies = [
"memchr", "memchr",
] ]
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]]
name = "async-compression"
version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "345fd392ab01f746c717b1357165b76f0b67a60192007b234058c9045fdcf695"
dependencies = [
"bzip2",
"flate2",
"futures-core",
"memchr",
"pin-project-lite",
"tokio",
"xz2",
"zstd",
"zstd-safe",
]
[[package]]
name = "async_io_utilities"
version = "0.1.3"
source = "git+https://github.com/Majored/rs-async-io-utilities#6661a0fb914ccc1f97a6acd446bc03e9547d64fc"
dependencies = [
"tokio",
]
[[package]]
name = "async_zip"
version = "0.0.8"
source = "git+https://github.com/Majored/rs-async-zip?rev=0f4bf7f0#0f4bf7f09d2fef1236924a6af45a5641392faf6b"
dependencies = [
"async-compression",
"async_io_utilities",
"chrono",
"crc32fast",
"thiserror",
"tokio",
]
[[package]] [[package]]
name = "atty" name = "atty"
version = "0.2.14" version = "0.2.14"
@ -46,12 +81,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "base64ct"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a32fd6af2b5827bce66c29053ba0e7c42b9dcab01835835058558c10851a46b"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "1.3.2" version = "1.3.2"
@ -59,19 +88,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]] [[package]]
name = "block-buffer" name = "bumpalo"
version = "0.10.3" version = "3.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d"
dependencies = [
"generic-array",
]
[[package]] [[package]]
name = "byteorder" name = "bytes"
version = "1.4.3" version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
[[package]] [[package]]
name = "bzip2" name = "bzip2"
@ -110,12 +136,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]] [[package]]
name = "cipher" name = "chrono"
version = "0.3.0" version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1"
dependencies = [ dependencies = [
"generic-array", "iana-time-zone",
"js-sys",
"num-integer",
"num-traits",
"time",
"wasm-bindgen",
"winapi",
] ]
[[package]] [[package]]
@ -158,19 +190,10 @@ dependencies = [
] ]
[[package]] [[package]]
name = "constant_time_eq" name = "core-foundation-sys"
version = "0.1.5" version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]]
name = "cpufeatures"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "crc32fast" name = "crc32fast"
@ -181,37 +204,6 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "crossbeam-utils"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc"
dependencies = [
"cfg-if",
"once_cell",
]
[[package]]
name = "crypto-common"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "digest"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
dependencies = [
"block-buffer",
"crypto-common",
"subtle",
]
[[package]] [[package]]
name = "encoding" name = "encoding"
version = "0.2.33" version = "0.2.33"
@ -287,14 +279,10 @@ dependencies = [
] ]
[[package]] [[package]]
name = "generic-array" name = "futures-core"
version = "0.14.6" version = "0.3.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf"
dependencies = [
"typenum",
"version_check",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
@ -318,12 +306,17 @@ dependencies = [
] ]
[[package]] [[package]]
name = "hmac" name = "iana-time-zone"
version = "0.12.1" version = "0.1.48"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" checksum = "237a0714f28b1ee39ccec0770ccb544eb02c9ef2c82bb096230eefcffa6468b0"
dependencies = [ dependencies = [
"digest", "android_system_properties",
"core-foundation-sys",
"js-sys",
"once_cell",
"wasm-bindgen",
"winapi",
] ]
[[package]] [[package]]
@ -336,12 +329,6 @@ dependencies = [
"hashbrown", "hashbrown",
] ]
[[package]]
name = "itoa"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"
[[package]] [[package]]
name = "jobserver" name = "jobserver"
version = "0.1.24" version = "0.1.24"
@ -351,12 +338,51 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "js-sys"
version = "0.3.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
dependencies = [
"wasm-bindgen",
]
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.132" version = "0.2.132"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5"
[[package]]
name = "lock_api"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"
dependencies = [
"autocfg",
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
dependencies = [
"cfg-if",
]
[[package]]
name = "lzma-sys"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e06754c4acf47d49c727d5665ca9fb828851cda315ed3bd51edd148ef78a8772"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.5.0" version = "2.5.0"
@ -373,12 +399,44 @@ dependencies = [
] ]
[[package]] [[package]]
name = "num_threads" name = "mio"
version = "0.1.6" version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf"
dependencies = [ dependencies = [
"libc", "libc",
"log",
"wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys",
]
[[package]]
name = "num-integer"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
dependencies = [
"hermit-abi",
"libc",
] ]
[[package]] [[package]]
@ -387,12 +445,6 @@ version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0" checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0"
[[package]]
name = "opaque-debug"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]] [[package]]
name = "os_str_bytes" name = "os_str_bytes"
version = "6.3.0" version = "6.3.0"
@ -400,28 +452,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff"
[[package]] [[package]]
name = "password-hash" name = "parking_lot"
version = "0.3.2" version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d791538a6dcc1e7cb7fe6f6b58aca40e7f79403c45b2bc274008b5e647af1d8" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [ dependencies = [
"base64ct", "lock_api",
"rand_core", "parking_lot_core",
"subtle",
] ]
[[package]] [[package]]
name = "pbkdf2" name = "parking_lot_core"
version = "0.10.1" version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271779f35b581956db91a3e55737327a03aa051e90b1c47aeb189508533adfd7" checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
dependencies = [ dependencies = [
"digest", "cfg-if",
"hmac", "libc",
"password-hash", "redox_syscall",
"sha2", "smallvec",
"windows-sys",
] ]
[[package]]
name = "pin-project-lite"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
[[package]] [[package]]
name = "pkg-config" name = "pkg-config"
version = "0.3.25" version = "0.3.25"
@ -471,10 +529,13 @@ dependencies = [
] ]
[[package]] [[package]]
name = "rand_core" name = "redox_syscall"
version = "0.6.3" version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags",
]
[[package]] [[package]]
name = "regex" name = "regex"
@ -494,25 +555,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
[[package]] [[package]]
name = "sha1" name = "scopeguard"
version = "0.10.4" version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "006769ba83e921b3085caa8334186b00cf92b4cb1a6cf4632fbccc8eff5c7549" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "signal-hook-registry"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0"
dependencies = [ dependencies = [
"cfg-if", "libc",
"cpufeatures",
"digest",
] ]
[[package]] [[package]]
name = "sha2" name = "smallvec"
version = "0.10.5" version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf9db03534dff993187064c4e0c05a5708d2a9728ace9a8959b77bedf415dac5" checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"
[[package]]
name = "socket2"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd"
dependencies = [ dependencies = [
"cfg-if", "libc",
"cpufeatures", "winapi",
"digest",
] ]
[[package]] [[package]]
@ -521,12 +591,6 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "subtle"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]] [[package]]
name = "syn" name = "syn"
version = "1.0.100" version = "1.0.100"
@ -554,28 +618,67 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16"
[[package]] [[package]]
name = "time" name = "thiserror"
version = "0.3.14" version = "1.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b" checksum = "c53f98874615aea268107765aa1ed8f6116782501d18e53d08b471733bea6c85"
dependencies = [ dependencies = [
"itoa", "thiserror-impl",
"libc",
"num_threads",
"time-macros",
] ]
[[package]] [[package]]
name = "time-macros" name = "thiserror-impl"
version = "0.2.4" version = "1.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" checksum = "f8b463991b4eab2d801e724172285ec4195c650e8ec79b149e6c2a8e6dd3f783"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "typenum" name = "time"
version = "1.15.0" version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
dependencies = [
"libc",
"wasi 0.10.0+wasi-snapshot-preview1",
"winapi",
]
[[package]]
name = "tokio"
version = "1.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0020c875007ad96677dcc890298f4b942882c5d4eb7cc8f439fc3bf813dc9c95"
dependencies = [
"autocfg",
"bytes",
"libc",
"memchr",
"mio",
"num_cpus",
"once_cell",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"tokio-macros",
"winapi",
]
[[package]]
name = "tokio-macros"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "unicode-ident" name = "unicode-ident"
@ -589,6 +692,72 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wasi"
version = "0.10.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
[[package]] [[package]]
name = "winapi" name = "winapi"
version = "0.3.9" version = "0.3.9"
@ -621,39 +790,71 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]] [[package]]
name = "zip" name = "windows-sys"
version = "0.6.2" version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf225bcf73bb52cbb496e70475c7bd7a3f769df699c0020f6c7bd9a96dcf0b8d" checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
dependencies = [ dependencies = [
"aes", "windows_aarch64_msvc",
"byteorder", "windows_i686_gnu",
"bzip2", "windows_i686_msvc",
"constant_time_eq", "windows_x86_64_gnu",
"crc32fast", "windows_x86_64_msvc",
"crossbeam-utils", ]
"flate2",
"hmac", [[package]]
"pbkdf2", name = "windows_aarch64_msvc"
"sha1", version = "0.36.1"
"time", source = "registry+https://github.com/rust-lang/crates.io-index"
"zstd", checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
[[package]]
name = "windows_i686_gnu"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
[[package]]
name = "windows_i686_msvc"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
[[package]]
name = "windows_x86_64_gnu"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
[[package]]
name = "xz2"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
dependencies = [
"lzma-sys",
] ]
[[package]] [[package]]
name = "zstd" name = "zstd"
version = "0.10.2+zstd.1.5.2" version = "0.11.2+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f4a6bd64f22b5e3e94b4e238669ff9f10815c27a5180108b849d24174a83847" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4"
dependencies = [ dependencies = [
"zstd-safe", "zstd-safe",
] ]
[[package]] [[package]]
name = "zstd-safe" name = "zstd-safe"
version = "4.1.6+zstd.1.5.2" version = "5.0.2+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94b61c51bb270702d6167b8ce67340d2754b088d0c091b06e593aa772c3ee9bb" checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db"
dependencies = [ dependencies = [
"libc", "libc",
"zstd-sys", "zstd-sys",
@ -661,9 +862,9 @@ dependencies = [
[[package]] [[package]]
name = "zstd-sys" name = "zstd-sys"
version = "1.6.3+zstd.1.5.2" version = "2.0.1+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc49afa5c8d634e75761feda8c592051e7eeb4683ba827211eb0d731d3402ea8" checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b"
dependencies = [ dependencies = [
"cc", "cc",
"libc", "libc",
@ -673,8 +874,9 @@ dependencies = [
name = "ztb_recode" name = "ztb_recode"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"async_zip",
"clap", "clap",
"encoding", "encoding",
"regex", "regex",
"zip", "tokio",
] ]

3
Cargo.toml
View File

@ -6,10 +6,11 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
async_zip = { git = "https://github.com/Majored/rs-async-zip", rev="0f4bf7f0" }
clap = { version = "3.2.22", features = ["derive"] } clap = { version = "3.2.22", features = ["derive"] }
encoding = "0.2.33" encoding = "0.2.33"
regex = "1.6.0" regex = "1.6.0"
zip = "0.6.2" tokio = { version = "1.21.1", features = ["full"] }
[profile.release] [profile.release]
opt-level = 3 opt-level = 3

151
src/main.rs
View File

@ -1,18 +1,20 @@
extern crate async_zip;
extern crate encoding; extern crate encoding;
extern crate zip; extern crate tokio;
use async_zip::read::fs::ZipFileReader;
use async_zip::write::{EntryOptions, ZipFileWriter};
use async_zip::Compression;
use clap::{Parser, ValueEnum}; use clap::{Parser, ValueEnum};
use encoding::label::encoding_from_whatwg_label; use encoding::label::encoding_from_whatwg_label;
use encoding::EncodingRef; use encoding::EncodingRef;
use encoding::{DecoderTrap, EncoderTrap}; use encoding::{DecoderTrap, EncoderTrap};
use regex::Regex; use regex::Regex;
use std::fs; use tokio::io::AsyncReadExt;
use std::io::{Read, Write}; use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
use std::sync::mpsc; use tokio::{fs, task};
use std::thread;
use zip::ZipWriter;
/// transcode txt files in zip archieve /// transcode txt files in zip archive
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
#[clap(author, version, about, long_about = None)] #[clap(author, version, about, long_about = None)]
#[clap(propagate_version = true)] #[clap(propagate_version = true)]
@ -29,10 +31,6 @@ struct Cli {
#[clap(arg_enum, short, long, default_value = "zstd")] #[clap(arg_enum, short, long, default_value = "zstd")]
compression: OutputFileCompression, compression: OutputFileCompression,
/// output compression level
#[clap(arg_enum, short='l', long, value_parser = clap::value_parser!(i32).range(1..=9), default_value = "5")]
compression_level: i32,
/// filename filter (regex) /// filename filter (regex)
#[clap(short, long, default_value = r#".*\.txt$"#)] #[clap(short, long, default_value = r#".*\.txt$"#)]
regex: String, regex: String,
@ -55,17 +53,23 @@ enum OutputFileCompression {
Deflate, Deflate,
/// Compress the file using BZIP2 /// Compress the file using BZIP2
Bzip2, Bzip2,
/// Compress the file using LZMA
Lzma,
/// Compress the file using ZStandard /// Compress the file using ZStandard
Zstd, Zstd,
/// Compress the file using XZ
Xz,
} }
impl From<OutputFileCompression> for zip::CompressionMethod { impl From<OutputFileCompression> for Compression {
fn from(compression: OutputFileCompression) -> Self { fn from(compression: OutputFileCompression) -> Self {
match compression { match compression {
OutputFileCompression::Store => Self::Stored, OutputFileCompression::Store => Self::Stored,
OutputFileCompression::Deflate => Self::Deflated, OutputFileCompression::Deflate => Self::Deflate,
OutputFileCompression::Bzip2 => Self::Bzip2, OutputFileCompression::Bzip2 => Self::Bz,
OutputFileCompression::Lzma => Self::Lzma,
OutputFileCompression::Zstd => Self::Zstd, OutputFileCompression::Zstd => Self::Zstd,
OutputFileCompression::Xz => Self::Xz,
} }
} }
} }
@ -76,43 +80,55 @@ struct FileData {
data: Vec<u8>, data: Vec<u8>,
} }
fn reader_task(tx: mpsc::Sender<FileData>, input_filename: String, regex: Regex) { async fn reader_task(tx: UnboundedSender<FileData>, input_filename: String, regex: Regex) {
let zip_file = fs::File::open(input_filename).unwrap(); let archive = ZipFileReader::new(input_filename).await.unwrap();
let mut archive = zip::ZipArchive::new(zip_file).unwrap();
let mut source_files: Vec<String> = archive let mut source_files: Vec<(usize, String, u32)> = archive
.file_names() .entries()
.filter(|name| regex.is_match(name)) .iter()
.map(|s| s.to_string()) .enumerate()
.filter(|(_, entry)| !entry.dir())
.filter(|(_, entry)| regex.is_match(entry.name()))
.map(|(index, entry)| {
(
index,
entry.name().to_string(),
entry.uncompressed_size().unwrap(),
)
})
.collect(); .collect();
source_files.sort_by(|(_, name_a, _), (_, name_b, _)| name_a.partial_cmp(name_b).unwrap());
println!("processing {} files...", source_files.len()); let mut count: usize = 0;
for (index, name, uncompressed_size) in source_files {
source_files.sort(); let mut entry_reader = archive.entry_reader(index).await.unwrap();
let mut data = Vec::with_capacity(uncompressed_size.try_into().unwrap());
for name in source_files { entry_reader.read_to_end(&mut data).await.unwrap();
let mut file = archive.by_name(name.as_str()).unwrap(); drop(entry_reader);
let mut data = Vec::with_capacity(file.size().try_into().unwrap());
file.read_to_end(&mut data).unwrap();
drop(file);
tx.send(FileData { name, data }).unwrap(); tx.send(FileData { name, data }).unwrap();
count += 1;
} }
println!("read done ✅"); println!("read {count} files done ✅");
} }
fn transcoder_task( async fn transcoder_task(
rx: mpsc::Receiver<FileData>, mut rx: UnboundedReceiver<FileData>,
tx: mpsc::Sender<FileData>, tx: UnboundedSender<FileData>,
encoding_from: EncodingRef, encoding_from: EncodingRef,
encoding_to: EncodingRef, encoding_to: EncodingRef,
) { ) {
while let Ok(FileData { name, data }) = rx.recv() { while let Some(FileData { name, data }) = rx.recv().await {
let new_data = task::spawn_blocking(move || {
let text = encoding_from.decode(&data, DecoderTrap::Ignore).unwrap(); let text = encoding_from.decode(&data, DecoderTrap::Ignore).unwrap();
let new_data = encoding_to encoding_to
.encode(text.as_str(), EncoderTrap::Ignore) .encode(text.as_str(), EncoderTrap::Ignore)
.unwrap()
})
.await
.unwrap(); .unwrap();
tx.send(FileData { tx.send(FileData {
name, name,
data: new_data, data: new_data,
@ -122,59 +138,48 @@ fn transcoder_task(
println!("transcode done ✅"); println!("transcode done ✅");
} }
fn writer_task( async fn writer_task(
rx: mpsc::Receiver<FileData>, mut rx: UnboundedReceiver<FileData>,
output_filename: String, output_filename: String,
compression: zip::CompressionMethod, compression: Compression,
compression_level: i32,
) { ) {
let options = zip::write::FileOptions::default() let mut outfile = fs::File::create(output_filename)
.compression_method(compression) .await
.compression_level(Some(compression_level)); .expect("output file");
let mut writer = ZipFileWriter::new(&mut outfile);
let mut outfile = fs::File::create(output_filename).expect("output file"); while let Some(FileData { name, data }) = rx.recv().await {
let mut zip_writer = ZipWriter::new(&mut outfile); let opts = EntryOptions::new(name, compression);
writer.write_entry_whole(opts, &data).await.unwrap();
while let Ok(FileData { name, data }) = rx.recv() {
zip_writer.start_file(name, options).unwrap();
zip_writer.write_all(&data).unwrap();
} }
zip_writer.finish().unwrap(); writer.close().await.unwrap();
println!("write done ✅"); println!("write done ✅");
} }
fn main() -> Result<(), Box<dyn std::error::Error>> { #[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let args = Cli::parse(); let args = Cli::parse();
let regex = Regex::new(&args.regex).expect("regex"); let regex = Regex::new(&args.regex).expect("regex");
let encoding_from = encoding_from_whatwg_label(&args.from).expect("input encoding"); let encoding_from = encoding_from_whatwg_label(&args.from).expect("input encoding");
let encoding_to = encoding_from_whatwg_label(&args.to).expect("output encoding"); let encoding_to = encoding_from_whatwg_label(&args.to).expect("output encoding");
let compression: zip::CompressionMethod = args.compression.into(); let compression: Compression = args.compression.into();
let compression_level = args.compression_level;
let input_filename = args.src; let input_filename = args.src;
let output_filename = args.dst; let output_filename = args.dst;
let (reader_tx, reader_rx) = mpsc::channel::<FileData>(); let (reader_tx, reader_rx) = mpsc::unbounded_channel::<FileData>();
let (transcoder_tx, transcoder_rx) = mpsc::channel::<FileData>(); let (transcoder_tx, transcoder_rx) = mpsc::unbounded_channel::<FileData>();
let handles = vec![ tokio::try_join!(
thread::spawn(move || reader_task(reader_tx, input_filename, regex)), tokio::spawn(reader_task(reader_tx, input_filename, regex)),
thread::spawn(move || { tokio::spawn(transcoder_task(
transcoder_task(reader_rx, transcoder_tx, encoding_from, encoding_to) reader_rx,
}), transcoder_tx,
thread::spawn(move || { encoding_from,
writer_task( encoding_to
transcoder_rx, )),
output_filename, tokio::spawn(writer_task(transcoder_rx, output_filename, compression))
compression, )?;
compression_level,
)
}),
];
for handle in handles {
handle.join().expect("thread paniced");
}
println!("all done ✅"); println!("all done ✅");
Ok(()) Ok(())