diff --git a/.gitignore b/.gitignore index 3c3bdd0e..17af785e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ /target **/*.rs.bk +wasi-sysroot*.tar.gz +wasi-sysroot/ + # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html Cargo.lock diff --git a/Cargo.toml b/Cargo.toml index 3a1e3d8b..85782c7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,16 +1,32 @@ -[package] -name = "async-compression" -version = "0.4.27" -authors = ["Wim Looman ", "Allen Bui "] -edition = "2018" +[workspace] +members = ["crates/*"] resolver = "2" + +[workspace.package] +authors = ["Wim Looman ", "Allen Bui "] license = "MIT OR Apache-2.0" -keywords = ["compression", "gzip", "zstd", "brotli", "async"] -categories = ["compression", "asynchronous"] repository = "https://github.com/Nullus157/async-compression" +categories = ["compression", "asynchronous"] +version = "0.4.27" +edition = "2018" + +[workspace.dependencies] +compression-codecs = { version = "0.4.27", path = "crates/compression-codecs" } +compression-core = { version = "0.4.27", path = "crates/compression-core" } +futures-core = { version = "0.3", default-features = false } +memchr = "2" +pin-project-lite = "0.2" + +[package] +name = "async-compression" description = """ Adaptors between compression crates and Rust's modern asynchronous IO types. """ +version.workspace = true +authors.workspace = true +license.workspace = true +categories.workspace = true +edition.workspace = true [package.metadata.docs.rs] all-features = true @@ -20,31 +36,52 @@ rustdoc-args = ["--cfg", "docsrs"] # groups all = ["all-implementations", "all-algorithms"] all-implementations = ["futures-io", "tokio"] -all-algorithms = ["brotli", "bzip2", "deflate", "gzip", "lz4", "lzma", "xz-parallel", "xz", "zlib", "zstd", "deflate64"] +all-algorithms = [ + "brotli", + "bzip2", + "deflate", + "deflate64", + "gzip", + "lz4", + "lzma", + "xz", + "xz-parallel", + "zlib", + "zstd", +] # algorithms -deflate = ["flate2"] -gzip = ["flate2"] -lz4 = ["dep:lz4"] -lzma = ["dep:liblzma"] -xz = ["lzma"] -xz-parallel = ["xz", "liblzma/parallel"] -xz2 = ["xz"] -zlib = ["flate2"] -zstd = ["libzstd", "zstd-safe"] -zstdmt = ["zstd", "zstd-safe/zstdmt"] -deflate64 = ["dep:deflate64"] +brotli = ["compression-codecs/brotli", "dep:brotli"] +bzip2 = ["compression-codecs/bzip2", "dep:bzip2"] +deflate = ["compression-codecs/deflate", "flate2"] +deflate64 = ["compression-codecs/deflate64", "dep:deflate64"] +gzip = ["compression-codecs/gzip", "flate2"] +lz4 = ["compression-codecs/lz4", "dep:lz4"] +lzma = ["compression-codecs/lzma", "liblzma"] +xz = ["compression-codecs/xz", "lzma"] +xz-parallel = ["compression-codecs/xz-parallel", "xz", "liblzma/parallel"] +xz2 = ["compression-codecs/xz2", "xz"] +zlib = ["compression-codecs/zlib", "flate2"] +zstd = ["compression-codecs/zstd", "libzstd", "zstd-safe"] +zstdmt = ["compression-codecs/zstdmt", "zstd", "zstd-safe/zstdmt"] + [dependencies] +# core dependencies +futures-core.workspace = true +memchr.workspace = true +pin-project-lite.workspace = true +compression-codecs.workspace = true +compression-core.workspace = true +# optionals deps brotli = { version = "8", optional = true } bzip2 = { version = "0.6", optional = true } flate2 = { version = "1.0.13", optional = true } -futures-core = { version = "0.3", default-features = false } -futures-io = { version = "0.3", default-features = false, features = ["std"], optional = true } +futures-io = { version = "0.3", default-features = false, features = [ + "std", +], optional = true } libzstd = { package = "zstd", version = "0.13.1", optional = true, default-features = false } lz4 = { version = "1.28.1", optional = true } -memchr = "2" -pin-project-lite = "0.2" tokio = { version = "1.24.2", optional = true, default-features = false } liblzma = { version = "0.4.2", optional = true } zstd-safe = { version = "7", optional = true, default-features = false } @@ -58,7 +95,12 @@ ntest = "0.9" proptest = "1" proptest-derive = "0.6" rand = "0.9" -tokio = { version = "1.38.2", default-features = false, features = ["io-util", "macros", "rt-multi-thread", "io-std"] } +tokio = { version = "1.38.2", default-features = false, features = [ + "io-util", + "macros", + "rt-multi-thread", + "io-std", +] } tokio-util = { version = "0.7", default-features = false, features = ["io"] } [[test]] @@ -112,3 +154,8 @@ required-features = ["zlib", "tokio"] [[example]] name = "zstd_gzip" required-features = ["zstd", "gzip", "tokio"] + + +[[example]] +name = "lzma_filters" +required-features = ["xz", "tokio"] diff --git a/README.md b/README.md index 7757d730..89673d62 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,43 @@ enable different subsets of features as appropriate for the code you are testing to avoid compiling all dependencies, e.g. `cargo test --features tokio,gzip`. +To prepare for a pull request, you can run several other checks: + +1. `fmt` + + ```bash + cargo fmt --all + cargo clippy --no-deps + ``` + +2. `build` + + ```bash + cargo build --lib --all-features + ``` + +3. `nextest` + + ```bash + cargo --locked nextest run --workspace --all-features + ``` + +4. `hack check` + + ```bash + cargo hack check --workspace --feature-powerset --all-targets --skip 'all,all-algorithms,all-implementations' + ``` + +5. `wasm32` - Linux only + + ```bash + gh release download --repo WebAssembly/wasi-sdk --pattern 'wasi-sysroot-*.tar.gz' + rustup target add wasm32-wasip1-threads + + export "CFLAGS_wasm32_wasip1_threads=--sysroot=\"${PWD}/wasi-sysroot\" -I\"${PWD}/wasi-sysroot/include/wasm32-wasip1-threads\" -L-I\"${PWD}/wasi-sysroot/lib/wasm32-wasip1-threads\"" + cargo build --lib --features all-implementations,brotli,bzip2,deflate,gzip,lz4,lzma,xz,zlib,zstd,deflate64 --target wasm32-wasip1-threads + ``` + ## License Licensed under either of diff --git a/crates/compression-codecs/Cargo.toml b/crates/compression-codecs/Cargo.toml new file mode 100644 index 00000000..53bd4af0 --- /dev/null +++ b/crates/compression-codecs/Cargo.toml @@ -0,0 +1,57 @@ +[package] +name = "compression-codecs" +description = """ +Adaptors for various compression algorithms. +""" +version.workspace = true +authors.workspace = true +license.workspace = true +categories.workspace = true +edition.workspace = true +repository.workspace = true +readme = "../../README.md" + +[features] +all-algorithms = [ + "brotli", + "bzip2", + "deflate", + "gzip", + "lz4", + "lzma", + "xz-parallel", + "xz", + "zlib", + "zstd", + "deflate64", +] + +# algorithms +deflate = ["flate2"] +gzip = ["flate2"] +lz4 = ["dep:lz4"] +lzma = ["dep:liblzma"] +xz = ["lzma"] +xz-parallel = ["xz", "liblzma/parallel"] +xz2 = ["xz"] +zlib = ["flate2"] +zstd = ["libzstd", "zstd-safe"] +zstdmt = ["zstd", "zstd-safe/zstdmt"] +deflate64 = ["dep:deflate64"] + + +[dependencies] +# Workspace dependencies. +compression-core.workspace = true +futures-core.workspace = true +memchr.workspace = true +pin-project-lite.workspace = true +# features +brotli = { version = "8", optional = true } +bzip2 = { version = "0.6", optional = true } +flate2 = { version = "1.0.13", optional = true } +libzstd = { package = "zstd", version = "0.13.1", optional = true, default-features = false } +lz4 = { version = "1.28.1", optional = true } +liblzma = { version = "0.4.2", optional = true } +zstd-safe = { version = "7", optional = true, default-features = false } +deflate64 = { version = "0.1.5", optional = true } diff --git a/src/codec/brotli/decoder.rs b/crates/compression-codecs/src/brotli/decoder.rs similarity index 93% rename from src/codec/brotli/decoder.rs rename to crates/compression-codecs/src/brotli/decoder.rs index 27e19b99..21e57f3e 100644 --- a/src/codec/brotli/decoder.rs +++ b/crates/compression-codecs/src/brotli/decoder.rs @@ -1,15 +1,15 @@ -use crate::{codec::Decode, util::PartialBuffer}; -use std::{fmt, io}; - +use crate::Decode; use brotli::{enc::StandardAlloc, BrotliDecompressStream, BrotliResult, BrotliState}; +use compression_core::util::PartialBuffer; +use std::{fmt, io}; pub struct BrotliDecoder { // `BrotliState` is very large (over 2kb) which is why we're boxing it. state: Box>, } -impl BrotliDecoder { - pub(crate) fn new() -> Self { +impl Default for BrotliDecoder { + fn default() -> Self { Self { state: Box::new(BrotliState::new( StandardAlloc::default(), @@ -18,6 +18,12 @@ impl BrotliDecoder { )), } } +} + +impl BrotliDecoder { + pub fn new() -> Self { + Self::default() + } fn decode( &mut self, @@ -25,7 +31,7 @@ impl BrotliDecoder { output: &mut PartialBuffer + AsMut<[u8]>>, ) -> io::Result { let in_buf = input.unwritten(); - let mut out_buf = output.unwritten_mut(); + let out_buf = output.unwritten_mut(); let mut input_len = 0; let mut output_len = 0; diff --git a/src/codec/brotli/encoder.rs b/crates/compression-codecs/src/brotli/encoder.rs similarity index 90% rename from src/codec/brotli/encoder.rs rename to crates/compression-codecs/src/brotli/encoder.rs index fe363f9c..c14dd331 100644 --- a/src/codec/brotli/encoder.rs +++ b/crates/compression-codecs/src/brotli/encoder.rs @@ -1,18 +1,19 @@ -use crate::{codec::Encode, util::PartialBuffer}; -use std::{fmt, io}; - +use crate::{brotli::params::EncoderParams, Encode}; use brotli::enc::{ backward_references::BrotliEncoderParams, encode::{BrotliEncoderOperation, BrotliEncoderStateStruct}, StandardAlloc, }; +use compression_core::util::PartialBuffer; +use std::{fmt, io}; pub struct BrotliEncoder { state: BrotliEncoderStateStruct, } impl BrotliEncoder { - pub(crate) fn new(params: BrotliEncoderParams) -> Self { + pub fn new(params: EncoderParams) -> Self { + let params = BrotliEncoderParams::from(params); let mut state = BrotliEncoderStateStruct::new(StandardAlloc::default()); state.params = params; Self { state } @@ -25,7 +26,7 @@ impl BrotliEncoder { op: BrotliEncoderOperation, ) -> io::Result<()> { let in_buf = input.unwritten(); - let mut out_buf = output.unwritten_mut(); + let out_buf = output.unwritten_mut(); let mut input_len = 0; let mut output_len = 0; diff --git a/crates/compression-codecs/src/brotli/mod.rs b/crates/compression-codecs/src/brotli/mod.rs new file mode 100644 index 00000000..3e62db46 --- /dev/null +++ b/crates/compression-codecs/src/brotli/mod.rs @@ -0,0 +1,5 @@ +mod decoder; +mod encoder; +pub mod params; + +pub use self::{decoder::BrotliDecoder, encoder::BrotliEncoder}; diff --git a/src/brotli.rs b/crates/compression-codecs/src/brotli/params.rs similarity index 56% rename from src/brotli.rs rename to crates/compression-codecs/src/brotli/params.rs index 70b622a5..d6bea896 100644 --- a/src/brotli.rs +++ b/crates/compression-codecs/src/brotli/params.rs @@ -1,6 +1,7 @@ //! This module contains Brotli-specific types for async-compression. use brotli::enc::backward_references::{BrotliEncoderMode, BrotliEncoderParams}; +use compression_core::Level; /// Brotli compression parameters builder. This is a stable wrapper around Brotli's own encoder /// params type, to abstract over different versions of the Brotli library. @@ -11,28 +12,49 @@ use brotli::enc::backward_references::{BrotliEncoderMode, BrotliEncoderParams}; /// # Examples /// /// ``` -/// use async_compression::brotli; +/// use compression_codecs::brotli; /// -/// let params = brotli::EncoderParams::default() +/// let params = brotli::params::EncoderParams::default() /// .window_size(12) /// .text_mode(); /// ``` -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[derive(Debug, Clone, Default)] pub struct EncoderParams { - window_size: Option, - block_size: Option, - size_hint: Option, - mode: Option, + inner: BrotliEncoderParams, +} + +impl From for BrotliEncoderParams { + fn from(value: EncoderParams) -> Self { + value.inner + } } impl EncoderParams { + pub fn quality(mut self, level: Level) -> Self { + let quality = match level { + Level::Fastest => Some(0), + Level::Best => Some(11), + Level::Precise(quality) => Some(quality.clamp(0, 11)), + _ => None, + }; + match quality { + Some(quality) => self.inner.quality = quality, + None => { + let default_params = BrotliEncoderParams::default(); + self.inner.quality = default_params.quality; + } + } + + self + } + /// Sets window size in bytes (as a power of two). /// /// Used as Brotli's `lgwin` parameter. /// /// `window_size` is clamped to `0 <= window_size <= 24`. pub fn window_size(mut self, window_size: i32) -> Self { - self.window_size = Some(window_size.clamp(0, 24)); + self.inner.lgwin = window_size.clamp(0, 24); self } @@ -42,13 +64,13 @@ impl EncoderParams { /// /// `block_size` is clamped to `16 <= block_size <= 24`. pub fn block_size(mut self, block_size: i32) -> Self { - self.block_size = Some(block_size.clamp(16, 24)); + self.inner.lgblock = block_size.clamp(16, 24); self } /// Sets hint for size of data to be compressed. pub fn size_hint(mut self, size_hint: usize) -> Self { - self.size_hint = Some(size_hint); + self.inner.size_hint = size_hint; self } @@ -59,36 +81,12 @@ impl EncoderParams { /// /// Used as Brotli's `mode` parameter. pub fn text_mode(mut self) -> Self { - self.mode = Some(BrotliEncoderMode::BROTLI_MODE_TEXT); + self.inner.mode = BrotliEncoderMode::BROTLI_MODE_TEXT; self } - pub(crate) fn as_brotli(&self) -> BrotliEncoderParams { - let mut params = BrotliEncoderParams::default(); - - let Self { - window_size, - block_size, - size_hint, - mode, - } = self; - - if let Some(window_size) = window_size { - params.lgwin = *window_size; - } - - if let Some(block_size) = block_size { - params.lgblock = *block_size; - } - - if let Some(size_hint) = size_hint { - params.size_hint = *size_hint; - } - - if let Some(mode) = mode { - params.mode = *mode; - } - - params + pub fn mode(mut self, mode: BrotliEncoderMode) -> Self { + self.inner.mode = mode; + self } } diff --git a/src/codec/bzip2/decoder.rs b/crates/compression-codecs/src/bzip2/decoder.rs similarity index 94% rename from src/codec/bzip2/decoder.rs rename to crates/compression-codecs/src/bzip2/decoder.rs index 255169b9..9364b3f3 100644 --- a/src/codec/bzip2/decoder.rs +++ b/crates/compression-codecs/src/bzip2/decoder.rs @@ -1,7 +1,7 @@ -use crate::{codec::Decode, util::PartialBuffer}; -use std::{fmt, io}; - +use crate::Decode; use bzip2::{Decompress, Status}; +use compression_core::util::PartialBuffer; +use std::{fmt, io}; pub struct BzDecoder { decompress: Decompress, @@ -18,12 +18,18 @@ impl fmt::Debug for BzDecoder { } } -impl BzDecoder { - pub(crate) fn new() -> Self { +impl Default for BzDecoder { + fn default() -> Self { Self { decompress: Decompress::new(false), } } +} + +impl BzDecoder { + pub fn new() -> Self { + Self::default() + } fn decode( &mut self, diff --git a/src/codec/bzip2/encoder.rs b/crates/compression-codecs/src/bzip2/encoder.rs similarity index 95% rename from src/codec/bzip2/encoder.rs rename to crates/compression-codecs/src/bzip2/encoder.rs index 3dbfc1ca..7a7dcced 100644 --- a/src/codec/bzip2/encoder.rs +++ b/crates/compression-codecs/src/bzip2/encoder.rs @@ -1,7 +1,7 @@ -use crate::{codec::Encode, util::PartialBuffer}; -use std::{fmt, io}; - +use crate::{bzip2::params::Bzip2EncoderParams, Encode}; use bzip2::{Action, Compress, Compression, Status}; +use compression_core::util::PartialBuffer; +use std::{fmt, io}; pub struct BzEncoder { compress: Compress, @@ -39,9 +39,10 @@ impl BzEncoder { /// /// Allowable values range from 0 to 250 inclusive. 0 is a special case, /// equivalent to using the default value of 30. - pub(crate) fn new(level: Compression, work_factor: u32) -> Self { + pub fn new(params: Bzip2EncoderParams, work_factor: u32) -> Self { + let params = Compression::from(params); Self { - compress: Compress::new(level, work_factor), + compress: Compress::new(params, work_factor), } } diff --git a/crates/compression-codecs/src/bzip2/mod.rs b/crates/compression-codecs/src/bzip2/mod.rs new file mode 100644 index 00000000..a43219c4 --- /dev/null +++ b/crates/compression-codecs/src/bzip2/mod.rs @@ -0,0 +1,5 @@ +mod decoder; +mod encoder; +pub mod params; + +pub use self::{decoder::BzDecoder, encoder::BzEncoder}; diff --git a/crates/compression-codecs/src/bzip2/params.rs b/crates/compression-codecs/src/bzip2/params.rs new file mode 100644 index 00000000..6ddae4ab --- /dev/null +++ b/crates/compression-codecs/src/bzip2/params.rs @@ -0,0 +1,33 @@ +use std::convert::TryInto; + +use bzip2::Compression; +use compression_core::Level; + +pub struct Bzip2EncoderParams { + inner: Compression, +} + +impl From for Compression { + fn from(value: Bzip2EncoderParams) -> Self { + value.inner + } +} +impl From for Bzip2EncoderParams { + fn from(value: Level) -> Self { + let fastest = bzip2::Compression::fast(); + let best = bzip2::Compression::best(); + + let inner = match value { + Level::Fastest => fastest, + Level::Best => best, + Level::Precise(quality) => bzip2::Compression::new( + quality + .try_into() + .unwrap_or(0) + .clamp(fastest.level(), best.level()), + ), + _ => bzip2::Compression::default(), + }; + Self { inner } + } +} diff --git a/src/codec/deflate/decoder.rs b/crates/compression-codecs/src/deflate/decoder.rs similarity index 72% rename from src/codec/deflate/decoder.rs rename to crates/compression-codecs/src/deflate/decoder.rs index e5f51671..d9452419 100644 --- a/src/codec/deflate/decoder.rs +++ b/crates/compression-codecs/src/deflate/decoder.rs @@ -1,20 +1,27 @@ -use crate::util::PartialBuffer; +use crate::{Decode, FlateDecoder}; +use compression_core::util::PartialBuffer; use std::io::Result; #[derive(Debug)] pub struct DeflateDecoder { - inner: crate::codec::FlateDecoder, + inner: FlateDecoder, } -impl DeflateDecoder { - pub(crate) fn new() -> Self { +impl Default for DeflateDecoder { + fn default() -> Self { Self { - inner: crate::codec::FlateDecoder::new(false), + inner: FlateDecoder::new(false), } } } -impl crate::codec::Decode for DeflateDecoder { +impl DeflateDecoder { + pub fn new() -> Self { + Self::default() + } +} + +impl Decode for DeflateDecoder { fn reinit(&mut self) -> Result<()> { self.inner.reinit()?; Ok(()) diff --git a/src/codec/deflate/encoder.rs b/crates/compression-codecs/src/deflate/encoder.rs similarity index 71% rename from src/codec/deflate/encoder.rs rename to crates/compression-codecs/src/deflate/encoder.rs index 7e6e7459..f24212bf 100644 --- a/src/codec/deflate/encoder.rs +++ b/crates/compression-codecs/src/deflate/encoder.rs @@ -1,21 +1,20 @@ -use crate::{codec::Encode, util::PartialBuffer}; +use crate::{flate::params::FlateEncoderParams, Encode, FlateEncoder}; +use compression_core::util::PartialBuffer; use std::io::Result; -use flate2::Compression; - #[derive(Debug)] pub struct DeflateEncoder { - inner: crate::codec::FlateEncoder, + inner: FlateEncoder, } impl DeflateEncoder { - pub(crate) fn new(level: Compression) -> Self { + pub fn new(level: FlateEncoderParams) -> Self { Self { - inner: crate::codec::FlateEncoder::new(level, false), + inner: FlateEncoder::new(level, false), } } - pub(crate) fn get_ref(&self) -> &crate::codec::FlateEncoder { + pub fn get_ref(&self) -> &FlateEncoder { &self.inner } } diff --git a/crates/compression-codecs/src/deflate/mod.rs b/crates/compression-codecs/src/deflate/mod.rs new file mode 100644 index 00000000..8b57f721 --- /dev/null +++ b/crates/compression-codecs/src/deflate/mod.rs @@ -0,0 +1,4 @@ +mod decoder; +mod encoder; + +pub use self::{decoder::DeflateDecoder, encoder::DeflateEncoder}; diff --git a/src/codec/deflate64/decoder.rs b/crates/compression-codecs/src/deflate64/decoder.rs similarity index 91% rename from src/codec/deflate64/decoder.rs rename to crates/compression-codecs/src/deflate64/decoder.rs index 5c8f1f0b..1bdf425f 100644 --- a/src/codec/deflate64/decoder.rs +++ b/crates/compression-codecs/src/deflate64/decoder.rs @@ -1,19 +1,25 @@ -use crate::{codec::Decode, util::PartialBuffer}; -use std::io::{Error, ErrorKind, Result}; - +use crate::Decode; +use compression_core::util::PartialBuffer; use deflate64::InflaterManaged; +use std::io::{Error, ErrorKind, Result}; #[derive(Debug)] pub struct Deflate64Decoder { inflater: Box, } -impl Deflate64Decoder { - pub(crate) fn new() -> Self { +impl Default for Deflate64Decoder { + fn default() -> Self { Self { inflater: Box::new(InflaterManaged::new()), } } +} + +impl Deflate64Decoder { + pub fn new() -> Self { + Self::default() + } fn decode( &mut self, diff --git a/crates/compression-codecs/src/deflate64/mod.rs b/crates/compression-codecs/src/deflate64/mod.rs new file mode 100644 index 00000000..b437bd24 --- /dev/null +++ b/crates/compression-codecs/src/deflate64/mod.rs @@ -0,0 +1,3 @@ +mod decoder; + +pub use self::decoder::Deflate64Decoder; diff --git a/src/codec/flate/decoder.rs b/crates/compression-codecs/src/flate/decoder.rs similarity index 97% rename from src/codec/flate/decoder.rs rename to crates/compression-codecs/src/flate/decoder.rs index 86fd139e..8797ff19 100644 --- a/src/codec/flate/decoder.rs +++ b/crates/compression-codecs/src/flate/decoder.rs @@ -1,7 +1,7 @@ -use crate::{codec::Decode, util::PartialBuffer}; -use std::io; - +use crate::Decode; +use compression_core::util::PartialBuffer; use flate2::{Decompress, FlushDecompress, Status}; +use std::io; #[derive(Debug)] pub struct FlateDecoder { diff --git a/src/codec/flate/encoder.rs b/crates/compression-codecs/src/flate/encoder.rs similarity index 90% rename from src/codec/flate/encoder.rs rename to crates/compression-codecs/src/flate/encoder.rs index bcf6c022..36223053 100644 --- a/src/codec/flate/encoder.rs +++ b/crates/compression-codecs/src/flate/encoder.rs @@ -1,8 +1,8 @@ -use crate::{codec::Encode, util::PartialBuffer}; +use crate::{flate::params::FlateEncoderParams, Encode}; +use compression_core::util::PartialBuffer; +use flate2::{Compress, FlushCompress, Status}; use std::io; -use flate2::{Compress, Compression, FlushCompress, Status}; - #[derive(Debug)] pub struct FlateEncoder { compress: Compress, @@ -10,14 +10,15 @@ pub struct FlateEncoder { } impl FlateEncoder { - pub(crate) fn new(level: Compression, zlib_header: bool) -> Self { + pub fn new(level: FlateEncoderParams, zlib_header: bool) -> Self { + let level = flate2::Compression::from(level); Self { compress: Compress::new(level, zlib_header), flushed: true, } } - pub(crate) fn get_ref(&self) -> &Compress { + pub fn get_ref(&self) -> &Compress { &self.compress } diff --git a/crates/compression-codecs/src/flate/mod.rs b/crates/compression-codecs/src/flate/mod.rs new file mode 100644 index 00000000..3b178a85 --- /dev/null +++ b/crates/compression-codecs/src/flate/mod.rs @@ -0,0 +1,5 @@ +mod decoder; +mod encoder; +pub mod params; + +pub use self::{decoder::FlateDecoder, encoder::FlateEncoder}; diff --git a/crates/compression-codecs/src/flate/params.rs b/crates/compression-codecs/src/flate/params.rs new file mode 100644 index 00000000..4c415bae --- /dev/null +++ b/crates/compression-codecs/src/flate/params.rs @@ -0,0 +1,40 @@ +use std::convert::TryInto; + +use compression_core::Level; + +#[derive(Clone)] +pub struct FlateEncoderParams { + inner: flate2::Compression, +} + +impl From for FlateEncoderParams { + fn from(inner: flate2::Compression) -> Self { + Self { inner } + } +} +impl From for flate2::Compression { + fn from(value: FlateEncoderParams) -> Self { + value.inner + } +} + +impl From for FlateEncoderParams { + fn from(value: Level) -> Self { + let fastest = flate2::Compression::fast(); + let best = flate2::Compression::best(); + let none = flate2::Compression::none(); + + let inner = match value { + Level::Fastest => fastest, + Level::Best => best, + Level::Precise(quality) => flate2::Compression::new( + quality + .try_into() + .unwrap_or(0) + .clamp(none.level(), best.level()), + ), + _ => flate2::Compression::default(), + }; + Self { inner } + } +} diff --git a/src/codec/gzip/decoder.rs b/crates/compression-codecs/src/gzip/decoder.rs similarity index 92% rename from src/codec/gzip/decoder.rs rename to crates/compression-codecs/src/gzip/decoder.rs index 4bb42606..2e20262f 100644 --- a/src/codec/gzip/decoder.rs +++ b/crates/compression-codecs/src/gzip/decoder.rs @@ -1,13 +1,8 @@ -use crate::{ - codec::{ - gzip::header::{self, Header}, - Decode, - }, - util::PartialBuffer, -}; -use std::io::{Error, ErrorKind, Result}; - +use super::header::{self, Header}; +use crate::{Decode, FlateDecoder}; +use compression_core::util::PartialBuffer; use flate2::Crc; +use std::io::{Error, ErrorKind, Result}; #[derive(Debug)] enum State { @@ -19,7 +14,7 @@ enum State { #[derive(Debug)] pub struct GzipDecoder { - inner: crate::codec::FlateDecoder, + inner: FlateDecoder, crc: Crc, state: State, header: Header, @@ -53,15 +48,21 @@ fn check_footer(crc: &Crc, input: &[u8]) -> Result<()> { Ok(()) } -impl GzipDecoder { - pub(crate) fn new() -> Self { +impl Default for GzipDecoder { + fn default() -> Self { Self { - inner: crate::codec::FlateDecoder::new(false), + inner: FlateDecoder::new(false), crc: Crc::new(), state: State::Header(header::Parser::default()), header: Header::default(), } } +} + +impl GzipDecoder { + pub fn new() -> Self { + Self::default() + } fn process, O: AsRef<[u8]> + AsMut<[u8]>>( &mut self, @@ -83,7 +84,7 @@ impl GzipDecoder { let res = inner(self, input, output); - if (output.written().len() > prior) { + if output.written().len() > prior { // update CRC even if there was an error self.crc.update(&output.written()[prior..]); } diff --git a/src/codec/gzip/encoder.rs b/crates/compression-codecs/src/gzip/encoder.rs similarity index 92% rename from src/codec/gzip/encoder.rs rename to crates/compression-codecs/src/gzip/encoder.rs index e6da04a8..b0d152a0 100644 --- a/src/codec/gzip/encoder.rs +++ b/crates/compression-codecs/src/gzip/encoder.rs @@ -1,7 +1,7 @@ -use crate::{codec::Encode, util::PartialBuffer}; -use std::io; - +use crate::{flate::params::FlateEncoderParams, Encode, FlateEncoder}; +use compression_core::util::PartialBuffer; use flate2::{Compression, Crc}; +use std::io; #[derive(Debug)] enum State { @@ -13,7 +13,7 @@ enum State { #[derive(Debug)] pub struct GzipEncoder { - inner: crate::codec::FlateEncoder, + inner: FlateEncoder, crc: Crc, state: State, } @@ -31,11 +31,11 @@ fn header(level: Compression) -> Vec { } impl GzipEncoder { - pub(crate) fn new(level: Compression) -> Self { + pub fn new(level: FlateEncoderParams) -> Self { Self { - inner: crate::codec::FlateEncoder::new(level, false), + inner: FlateEncoder::new(level.clone(), false), crc: Crc::new(), - state: State::Header(header(level).into()), + state: State::Header(header(Compression::from(level)).into()), } } diff --git a/src/codec/gzip/header.rs b/crates/compression-codecs/src/gzip/header.rs similarity index 97% rename from src/codec/gzip/header.rs rename to crates/compression-codecs/src/gzip/header.rs index ebe482af..abd98bff 100644 --- a/src/codec/gzip/header.rs +++ b/crates/compression-codecs/src/gzip/header.rs @@ -1,9 +1,9 @@ -use crate::util::PartialBuffer; +use compression_core::util::PartialBuffer; use std::io; #[derive(Debug, Default)] struct Flags { - ascii: bool, + _ascii: bool, crc: bool, extra: bool, filename: bool, @@ -50,7 +50,7 @@ impl Header { let flag = input[3]; let flags = Flags { - ascii: (flag & 0b0000_0001) != 0, + _ascii: (flag & 0b0000_0001) != 0, crc: (flag & 0b0000_0010) != 0, extra: (flag & 0b0000_0100) != 0, filename: (flag & 0b0000_1000) != 0, diff --git a/crates/compression-codecs/src/gzip/mod.rs b/crates/compression-codecs/src/gzip/mod.rs new file mode 100644 index 00000000..e0c5be3a --- /dev/null +++ b/crates/compression-codecs/src/gzip/mod.rs @@ -0,0 +1,5 @@ +mod decoder; +mod encoder; +mod header; + +pub use self::{decoder::GzipDecoder, encoder::GzipEncoder}; diff --git a/src/codec/mod.rs b/crates/compression-codecs/src/lib.rs similarity index 70% rename from src/codec/mod.rs rename to crates/compression-codecs/src/lib.rs index 52bdf4f9..3102bbcf 100644 --- a/src/codec/mod.rs +++ b/crates/compression-codecs/src/lib.rs @@ -1,55 +1,56 @@ -use crate::util::PartialBuffer; use std::io::Result; #[cfg(feature = "brotli")] -mod brotli; +pub mod brotli; #[cfg(feature = "bzip2")] -mod bzip2; +pub mod bzip2; #[cfg(feature = "deflate")] -mod deflate; +pub mod deflate; #[cfg(feature = "deflate64")] -mod deflate64; +pub mod deflate64; #[cfg(feature = "flate2")] -mod flate; +pub mod flate; #[cfg(feature = "gzip")] -mod gzip; +pub mod gzip; #[cfg(feature = "lz4")] -mod lz4; +pub mod lz4; #[cfg(feature = "lzma")] -mod lzma; +pub mod lzma; #[cfg(feature = "xz")] -mod xz; +pub mod xz; #[cfg(feature = "lzma")] -mod xz2; +pub mod xz2; #[cfg(feature = "zlib")] -mod zlib; +pub mod zlib; #[cfg(feature = "zstd")] -mod zstd; +pub mod zstd; + +use compression_core::util::PartialBuffer; #[cfg(feature = "brotli")] -pub(crate) use self::brotli::{BrotliDecoder, BrotliEncoder}; +pub use self::brotli::{BrotliDecoder, BrotliEncoder}; #[cfg(feature = "bzip2")] -pub(crate) use self::bzip2::{BzDecoder, BzEncoder}; +pub use self::bzip2::{BzDecoder, BzEncoder}; #[cfg(feature = "deflate")] -pub(crate) use self::deflate::{DeflateDecoder, DeflateEncoder}; +pub use self::deflate::{DeflateDecoder, DeflateEncoder}; #[cfg(feature = "deflate64")] -pub(crate) use self::deflate64::Deflate64Decoder; +pub use self::deflate64::Deflate64Decoder; #[cfg(feature = "flate2")] -pub(crate) use self::flate::{FlateDecoder, FlateEncoder}; +pub use self::flate::{FlateDecoder, FlateEncoder}; #[cfg(feature = "gzip")] -pub(crate) use self::gzip::{GzipDecoder, GzipEncoder}; +pub use self::gzip::{GzipDecoder, GzipEncoder}; #[cfg(feature = "lz4")] -pub(crate) use self::lz4::{Lz4Decoder, Lz4Encoder}; +pub use self::lz4::{Lz4Decoder, Lz4Encoder}; #[cfg(feature = "lzma")] -pub(crate) use self::lzma::{LzmaDecoder, LzmaEncoder}; +pub use self::lzma::{LzmaDecoder, LzmaEncoder}; #[cfg(feature = "xz")] -pub(crate) use self::xz::{XzDecoder, XzEncoder}; +pub use self::xz::{XzDecoder, XzEncoder}; #[cfg(feature = "lzma")] -pub(crate) use self::xz2::{Xz2Decoder, Xz2Encoder, Xz2FileFormat}; +pub use self::xz2::{Xz2Decoder, Xz2Encoder, Xz2FileFormat}; #[cfg(feature = "zlib")] -pub(crate) use self::zlib::{ZlibDecoder, ZlibEncoder}; +pub use self::zlib::{ZlibDecoder, ZlibEncoder}; #[cfg(feature = "zstd")] -pub(crate) use self::zstd::{ZstdDecoder, ZstdEncoder}; +pub use self::zstd::{ZstdDecoder, ZstdEncoder}; pub trait Encode { fn encode( diff --git a/src/codec/lz4/decoder.rs b/crates/compression-codecs/src/lz4/decoder.rs similarity index 92% rename from src/codec/lz4/decoder.rs rename to crates/compression-codecs/src/lz4/decoder.rs index e7e356ca..185d7a61 100644 --- a/src/codec/lz4/decoder.rs +++ b/crates/compression-codecs/src/lz4/decoder.rs @@ -1,11 +1,10 @@ -use std::io::Result; - +use crate::Decode; +use compression_core::{unshared::Unshared, util::PartialBuffer}; use lz4::liblz4::{ check_error, LZ4FDecompressionContext, LZ4F_createDecompressionContext, LZ4F_decompress, LZ4F_freeDecompressionContext, LZ4F_resetDecompressionContext, LZ4F_VERSION, }; - -use crate::{codec::Decode, unshared::Unshared, util::PartialBuffer}; +use std::io::Result; #[derive(Debug)] struct DecoderContext { @@ -31,14 +30,20 @@ impl Drop for DecoderContext { } } -impl Lz4Decoder { - pub(crate) fn new() -> Self { +impl Default for Lz4Decoder { + fn default() -> Self { Self { ctx: Unshared::new(DecoderContext::new().unwrap()), } } } +impl Lz4Decoder { + pub fn new() -> Self { + Self::default() + } +} + impl Decode for Lz4Decoder { fn reinit(&mut self) -> Result<()> { unsafe { LZ4F_resetDecompressionContext(self.ctx.get_mut().ctx) }; diff --git a/src/codec/lz4/encoder.rs b/crates/compression-codecs/src/lz4/encoder.rs similarity index 95% rename from src/codec/lz4/encoder.rs rename to crates/compression-codecs/src/lz4/encoder.rs index 42c6b55a..d306e027 100644 --- a/src/codec/lz4/encoder.rs +++ b/crates/compression-codecs/src/lz4/encoder.rs @@ -1,13 +1,11 @@ -use std::io::{self, Result}; - +use crate::{lz4::params::EncoderParams, Encode}; +use compression_core::{unshared::Unshared, util::PartialBuffer}; use lz4::liblz4::{ - check_error, BlockChecksum, BlockMode, BlockSize, ContentChecksum, FrameType, - LZ4FCompressionContext, LZ4FFrameInfo, LZ4FPreferences, LZ4F_compressBegin, LZ4F_compressBound, + check_error, LZ4FCompressionContext, LZ4FPreferences, LZ4F_compressBegin, LZ4F_compressBound, LZ4F_compressEnd, LZ4F_compressUpdate, LZ4F_createCompressionContext, LZ4F_flush, LZ4F_freeCompressionContext, LZ4F_VERSION, }; - -use crate::{codec::Encode, lz4::EncoderParams, unshared::Unshared, util::PartialBuffer}; +use std::io::{self, Result}; // https://github.com/lz4/lz4/blob/9d53d8bb6c4120345a0966e5d8b16d7def1f32c5/lib/lz4frame.h#L281 const LZ4F_HEADER_SIZE_MAX: usize = 19; @@ -68,7 +66,8 @@ impl Drop for EncoderContext { } impl Lz4Encoder { - pub(crate) fn new(preferences: LZ4FPreferences) -> Self { + pub fn new(params: EncoderParams) -> Self { + let preferences = LZ4FPreferences::from(params); let block_size = preferences.frame_info.block_size_id.get_size(); let block_buffer_size = min_dst_size(block_size, &preferences); @@ -85,7 +84,7 @@ impl Lz4Encoder { } } - pub(crate) fn buffer_size(&self) -> usize { + pub fn buffer_size(&self) -> usize { self.block_buffer_size } diff --git a/crates/compression-codecs/src/lz4/mod.rs b/crates/compression-codecs/src/lz4/mod.rs new file mode 100644 index 00000000..666c2d55 --- /dev/null +++ b/crates/compression-codecs/src/lz4/mod.rs @@ -0,0 +1,5 @@ +mod decoder; +mod encoder; +pub mod params; + +pub use self::{decoder::Lz4Decoder, encoder::Lz4Encoder}; diff --git a/src/lz4.rs b/crates/compression-codecs/src/lz4/params.rs similarity index 72% rename from src/lz4.rs rename to crates/compression-codecs/src/lz4/params.rs index 8c15b379..8e4dbfb7 100644 --- a/src/lz4.rs +++ b/crates/compression-codecs/src/lz4/params.rs @@ -1,5 +1,8 @@ //! This module contains lz4-specific types for async-compression. +use std::convert::TryInto; + +use compression_core::Level; pub use lz4::liblz4::BlockSize; use lz4::{ liblz4::{BlockChecksum, FrameType, LZ4FFrameInfo, LZ4FPreferences}, @@ -15,10 +18,10 @@ use lz4::{ /// # Examples /// /// ``` -/// use async_compression::lz4; +/// use compression_codecs::lz4; /// -/// let params = lz4::EncoderParams::default() -/// .block_size(lz4::BlockSize::Max1MB) +/// let params = lz4::params::EncoderParams::default() +/// .block_size(lz4::params::BlockSize::Max1MB) /// .content_checksum(true); /// ``` #[derive(Clone, Debug, Default)] @@ -26,9 +29,15 @@ pub struct EncoderParams { block_size: Option, block_checksum: Option, content_checksum: Option, + level: Level, } impl EncoderParams { + pub fn level(mut self, level: Level) -> Self { + self.level = level; + self + } + /// Sets input block size. pub fn block_size(mut self, block_size: BlockSize) -> Self { self.block_size = Some(block_size); @@ -54,17 +63,25 @@ impl EncoderParams { }); self } +} - pub(crate) fn as_lz4(&self) -> LZ4FPreferences { - let block_size_id = self.block_size.clone().unwrap_or(BlockSize::Default); - let content_checksum_flag = self +impl From for LZ4FPreferences { + fn from(value: EncoderParams) -> Self { + let block_size_id = value.block_size.clone().unwrap_or(BlockSize::Default); + let content_checksum_flag = value .content_checksum .clone() .unwrap_or(ContentChecksum::NoChecksum); - let block_checksum_flag = self + let block_checksum_flag = value .block_checksum .clone() .unwrap_or(BlockChecksum::NoBlockChecksum); + let compression_level = match value.level { + Level::Fastest => 0, + Level::Best => 12, + Level::Precise(quality) => quality.try_into().unwrap_or(0).clamp(0, 12), + _ => 0, + }; LZ4FPreferences { frame_info: LZ4FFrameInfo { @@ -76,7 +93,7 @@ impl EncoderParams { dict_id: 0, block_checksum_flag, }, - compression_level: 0, + compression_level, auto_flush: 0, favor_dec_speed: 0, reserved: [0; 3], diff --git a/src/codec/lzma/decoder.rs b/crates/compression-codecs/src/lzma/decoder.rs similarity index 64% rename from src/codec/lzma/decoder.rs rename to crates/compression-codecs/src/lzma/decoder.rs index 2a62b2f6..fd953f45 100644 --- a/src/codec/lzma/decoder.rs +++ b/crates/compression-codecs/src/lzma/decoder.rs @@ -1,22 +1,35 @@ -use crate::{codec::Decode, util::PartialBuffer}; - -use std::io::Result; +use crate::{Decode, Xz2Decoder}; +use compression_core::util::PartialBuffer; +use std::{convert::TryInto, io::Result}; +/// Lzma decoding stream #[derive(Debug)] pub struct LzmaDecoder { - inner: crate::codec::Xz2Decoder, + inner: Xz2Decoder, } -impl LzmaDecoder { - pub fn new() -> Self { +impl From for LzmaDecoder { + fn from(inner: Xz2Decoder) -> Self { + Self { inner } + } +} + +impl Default for LzmaDecoder { + fn default() -> Self { Self { - inner: crate::codec::Xz2Decoder::new(u64::MAX), + inner: Xz2Decoder::new(usize::MAX.try_into().unwrap()), } } +} + +impl LzmaDecoder { + pub fn new() -> Self { + Self::default() + } pub fn with_memlimit(memlimit: u64) -> Self { Self { - inner: crate::codec::Xz2Decoder::new(memlimit), + inner: Xz2Decoder::new(memlimit), } } } diff --git a/src/codec/lzma/encoder.rs b/crates/compression-codecs/src/lzma/encoder.rs similarity index 67% rename from src/codec/lzma/encoder.rs rename to crates/compression-codecs/src/lzma/encoder.rs index 5441ec23..b83a6b65 100644 --- a/src/codec/lzma/encoder.rs +++ b/crates/compression-codecs/src/lzma/encoder.rs @@ -1,20 +1,27 @@ -use crate::{codec::Encode, util::PartialBuffer}; - +use crate::{Encode, Xz2Encoder, Xz2FileFormat}; +use compression_core::{util::PartialBuffer, Level}; use std::io::Result; +/// Lzma encoding stream #[derive(Debug)] pub struct LzmaEncoder { - inner: crate::codec::Xz2Encoder, + inner: Xz2Encoder, } impl LzmaEncoder { - pub fn new(level: u32) -> Self { + pub fn new(level: Level) -> Self { Self { - inner: crate::codec::Xz2Encoder::new(crate::codec::Xz2FileFormat::Lzma, level), + inner: Xz2Encoder::new(Xz2FileFormat::Lzma, level), } } } +impl From for LzmaEncoder { + fn from(inner: Xz2Encoder) -> Self { + Self { inner } + } +} + impl Encode for LzmaEncoder { fn encode( &mut self, diff --git a/crates/compression-codecs/src/lzma/mod.rs b/crates/compression-codecs/src/lzma/mod.rs new file mode 100644 index 00000000..f2b6f985 --- /dev/null +++ b/crates/compression-codecs/src/lzma/mod.rs @@ -0,0 +1,6 @@ +mod decoder; +mod encoder; + +pub mod params; + +pub use self::{decoder::LzmaDecoder, encoder::LzmaEncoder}; diff --git a/crates/compression-codecs/src/lzma/params.rs b/crates/compression-codecs/src/lzma/params.rs new file mode 100644 index 00000000..094ff014 --- /dev/null +++ b/crates/compression-codecs/src/lzma/params.rs @@ -0,0 +1,416 @@ +//! Clonable structs to build the non-clonables in liblzma + +use std::convert::TryFrom; +#[cfg(feature = "xz-parallel")] +use std::num::NonZeroU32; + +/// Used to control how the lzma stream is created +#[derive(Clone)] +pub enum LzmaEncoderParams { + Easy { + preset: u32, + check: liblzma::stream::Check, + }, + Lzma { + options: LzmaOptions, + }, + Raw { + filters: LzmaFilters, + }, + Stream { + filters: LzmaFilters, + check: liblzma::stream::Check, + }, + + #[cfg(feature = "xz-parallel")] + MultiThread { + builder: MtStreamBuilder, + }, +} + +impl TryFrom<&LzmaEncoderParams> for liblzma::stream::Stream { + type Error = liblzma::stream::Error; + + fn try_from(value: &LzmaEncoderParams) -> Result { + let stream = match value { + LzmaEncoderParams::Easy { preset, check } => Self::new_easy_encoder(*preset, *check)?, + LzmaEncoderParams::Lzma { options } => { + let options = liblzma::stream::LzmaOptions::try_from(options)?; + Self::new_lzma_encoder(&options)? + } + LzmaEncoderParams::Raw { filters } => { + let filters = liblzma::stream::Filters::try_from(filters)?; + Self::new_raw_encoder(&filters)? + } + LzmaEncoderParams::Stream { filters, check } => { + let filters = liblzma::stream::Filters::try_from(filters)?; + Self::new_stream_encoder(&filters, *check)? + } + + #[cfg(feature = "xz-parallel")] + LzmaEncoderParams::MultiThread { builder } => { + let builder = liblzma::stream::MtStreamBuilder::try_from(builder)?; + builder.encoder()? + } + }; + + Ok(stream) + } +} + +/// Directly translate to how the stream is constructed +#[derive(Clone, Debug)] +pub enum LzmaDecoderParams { + Auto { + mem_limit: u64, + flags: u32, + }, + Lzip { + mem_limit: u64, + flags: u32, + }, + Lzma { + mem_limit: u64, + }, + Raw { + filters: LzmaFilters, + }, + Stream { + mem_limit: u64, + flags: u32, + }, + + #[cfg(feature = "xz-parallel")] + MultiThread { + builder: MtStreamBuilder, + }, +} + +impl TryFrom<&LzmaDecoderParams> for liblzma::stream::Stream { + type Error = liblzma::stream::Error; + + fn try_from(value: &LzmaDecoderParams) -> Result { + let stream = match value { + LzmaDecoderParams::Auto { mem_limit, flags } => { + Self::new_auto_decoder(*mem_limit, *flags)? + } + LzmaDecoderParams::Lzip { mem_limit, flags } => { + Self::new_lzip_decoder(*mem_limit, *flags)? + } + LzmaDecoderParams::Lzma { mem_limit } => Self::new_lzma_decoder(*mem_limit)?, + LzmaDecoderParams::Stream { mem_limit, flags } => { + Self::new_stream_decoder(*mem_limit, *flags)? + } + LzmaDecoderParams::Raw { filters } => { + let filters = liblzma::stream::Filters::try_from(filters)?; + Self::new_raw_decoder(&filters)? + } + #[cfg(feature = "xz-parallel")] + LzmaDecoderParams::MultiThread { builder } => { + let builder = liblzma::stream::MtStreamBuilder::try_from(builder)?; + builder.decoder()? + } + }; + + Ok(stream) + } +} + +/// Clonable liblzma::Filters +#[derive(Default, Clone, Debug)] +pub struct LzmaFilters { + filters: Vec, +} + +impl LzmaFilters { + /// Add `LzmaFilter` to the collection + pub fn add_filter(mut self, filter: LzmaFilter) -> Self { + self.filters.push(filter); + self + } +} + +/// An individual filter directly corresponding to liblzma Filters method calls +#[derive(Clone, Debug)] +pub enum LzmaFilter { + Arm(Option>), + Arm64(Option>), + ArmThumb(Option>), + Delta(Option>), + Ia64(Option>), + Lzma1(LzmaOptions), + Lzma1Properties(Vec), + Lzma2(LzmaOptions), + Lzma2Properties(Vec), + PowerPc(Option>), + Sparc(Option>), + X86(Option>), +} + +impl TryFrom<&LzmaFilters> for liblzma::stream::Filters { + type Error = liblzma::stream::Error; + + fn try_from(value: &LzmaFilters) -> Result { + let mut filters = liblzma::stream::Filters::new(); + for f in value.filters.iter() { + match f { + LzmaFilter::Arm(Some(p)) => filters.arm_properties(p)?, + LzmaFilter::Arm(None) => filters.arm(), + LzmaFilter::Arm64(Some(p)) => filters.arm64_properties(p)?, + LzmaFilter::Arm64(None) => filters.arm64(), + LzmaFilter::ArmThumb(Some(p)) => filters.arm_thumb_properties(p)?, + LzmaFilter::ArmThumb(None) => filters.arm_thumb(), + LzmaFilter::Delta(Some(p)) => filters.delta_properties(p)?, + LzmaFilter::Delta(None) => filters.delta(), + LzmaFilter::Ia64(Some(p)) => filters.ia64_properties(p)?, + LzmaFilter::Ia64(None) => filters.ia64(), + LzmaFilter::Lzma1(opts) => { + let opts = liblzma::stream::LzmaOptions::try_from(opts)?; + filters.lzma1(&opts) + } + LzmaFilter::Lzma1Properties(p) => filters.lzma1_properties(p)?, + LzmaFilter::Lzma2(opts) => { + let opts = liblzma::stream::LzmaOptions::try_from(opts)?; + filters.lzma2(&opts) + } + LzmaFilter::Lzma2Properties(p) => filters.lzma2_properties(p)?, + LzmaFilter::PowerPc(Some(p)) => filters.powerpc_properties(p)?, + LzmaFilter::PowerPc(None) => filters.powerpc(), + LzmaFilter::Sparc(Some(p)) => filters.sparc_properties(p)?, + LzmaFilter::Sparc(None) => filters.sparc(), + LzmaFilter::X86(Some(p)) => filters.x86_properties(p)?, + LzmaFilter::X86(None) => filters.x86(), + }; + } + + Ok(filters) + } +} + +/// A builder for liblzma::LzmaOptions, so that it can be cloned +#[derive(Default, Clone)] +pub struct LzmaOptions { + preset: Option, + depth: Option, + dict_size: Option, + literal_context_bits: Option, + literal_position_bits: Option, + match_finder: Option, + mode: Option, + nice_len: Option, + position_bits: Option, +} + +impl std::fmt::Debug for LzmaOptions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let match_finder = self.match_finder.map(|m| match m { + liblzma::stream::MatchFinder::HashChain3 => "HashChain3", + liblzma::stream::MatchFinder::HashChain4 => "HashChain4", + liblzma::stream::MatchFinder::BinaryTree2 => "BTree2", + liblzma::stream::MatchFinder::BinaryTree3 => "BTree3", + liblzma::stream::MatchFinder::BinaryTree4 => "BTree4", + }); + + let mode = self.mode.map(|m| match m { + liblzma::stream::Mode::Fast => "Fast", + liblzma::stream::Mode::Normal => "Normal", + }); + + f.debug_struct("LzmaOptions") + .field("preset", &self.preset) + .field("depth", &self.depth) + .field("dict_size", &self.dict_size) + .field("literal_context_bits", &self.literal_context_bits) + .field("literal_position_bits", &self.literal_position_bits) + .field("match_finder", &match_finder) + .field("mode", &mode) + .field("nice_len", &self.nice_len) + .field("position_bits", &self.position_bits) + .finish() + } +} + +impl LzmaOptions { + pub fn preset(mut self, value: u32) -> Self { + self.preset = Some(value); + self + } + pub fn depth(mut self, value: u32) -> Self { + self.depth = Some(value); + self + } + pub fn dict_size(mut self, value: u32) -> Self { + self.dict_size = Some(value); + self + } + pub fn literal_context_bits(mut self, value: u32) -> Self { + self.literal_context_bits = Some(value); + self + } + pub fn literal_position_bits(mut self, value: u32) -> Self { + self.literal_position_bits = Some(value); + self + } + pub fn match_finder(mut self, value: liblzma::stream::MatchFinder) -> Self { + self.match_finder = Some(value); + self + } + pub fn mode(mut self, value: liblzma::stream::Mode) -> Self { + self.mode = Some(value); + self + } + pub fn nice_len(mut self, value: u32) -> Self { + self.nice_len = Some(value); + self + } + pub fn position_bits(mut self, value: u32) -> Self { + self.position_bits = Some(value); + self + } +} + +impl TryFrom<&LzmaOptions> for liblzma::stream::LzmaOptions { + type Error = liblzma::stream::Error; + + fn try_from(value: &LzmaOptions) -> Result { + let mut s = match value.preset { + Some(preset) => liblzma::stream::LzmaOptions::new_preset(preset)?, + None => liblzma::stream::LzmaOptions::new(), + }; + if let Some(depth) = value.depth { + s.depth(depth); + } + if let Some(dict_size) = value.dict_size { + s.dict_size(dict_size); + } + if let Some(bits) = value.literal_context_bits { + s.literal_context_bits(bits); + } + if let Some(bits) = value.literal_position_bits { + s.literal_position_bits(bits); + } + if let Some(mf) = value.match_finder { + s.match_finder(mf); + } + if let Some(mode) = value.mode { + s.mode(mode); + } + if let Some(len) = value.nice_len { + s.nice_len(len); + } + + if let Some(bits) = value.position_bits { + s.position_bits(bits); + } + + Ok(s) + } +} + +#[cfg(feature = "xz-parallel")] +#[derive(Default, Clone)] +/// Used to build a clonable mt stream builder +pub struct MtStreamBuilder { + block_size: Option, + preset: Option, + check: Option, + filters: Option, + mem_limit_stop: Option, + mem_limit_threading: Option, + threads: Option, + timeout_ms: Option, +} + +#[cfg(feature = "xz-parallel")] +impl std::fmt::Debug for MtStreamBuilder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let check = self.check.map(|s| match s { + liblzma::stream::Check::None => "None", + liblzma::stream::Check::Crc32 => "Crc32", + liblzma::stream::Check::Crc64 => "Crc64", + liblzma::stream::Check::Sha256 => "Sha256", + }); + f.debug_struct("MtStreamBuilder") + .field("block_size", &self.block_size) + .field("preset", &self.preset) + .field("check", &check) + .field("filters", &self.filters) + .field("mem_limit_stop", &self.mem_limit_stop) + .field("mem_limit_threading", &self.mem_limit_threading) + .field("threads", &self.threads) + .field("timeout_ms", &self.timeout_ms) + .finish() + } +} + +#[cfg(feature = "xz-parallel")] +impl MtStreamBuilder { + pub fn block_size(&mut self, block_size: u64) -> &mut Self { + self.block_size = Some(block_size); + self + } + pub fn preset(&mut self, preset: u32) -> &mut Self { + self.preset = Some(preset); + self + } + pub fn check(&mut self, check: liblzma::stream::Check) -> &mut Self { + self.check = Some(check); + self + } + + pub fn filters(&mut self, filters: LzmaFilters) -> &mut Self { + self.filters = Some(filters); + self + } + pub fn mem_limit_stop(&mut self, mem_limit_stop: u64) -> &mut Self { + self.mem_limit_stop = Some(mem_limit_stop); + self + } + pub fn mem_limit_threading(&mut self, mem_limit_threading: u64) -> &mut Self { + self.mem_limit_threading = Some(mem_limit_threading); + self + } + pub fn threads(&mut self, threads: NonZeroU32) -> &mut Self { + self.threads = Some(threads); + self + } + pub fn timeout_ms(&mut self, timeout_ms: u32) -> &mut Self { + self.timeout_ms = Some(timeout_ms); + self + } +} + +#[cfg(feature = "xz-parallel")] +impl TryFrom<&MtStreamBuilder> for liblzma::stream::MtStreamBuilder { + type Error = liblzma::stream::Error; + + fn try_from(value: &MtStreamBuilder) -> Result { + let mut mt = liblzma::stream::MtStreamBuilder::new(); + if let Some(block_size) = value.block_size { + mt.block_size(block_size); + } + if let Some(preset) = value.preset { + mt.preset(preset); + } + if let Some(check) = value.check { + mt.check(check); + } + if let Some(filters) = &value.filters { + let filters = liblzma::stream::Filters::try_from(filters)?; + mt.filters(filters); + } + if let Some(memlimit) = value.mem_limit_stop { + mt.memlimit_stop(memlimit); + } + if let Some(memlimit) = value.mem_limit_threading { + mt.memlimit_threading(memlimit); + } + if let Some(threads) = value.threads { + mt.threads(threads.get()); + } + if let Some(timeout) = value.timeout_ms { + mt.timeout_ms(timeout); + } + Ok(mt) + } +} diff --git a/src/codec/xz/decoder.rs b/crates/compression-codecs/src/xz/decoder.rs similarity index 82% rename from src/codec/xz/decoder.rs rename to crates/compression-codecs/src/xz/decoder.rs index 95d552ef..48b6de67 100644 --- a/src/codec/xz/decoder.rs +++ b/crates/compression-codecs/src/xz/decoder.rs @@ -1,24 +1,34 @@ -use crate::{codec::Decode, util::PartialBuffer}; - -use std::io::{Error, ErrorKind, Result}; +use crate::{Decode, Xz2Decoder}; +use compression_core::util::PartialBuffer; +use std::{ + convert::TryInto, + io::{Error, ErrorKind, Result}, +}; +/// Xz decoding stream #[derive(Debug)] pub struct XzDecoder { - inner: crate::codec::Xz2Decoder, + inner: Xz2Decoder, skip_padding: Option, } -impl XzDecoder { - pub fn new() -> Self { +impl Default for XzDecoder { + fn default() -> Self { Self { - inner: crate::codec::Xz2Decoder::new(u64::MAX), + inner: Xz2Decoder::new(usize::MAX.try_into().unwrap()), skip_padding: None, } } +} + +impl XzDecoder { + pub fn new() -> Self { + Self::default() + } pub fn with_memlimit(memlimit: u64) -> Self { Self { - inner: crate::codec::Xz2Decoder::new(memlimit), + inner: Xz2Decoder::new(memlimit), skip_padding: None, } } @@ -26,7 +36,7 @@ impl XzDecoder { #[cfg(feature = "xz-parallel")] pub fn parallel(threads: std::num::NonZeroU32, memlimit: u64) -> Self { Self { - inner: crate::codec::Xz2Decoder::parallel(threads, memlimit), + inner: Xz2Decoder::parallel(threads, memlimit), skip_padding: None, } } diff --git a/src/codec/xz/encoder.rs b/crates/compression-codecs/src/xz/encoder.rs similarity index 66% rename from src/codec/xz/encoder.rs rename to crates/compression-codecs/src/xz/encoder.rs index 22595ca2..5af99ee9 100644 --- a/src/codec/xz/encoder.rs +++ b/crates/compression-codecs/src/xz/encoder.rs @@ -1,23 +1,24 @@ -use crate::{codec::Encode, util::PartialBuffer}; - +use crate::{Encode, Xz2Encoder, Xz2FileFormat}; +use compression_core::{util::PartialBuffer, Level}; use std::io::Result; +/// Xz encoding stream #[derive(Debug)] pub struct XzEncoder { - inner: crate::codec::Xz2Encoder, + inner: Xz2Encoder, } impl XzEncoder { - pub fn new(level: u32) -> Self { + pub fn new(level: Level) -> Self { Self { - inner: crate::codec::Xz2Encoder::new(crate::codec::Xz2FileFormat::Xz, level), + inner: Xz2Encoder::new(Xz2FileFormat::Xz, level), } } #[cfg(feature = "xz-parallel")] - pub fn parallel(threads: std::num::NonZeroU32, level: u32) -> Self { + pub fn parallel(threads: std::num::NonZeroU32, level: Level) -> Self { Self { - inner: crate::codec::Xz2Encoder::xz_parallel(level, threads), + inner: Xz2Encoder::xz_parallel(level, threads), } } } diff --git a/crates/compression-codecs/src/xz/mod.rs b/crates/compression-codecs/src/xz/mod.rs new file mode 100644 index 00000000..11062c07 --- /dev/null +++ b/crates/compression-codecs/src/xz/mod.rs @@ -0,0 +1,4 @@ +mod decoder; +mod encoder; + +pub use self::{decoder::XzDecoder, encoder::XzEncoder}; diff --git a/src/codec/xz2/decoder.rs b/crates/compression-codecs/src/xz2/decoder.rs similarity index 61% rename from src/codec/xz2/decoder.rs rename to crates/compression-codecs/src/xz2/decoder.rs index 80eed14a..9c9cac44 100644 --- a/src/codec/xz2/decoder.rs +++ b/crates/compression-codecs/src/xz2/decoder.rs @@ -1,13 +1,12 @@ -use std::{fmt, io}; - +use crate::{lzma::params::LzmaDecoderParams, Decode}; +use compression_core::util::PartialBuffer; use liblzma::stream::{Action, Status, Stream}; +use std::{convert::TryFrom, fmt, io}; -use crate::{codec::Decode, util::PartialBuffer}; - +/// Xz2 decoding stream pub struct Xz2Decoder { stream: Stream, - #[cfg(feature = "xz-parallel")] - threads: Option, + params: LzmaDecoderParams, } impl fmt::Debug for Xz2Decoder { @@ -16,44 +15,44 @@ impl fmt::Debug for Xz2Decoder { } } +impl TryFrom for Xz2Decoder { + type Error = liblzma::stream::Error; + + fn try_from(params: LzmaDecoderParams) -> Result { + let stream = Stream::try_from(¶ms)?; + Ok(Self { stream, params }) + } +} + impl Xz2Decoder { pub fn new(mem_limit: u64) -> Self { - Self { - stream: Stream::new_auto_decoder(mem_limit, 0).unwrap(), - #[cfg(feature = "xz-parallel")] - threads: None, - } + let params = LzmaDecoderParams::Auto { + mem_limit, + flags: 0, + }; + Self::try_from(params).unwrap() } #[cfg(feature = "xz-parallel")] pub fn parallel(threads: std::num::NonZeroU32, mem_limit: u64) -> Self { - Self { - stream: liblzma::stream::MtStreamBuilder::new() - .threads(threads.get()) - .timeout_ms(300) - .memlimit_stop(mem_limit) - .decoder() - .unwrap(), - threads: Some(threads), - } + use crate::lzma::params::MtStreamBuilder; + + let mut builder = MtStreamBuilder::default(); + + builder + .threads(threads) + .timeout_ms(300) + .mem_limit_stop(mem_limit); + + let params = LzmaDecoderParams::MultiThread { builder }; + + Self::try_from(params).unwrap() } } impl Decode for Xz2Decoder { fn reinit(&mut self) -> io::Result<()> { - #[cfg(feature = "xz-parallel")] - { - *self = match self.threads { - Some(threads) => Self::parallel(threads, self.stream.memlimit()), - None => Self::new(self.stream.memlimit()), - }; - } - - #[cfg(not(feature = "xz-parallel"))] - { - *self = Self::new(self.stream.memlimit()); - } - + *self = Self::try_from(self.params.clone())?; Ok(()) } @@ -108,3 +107,20 @@ impl Decode for Xz2Decoder { } } } + +#[cfg(test)] +mod tests { + use std::convert::TryFrom; + + use crate::{ + lzma::params::{LzmaDecoderParams, LzmaFilter, LzmaFilters, LzmaOptions}, + Xz2Decoder, + }; + + #[test] + fn test_lzma_decoder_from_params() { + let filters = LzmaFilters::default().add_filter(LzmaFilter::Lzma2(LzmaOptions::default())); + let params = LzmaDecoderParams::Raw { filters }; + Xz2Decoder::try_from(params).unwrap(); + } +} diff --git a/src/codec/xz2/encoder.rs b/crates/compression-codecs/src/xz2/encoder.rs similarity index 57% rename from src/codec/xz2/encoder.rs rename to crates/compression-codecs/src/xz2/encoder.rs index 0c3afa89..4d58a9ea 100644 --- a/src/codec/xz2/encoder.rs +++ b/crates/compression-codecs/src/xz2/encoder.rs @@ -1,16 +1,19 @@ -use std::{fmt, io}; - -use liblzma::stream::{Action, Check, LzmaOptions, Status, Stream}; +use compression_core::{util::PartialBuffer, Level}; +use liblzma::stream::{Action, Check, Status, Stream}; +use std::{ + convert::{TryFrom, TryInto}, + fmt, io, +}; use crate::{ - codec::{Encode, Xz2FileFormat}, - util::PartialBuffer, + lzma::params::{LzmaEncoderParams, LzmaOptions}, + Encode, Xz2FileFormat, }; +/// Xz2 encoding stream pub struct Xz2Encoder { stream: Stream, - #[cfg(feature = "xz-parallel")] - threads: Option, + params: LzmaEncoderParams, } impl fmt::Debug for Xz2Encoder { @@ -19,36 +22,57 @@ impl fmt::Debug for Xz2Encoder { } } +impl TryFrom for Xz2Encoder { + type Error = liblzma::stream::Error; + + fn try_from(params: LzmaEncoderParams) -> Result { + let stream = Stream::try_from(¶ms)?; + Ok(Self { + stream, + params: params.clone(), + }) + } +} + +fn xz2_level(level: Level) -> u32 { + match level { + Level::Fastest => 0, + Level::Best => 9, + Level::Precise(quality) => quality.try_into().unwrap_or(0).clamp(0, 9), + _ => 5, + } +} + impl Xz2Encoder { - pub fn new(format: Xz2FileFormat, level: u32) -> Self { - let stream = match format { - Xz2FileFormat::Xz => Stream::new_easy_encoder(level, Check::Crc64).unwrap(), + pub fn new(format: Xz2FileFormat, level: Level) -> Self { + let preset = xz2_level(level); + let params = match format { + Xz2FileFormat::Xz => LzmaEncoderParams::Easy { + preset, + check: Check::Crc64, + }, Xz2FileFormat::Lzma => { - Stream::new_lzma_encoder(&LzmaOptions::new_preset(level).unwrap()).unwrap() + let options = LzmaOptions::default().preset(preset); + LzmaEncoderParams::Lzma { options } } }; - Self { - stream, - #[cfg(feature = "xz-parallel")] - threads: None, - } + Self::try_from(params).unwrap() } #[cfg(feature = "xz-parallel")] - pub fn xz_parallel(level: u32, threads: std::num::NonZeroU32) -> Self { - let stream = liblzma::stream::MtStreamBuilder::new() - .threads(threads.get()) - .timeout_ms(300) - .preset(level) - .check(Check::Crc64) - .encoder() - .unwrap(); + pub fn xz_parallel(level: Level, threads: std::num::NonZeroU32) -> Self { + use crate::lzma::params::MtStreamBuilder; - Self { - stream, - threads: Some(threads), - } + let preset = xz2_level(level); + let mut builder = MtStreamBuilder::default(); + builder + .threads(threads) + .timeout_ms(300) + .preset(preset) + .check(Check::Crc64); + let params = LzmaEncoderParams::MultiThread { builder }; + Self::try_from(params).unwrap() } } @@ -81,16 +105,13 @@ impl Encode for Xz2Encoder { ) -> io::Result { let previous_out = self.stream.total_out() as usize; - // Multi-threaded streams don't support SyncFlush, use FullFlush instead - #[cfg(feature = "xz-parallel")] - let action = match self.threads { - Some(_) => Action::FullFlush, - None => Action::SyncFlush, + let action = match &self.params { + // Multi-threaded streams don't support SyncFlush, use FullFlush instead + #[cfg(feature = "xz-parallel")] + LzmaEncoderParams::MultiThread { builder: _ } => Action::FullFlush, + _ => Action::SyncFlush, }; - #[cfg(not(feature = "xz-parallel"))] - let action = Action::SyncFlush; - let status = self.stream.process(&[], output.unwritten_mut(), action)?; output.advance(self.stream.total_out() as usize - previous_out); diff --git a/src/codec/xz2/mod.rs b/crates/compression-codecs/src/xz2/mod.rs similarity index 52% rename from src/codec/xz2/mod.rs rename to crates/compression-codecs/src/xz2/mod.rs index b881208f..e105b10a 100644 --- a/src/codec/xz2/mod.rs +++ b/crates/compression-codecs/src/xz2/mod.rs @@ -6,4 +6,4 @@ pub enum Xz2FileFormat { Lzma, } -pub(crate) use self::{decoder::Xz2Decoder, encoder::Xz2Encoder}; +pub use self::{decoder::Xz2Decoder, encoder::Xz2Encoder}; diff --git a/src/codec/zlib/decoder.rs b/crates/compression-codecs/src/zlib/decoder.rs similarity index 72% rename from src/codec/zlib/decoder.rs rename to crates/compression-codecs/src/zlib/decoder.rs index 8c159a91..bd72fcc4 100644 --- a/src/codec/zlib/decoder.rs +++ b/crates/compression-codecs/src/zlib/decoder.rs @@ -1,20 +1,25 @@ -use crate::util::PartialBuffer; +use crate::{Decode, FlateDecoder}; +use compression_core::util::PartialBuffer; use std::io::Result; #[derive(Debug)] pub struct ZlibDecoder { - inner: crate::codec::FlateDecoder, + inner: FlateDecoder, } - -impl ZlibDecoder { - pub(crate) fn new() -> Self { +impl Default for ZlibDecoder { + fn default() -> Self { Self { - inner: crate::codec::FlateDecoder::new(true), + inner: FlateDecoder::new(true), } } } +impl ZlibDecoder { + pub fn new() -> Self { + Self::default() + } +} -impl crate::codec::Decode for ZlibDecoder { +impl Decode for ZlibDecoder { fn reinit(&mut self) -> Result<()> { self.inner.reinit()?; Ok(()) diff --git a/src/codec/zlib/encoder.rs b/crates/compression-codecs/src/zlib/encoder.rs similarity index 71% rename from src/codec/zlib/encoder.rs rename to crates/compression-codecs/src/zlib/encoder.rs index 934516c9..bf567ea4 100644 --- a/src/codec/zlib/encoder.rs +++ b/crates/compression-codecs/src/zlib/encoder.rs @@ -1,21 +1,20 @@ -use crate::{codec::Encode, util::PartialBuffer}; +use crate::{flate::params::FlateEncoderParams, Encode, FlateEncoder}; +use compression_core::util::PartialBuffer; use std::io::Result; -use flate2::Compression; - #[derive(Debug)] pub struct ZlibEncoder { - inner: crate::codec::FlateEncoder, + inner: FlateEncoder, } impl ZlibEncoder { - pub(crate) fn new(level: Compression) -> Self { + pub fn new(level: FlateEncoderParams) -> Self { Self { - inner: crate::codec::FlateEncoder::new(level, true), + inner: FlateEncoder::new(level, true), } } - pub(crate) fn get_ref(&self) -> &crate::codec::FlateEncoder { + pub fn get_ref(&self) -> &FlateEncoder { &self.inner } } diff --git a/crates/compression-codecs/src/zlib/mod.rs b/crates/compression-codecs/src/zlib/mod.rs new file mode 100644 index 00000000..137a4314 --- /dev/null +++ b/crates/compression-codecs/src/zlib/mod.rs @@ -0,0 +1,4 @@ +mod decoder; +mod encoder; + +pub use self::{decoder::ZlibDecoder, encoder::ZlibEncoder}; diff --git a/src/codec/zstd/decoder.rs b/crates/compression-codecs/src/zstd/decoder.rs similarity index 81% rename from src/codec/zstd/decoder.rs rename to crates/compression-codecs/src/zstd/decoder.rs index d1a92f78..a35161be 100644 --- a/src/codec/zstd/decoder.rs +++ b/crates/compression-codecs/src/zstd/decoder.rs @@ -1,22 +1,30 @@ +use crate::zstd::params::DParameter; +use crate::Decode; +use compression_core::unshared::Unshared; +use compression_core::util::PartialBuffer; +use libzstd::stream::raw::{Decoder, Operation}; use std::io; use std::io::Result; -use crate::{codec::Decode, unshared::Unshared, util::PartialBuffer}; -use libzstd::stream::raw::{Decoder, Operation}; - #[derive(Debug)] pub struct ZstdDecoder { decoder: Unshared>, } -impl ZstdDecoder { - pub(crate) fn new() -> Self { +impl Default for ZstdDecoder { + fn default() -> Self { Self { decoder: Unshared::new(Decoder::new().unwrap()), } } +} + +impl ZstdDecoder { + pub fn new() -> Self { + Self::default() + } - pub(crate) fn new_with_params(params: &[crate::zstd::DParameter]) -> Self { + pub fn new_with_params(params: &[DParameter]) -> Self { let mut decoder = Decoder::new().unwrap(); for param in params { decoder.set_parameter(param.as_zstd()).unwrap(); @@ -26,8 +34,8 @@ impl ZstdDecoder { } } - pub(crate) fn new_with_dict(dictionary: &[u8]) -> io::Result { - let mut decoder = Decoder::with_dictionary(dictionary)?; + pub fn new_with_dict(dictionary: &[u8]) -> io::Result { + let decoder = Decoder::with_dictionary(dictionary)?; Ok(Self { decoder: Unshared::new(decoder), }) diff --git a/src/codec/zstd/encoder.rs b/crates/compression-codecs/src/zstd/encoder.rs similarity index 80% rename from src/codec/zstd/encoder.rs rename to crates/compression-codecs/src/zstd/encoder.rs index ca3f5317..f7695d08 100644 --- a/src/codec/zstd/encoder.rs +++ b/crates/compression-codecs/src/zstd/encoder.rs @@ -1,5 +1,8 @@ -use crate::{codec::Encode, unshared::Unshared, util::PartialBuffer}; -use libzstd::stream::raw::{CParameter, Encoder, Operation}; +use crate::zstd::params::CParameter; +use crate::Encode; +use compression_core::unshared::Unshared; +use compression_core::util::PartialBuffer; +use libzstd::stream::raw::{Encoder, Operation}; use std::io; use std::io::Result; @@ -9,13 +12,13 @@ pub struct ZstdEncoder { } impl ZstdEncoder { - pub(crate) fn new(level: i32) -> Self { + pub fn new(level: i32) -> Self { Self { encoder: Unshared::new(Encoder::new(level).unwrap()), } } - pub(crate) fn new_with_params(level: i32, params: &[crate::zstd::CParameter]) -> Self { + pub fn new_with_params(level: i32, params: &[CParameter]) -> Self { let mut encoder = Encoder::new(level).unwrap(); for param in params { encoder.set_parameter(param.as_zstd()).unwrap(); @@ -25,8 +28,8 @@ impl ZstdEncoder { } } - pub(crate) fn new_with_dict(level: i32, dictionary: &[u8]) -> io::Result { - let mut encoder = Encoder::with_dictionary(level, dictionary)?; + pub fn new_with_dict(level: i32, dictionary: &[u8]) -> io::Result { + let encoder = Encoder::with_dictionary(level, dictionary)?; Ok(Self { encoder: Unshared::new(encoder), }) diff --git a/crates/compression-codecs/src/zstd/mod.rs b/crates/compression-codecs/src/zstd/mod.rs new file mode 100644 index 00000000..5a0b9be0 --- /dev/null +++ b/crates/compression-codecs/src/zstd/mod.rs @@ -0,0 +1,5 @@ +mod decoder; +mod encoder; +pub mod params; + +pub use self::{decoder::ZstdDecoder, encoder::ZstdEncoder}; diff --git a/src/zstd.rs b/crates/compression-codecs/src/zstd/params.rs similarity index 84% rename from src/zstd.rs rename to crates/compression-codecs/src/zstd/params.rs index 931cd556..3252ed81 100644 --- a/src/zstd.rs +++ b/crates/compression-codecs/src/zstd/params.rs @@ -1,5 +1,7 @@ //! This module contains zstd-specific types for async-compression. +use compression_core::Level; + /// A compression parameter for zstd. This is a stable wrapper around zstd's own `CParameter` /// type, to abstract over different versions of the zstd library. /// @@ -8,7 +10,30 @@ #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub struct CParameter(libzstd::stream::raw::CParameter); +impl From for libzstd::stream::raw::CParameter { + fn from(value: CParameter) -> Self { + value.0 + } +} + impl CParameter { + pub fn quality(level: Level) -> i32 { + let (fastest, best) = libzstd::compression_level_range().into_inner(); + + // NOTE: zstd's "fastest" level is -131072 which can create outputs larger than inputs. + // This library chooses a "fastest" level which has a more-or-less equivalent compression + // ratio to gzip's fastest mode. We still allow precise levels to go negative. + // See discussion in https://github.com/Nullus157/async-compression/issues/352 + const OUR_FASTEST: i32 = 1; + + match level { + Level::Fastest => OUR_FASTEST, + Level::Best => best, + Level::Precise(quality) => quality.clamp(fastest, best), + _ => libzstd::DEFAULT_COMPRESSION_LEVEL, + } + } + /// Window size in bytes (as a power of two) pub fn window_log(value: u32) -> Self { Self(libzstd::stream::raw::CParameter::WindowLog(value)) diff --git a/crates/compression-core/Cargo.toml b/crates/compression-core/Cargo.toml new file mode 100644 index 00000000..93979cff --- /dev/null +++ b/crates/compression-core/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "compression-core" +description = """ +Abstractions for compression algorithms. +""" +version.workspace = true +authors.workspace = true +license.workspace = true +categories.workspace = true +edition.workspace = true +repository.workspace = true +readme = "../../README.md" + +[dependencies] diff --git a/crates/compression-core/src/level.rs b/crates/compression-core/src/level.rs new file mode 100644 index 00000000..15d6120f --- /dev/null +++ b/crates/compression-core/src/level.rs @@ -0,0 +1,19 @@ +/// Level of compression data should be compressed with. +#[non_exhaustive] +#[derive(Default, Clone, Copy, Debug)] +pub enum Level { + /// Fastest quality of compression, usually produces bigger size. + Fastest, + + /// Best quality of compression, usually produces the smallest size. + Best, + + /// Default quality of compression defined by the selected compression algorithm. + #[default] + Default, + + /// Precise quality based on the underlying compression algorithms' qualities. The + /// interpretation of this depends on the algorithm chosen and the specific implementation + /// backing it. Qualities are implicitly clamped to the algorithm's maximum. + Precise(i32), +} diff --git a/crates/compression-core/src/lib.rs b/crates/compression-core/src/lib.rs new file mode 100644 index 00000000..6517580f --- /dev/null +++ b/crates/compression-core/src/lib.rs @@ -0,0 +1,5 @@ +mod level; +pub mod unshared; +pub mod util; + +pub use level::Level; diff --git a/src/unshared.rs b/crates/compression-core/src/unshared.rs similarity index 92% rename from src/unshared.rs rename to crates/compression-core/src/unshared.rs index 35c2945d..6849e269 100644 --- a/src/unshared.rs +++ b/crates/compression-core/src/unshared.rs @@ -18,10 +18,15 @@ use core::fmt::{self, Debug}; pub struct Unshared { inner: T, } +impl From for Unshared { + fn from(inner: T) -> Self { + Unshared { inner } + } +} impl Unshared { pub fn new(inner: T) -> Self { - Unshared { inner } + Self::from(inner) } pub fn get_mut(&mut self) -> &mut T { diff --git a/src/util.rs b/crates/compression-core/src/util.rs similarity index 66% rename from src/util.rs rename to crates/compression-core/src/util.rs index a00bce2d..1d96a1af 100644 --- a/src/util.rs +++ b/crates/compression-core/src/util.rs @@ -8,44 +8,41 @@ pub struct PartialBuffer> { } impl> PartialBuffer { - pub(crate) fn new(buffer: B) -> Self { + pub fn new(buffer: B) -> Self { Self { buffer, index: 0 } } - pub(crate) fn written(&self) -> &[u8] { + pub fn written(&self) -> &[u8] { &self.buffer.as_ref()[..self.index] } - pub(crate) fn unwritten(&self) -> &[u8] { + pub fn unwritten(&self) -> &[u8] { &self.buffer.as_ref()[self.index..] } - pub(crate) fn advance(&mut self, amount: usize) { + pub fn advance(&mut self, amount: usize) { self.index += amount; } - pub(crate) fn get_mut(&mut self) -> &mut B { + pub fn get_mut(&mut self) -> &mut B { &mut self.buffer } - pub(crate) fn into_inner(self) -> B { + pub fn into_inner(self) -> B { self.buffer } - pub(crate) fn reset(&mut self) { + pub fn reset(&mut self) { self.index = 0; } } impl + AsMut<[u8]>> PartialBuffer { - pub(crate) fn unwritten_mut(&mut self) -> &mut [u8] { + pub fn unwritten_mut(&mut self) -> &mut [u8] { &mut self.buffer.as_mut()[self.index..] } - pub(crate) fn copy_unwritten_from>( - &mut self, - other: &mut PartialBuffer, - ) -> usize { + pub fn copy_unwritten_from>(&mut self, other: &mut PartialBuffer) -> usize { let len = std::cmp::min(self.unwritten().len(), other.unwritten().len()); self.unwritten_mut()[..len].copy_from_slice(&other.unwritten()[..len]); @@ -57,7 +54,7 @@ impl + AsMut<[u8]>> PartialBuffer { } impl + Default> PartialBuffer { - pub(crate) fn take(&mut self) -> Self { + pub fn take(&mut self) -> Self { std::mem::replace(self, Self::new(B::default())) } } diff --git a/examples/lzma_filters.rs b/examples/lzma_filters.rs new file mode 100644 index 00000000..5b704eec --- /dev/null +++ b/examples/lzma_filters.rs @@ -0,0 +1,95 @@ +//! Run this example with the following command in a terminal: +//! +//! ```console +//! $ echo -n 'example' | zstd | cargo run --example zstd_gzip --features="tokio,zstd,gzip" | gunzip -c +//! 7example +//! ``` +//! +//! Note that the "7" prefix (input length) is printed to stdout but will likely show up as shown +//! above. This is not an encoding error; see the code in `main`. + +use std::convert::TryFrom; +use std::io::Result; + +use compression_codecs::lzma::params::{ + LzmaDecoderParams, LzmaEncoderParams, LzmaFilter, LzmaFilters, LzmaOptions, +}; +use tokio::io::{stdin, stdout, BufReader, Stdin, Stdout}; +use tokio::io::{ + AsyncReadExt as _, // for `read_to_end` + AsyncWriteExt as _, // for `write_all` and `shutdown` +}; + +const OPTIONS: &str = r" +Usage: lzma_filter [OPTIONS] + + Read stdin and output result to stdout + + Options: + -d Decompress (Default) + -c Compress +"; + +fn usage(msg: &str) -> Result<()> { + if !msg.is_empty() { + eprintln!("{msg}...") + } + + eprintln!("{}", OPTIONS); + std::process::exit(1); +} + +/// Read compressed and output decompressed or +/// Read decompressed and output compressed +/// echo "this is a test" | lzma_filters -- -c | lzma_filters -- -d | xxd +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<()> { + let args: Vec = std::env::args().collect(); + if args.len() != 2 { + return usage("too many args"); + } + + let stdin = stdin(); + let stdout = stdout(); + let filters = + LzmaFilters::default().add_filter(LzmaFilter::Lzma2(LzmaOptions::default().preset(7))); + let mut stdout = if args[1] == "-d" { + decompress(stdin, filters, stdout).await? + } else if args[1] == "-c" { + compress(stdin, filters, stdout).await? + } else { + return usage(&format!("invalid argument: {}", args[1])); + }; + + stdout.flush().await?; + stdout.shutdown().await?; + + Ok(()) +} + +async fn decompress(stdin: Stdin, filters: LzmaFilters, mut stdout: Stdout) -> Result { + let params = LzmaDecoderParams::Raw { filters }; + let codec = compression_codecs::Xz2Decoder::try_from(params).expect("Could not create encoder"); + let codec = compression_codecs::LzmaDecoder::from(codec); + let mut reader = + async_compression::tokio::bufread::LzmaDecoder::with_codec(BufReader::new(stdin), codec); + + let mut buf = vec![]; + reader.read_to_end(&mut buf).await?; + stdout.write_all(&buf).await?; + + Ok(stdout) +} +async fn compress(mut stdin: Stdin, filters: LzmaFilters, stdout: Stdout) -> Result { + let params = LzmaEncoderParams::Raw { filters }; + let codec = compression_codecs::Xz2Encoder::try_from(params).expect("Could not create encoder"); + let codec = compression_codecs::LzmaEncoder::from(codec); + let mut writer = async_compression::tokio::write::LzmaEncoder::with_codec(stdout, codec); + + let mut buf = vec![]; + stdin.read_to_end(&mut buf).await?; + writer.write_all(&buf).await?; + writer.shutdown().await?; + + Ok(writer.into_inner()) +} diff --git a/src/codec/brotli/mod.rs b/src/codec/brotli/mod.rs deleted file mode 100644 index ab4dd7ac..00000000 --- a/src/codec/brotli/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod decoder; -mod encoder; - -pub(crate) use self::{decoder::BrotliDecoder, encoder::BrotliEncoder}; diff --git a/src/codec/bzip2/mod.rs b/src/codec/bzip2/mod.rs deleted file mode 100644 index 2abd9282..00000000 --- a/src/codec/bzip2/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod decoder; -mod encoder; - -pub(crate) use self::{decoder::BzDecoder, encoder::BzEncoder}; diff --git a/src/codec/deflate/mod.rs b/src/codec/deflate/mod.rs deleted file mode 100644 index bd75f911..00000000 --- a/src/codec/deflate/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod decoder; -mod encoder; - -pub(crate) use self::{decoder::DeflateDecoder, encoder::DeflateEncoder}; diff --git a/src/codec/deflate64/mod.rs b/src/codec/deflate64/mod.rs deleted file mode 100644 index 78dee675..00000000 --- a/src/codec/deflate64/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod decoder; - -pub(crate) use self::decoder::Deflate64Decoder; diff --git a/src/codec/flate/mod.rs b/src/codec/flate/mod.rs deleted file mode 100644 index 38934c3e..00000000 --- a/src/codec/flate/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod decoder; -mod encoder; - -pub(crate) use self::{decoder::FlateDecoder, encoder::FlateEncoder}; diff --git a/src/codec/gzip/mod.rs b/src/codec/gzip/mod.rs deleted file mode 100644 index 90be4e1f..00000000 --- a/src/codec/gzip/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod decoder; -mod encoder; -mod header; - -pub(crate) use self::{decoder::GzipDecoder, encoder::GzipEncoder}; diff --git a/src/codec/lz4/mod.rs b/src/codec/lz4/mod.rs deleted file mode 100644 index 8b249165..00000000 --- a/src/codec/lz4/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod decoder; -mod encoder; - -pub(crate) use self::{decoder::Lz4Decoder, encoder::Lz4Encoder}; diff --git a/src/codec/lzma/mod.rs b/src/codec/lzma/mod.rs deleted file mode 100644 index 4a9e73bb..00000000 --- a/src/codec/lzma/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod decoder; -mod encoder; - -pub(crate) use self::{decoder::LzmaDecoder, encoder::LzmaEncoder}; diff --git a/src/codec/xz/mod.rs b/src/codec/xz/mod.rs deleted file mode 100644 index 7178e154..00000000 --- a/src/codec/xz/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod decoder; -mod encoder; - -pub(crate) use self::{decoder::XzDecoder, encoder::XzEncoder}; diff --git a/src/codec/zlib/mod.rs b/src/codec/zlib/mod.rs deleted file mode 100644 index 9ccfae61..00000000 --- a/src/codec/zlib/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod decoder; -mod encoder; - -pub(crate) use self::{decoder::ZlibDecoder, encoder::ZlibEncoder}; diff --git a/src/codec/zstd/mod.rs b/src/codec/zstd/mod.rs deleted file mode 100644 index e6b5525f..00000000 --- a/src/codec/zstd/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod decoder; -mod encoder; - -pub(crate) use self::{decoder::ZstdDecoder, encoder::ZstdEncoder}; diff --git a/src/futures/bufread/generic/decoder.rs b/src/futures/bufread/generic/decoder.rs index e97f0fc7..b0f96b31 100644 --- a/src/futures/bufread/generic/decoder.rs +++ b/src/futures/bufread/generic/decoder.rs @@ -1,13 +1,14 @@ +use crate::codec::Decode; +use crate::core::util::PartialBuffer; + use core::{ pin::Pin, task::{Context, Poll}, }; -use std::io::{IoSlice, Result}; - -use crate::{codec::Decode, util::PartialBuffer}; use futures_core::ready; use futures_io::{AsyncBufRead, AsyncRead, AsyncWrite}; use pin_project_lite::pin_project; +use std::io::{IoSlice, Result}; #[derive(Debug)] enum State { diff --git a/src/futures/bufread/generic/encoder.rs b/src/futures/bufread/generic/encoder.rs index d31e0e13..865778ed 100644 --- a/src/futures/bufread/generic/encoder.rs +++ b/src/futures/bufread/generic/encoder.rs @@ -4,7 +4,8 @@ use core::{ }; use std::io::Result; -use crate::{codec::Encode, util::PartialBuffer}; +use crate::codec::Encode; +use crate::core::util::PartialBuffer; use futures_core::ready; use futures_io::{AsyncBufRead, AsyncRead, AsyncWrite, IoSlice}; use pin_project_lite::pin_project; diff --git a/src/futures/bufread/macros/decoder.rs b/src/futures/bufread/macros/decoder.rs index ba886c72..17c0350e 100644 --- a/src/futures/bufread/macros/decoder.rs +++ b/src/futures/bufread/macros/decoder.rs @@ -14,13 +14,21 @@ macro_rules! decoder { impl<$inner: futures_io::AsyncBufRead> $name<$inner> { /// Creates a new decoder which will read compressed data from the given stream and - /// emit a uncompressed stream. + /// emit an uncompressed stream. pub fn new(read: $inner) -> $name<$inner> { $name { inner: crate::futures::bufread::Decoder::new(read, crate::codec::$name::new()), } } + /// Creates a new decoder with the given codec, which will read compressed data from the given stream and + /// emit an uncompressed stream. + pub fn with_codec(read: $inner, codec: crate::codec::$name) -> $name<$inner> { + $name { + inner: crate::futures::bufread::Decoder::new(read, codec) + } + } + $($($inherent_methods)*)* } @@ -108,7 +116,7 @@ macro_rules! decoder { const _: () = { fn _assert() { - use crate::util::{_assert_send, _assert_sync}; + use crate::core::util::{_assert_send, _assert_sync}; use core::pin::Pin; use futures_io::AsyncBufRead; diff --git a/src/futures/bufread/macros/encoder.rs b/src/futures/bufread/macros/encoder.rs index f26833eb..e82033af 100644 --- a/src/futures/bufread/macros/encoder.rs +++ b/src/futures/bufread/macros/encoder.rs @@ -15,11 +15,19 @@ macro_rules! encoder { impl<$inner: futures_io::AsyncBufRead> $name<$inner> { $( /// Creates a new encoder which will read uncompressed data from the given stream - /// and emit a compressed stream. + /// and emit an compressed stream. /// $($inherent_methods)* )* + /// Creates a new encoder with the given codec, which will read uncompressed data from the given stream + /// and emit an compressed stream. + pub fn with_codec(read: $inner, codec: crate::codec::$name) -> $name<$inner> { + $name { + inner: crate::futures::bufread::Encoder::new(read, codec) + } + } + /// Acquires a reference to the underlying reader that this encoder is wrapping. pub fn get_ref(&self) -> &$inner { self.inner.get_ref() @@ -96,7 +104,7 @@ macro_rules! encoder { const _: () = { fn _assert() { - use crate::util::{_assert_send, _assert_sync}; + use crate::core::util::{_assert_send, _assert_sync}; use core::pin::Pin; use futures_io::AsyncBufRead; diff --git a/src/futures/write/generic/decoder.rs b/src/futures/write/generic/decoder.rs index 31f65042..2a7ed677 100644 --- a/src/futures/write/generic/decoder.rs +++ b/src/futures/write/generic/decoder.rs @@ -1,18 +1,15 @@ +use crate::codec::Decode; +use crate::core::util::PartialBuffer; +use crate::futures::write::{AsyncBufWrite, BufWriter}; +use futures_core::ready; +use futures_io::{AsyncBufRead, AsyncRead, AsyncWrite, IoSliceMut}; +use pin_project_lite::pin_project; use std::{ io, pin::Pin, task::{Context, Poll}, }; -use crate::{ - codec::Decode, - futures::write::{AsyncBufWrite, BufWriter}, - util::PartialBuffer, -}; -use futures_core::ready; -use futures_io::{AsyncBufRead, AsyncRead, AsyncWrite, IoSliceMut}; -use pin_project_lite::pin_project; - #[derive(Debug)] enum State { Decoding, diff --git a/src/futures/write/generic/encoder.rs b/src/futures/write/generic/encoder.rs index abd18c40..9076c04f 100644 --- a/src/futures/write/generic/encoder.rs +++ b/src/futures/write/generic/encoder.rs @@ -4,11 +4,9 @@ use std::{ task::{Context, Poll}, }; -use crate::{ - codec::Encode, - futures::write::{AsyncBufWrite, BufWriter}, - util::PartialBuffer, -}; +use crate::codec::Encode; +use crate::core::util::PartialBuffer; +use crate::futures::write::{AsyncBufWrite, BufWriter}; use futures_core::ready; use futures_io::{AsyncBufRead, AsyncRead, AsyncWrite, IoSliceMut}; use pin_project_lite::pin_project; diff --git a/src/futures/write/macros/decoder.rs b/src/futures/write/macros/decoder.rs index 798540ab..4a0a13af 100644 --- a/src/futures/write/macros/decoder.rs +++ b/src/futures/write/macros/decoder.rs @@ -4,7 +4,7 @@ macro_rules! decoder { $(#[$attr])* /// /// This structure implements an [`AsyncWrite`](futures_io::AsyncWrite) interface and will - /// take in compressed data and write it uncompressed to an underlying stream. + /// take in compressed data and write it, uncompressed, to an underlying stream. #[derive(Debug)] pub struct $name<$inner> { #[pin] @@ -13,7 +13,7 @@ macro_rules! decoder { } impl<$inner: futures_io::AsyncWrite> $name<$inner> { - /// Creates a new decoder which will take in compressed data and write it uncompressed + /// Creates a new decoder which will take in compressed data and write it, uncompressed, /// to the given stream. pub fn new(read: $inner) -> $name<$inner> { $name { @@ -21,6 +21,14 @@ macro_rules! decoder { } } + /// Creates a new decoder with the given codec, which will take in compressed data and write it, uncompressed, + /// to the given stream. + pub fn with_codec(read: $inner, codec: crate::codec::$name) -> $name<$inner> { + $name { + inner: crate::futures::write::Decoder::new(read, codec) + } + } + $($($inherent_methods)*)* } @@ -114,7 +122,7 @@ macro_rules! decoder { const _: () = { fn _assert() { - use crate::util::{_assert_send, _assert_sync}; + use crate::core::util::{_assert_send, _assert_sync}; use core::pin::Pin; use futures_io::AsyncWrite; diff --git a/src/futures/write/macros/encoder.rs b/src/futures/write/macros/encoder.rs index 7e0f929a..81632ee9 100644 --- a/src/futures/write/macros/encoder.rs +++ b/src/futures/write/macros/encoder.rs @@ -19,6 +19,14 @@ macro_rules! encoder { /// $($inherent_methods)* )* + + /// Creates a new encoder with the given codec, which will take in uncompressed data and write it + /// compressed to the given stream. + pub fn with_codec(read: $inner, codec: crate::codec::$name) -> $name<$inner> { + $name { + inner: crate::futures::write::Encoder::new(read, codec) + } + } } impl<$inner> $name<$inner> { @@ -111,7 +119,7 @@ macro_rules! encoder { const _: () = { fn _assert() { - use crate::util::{_assert_send, _assert_sync}; + use crate::core::util::{_assert_send, _assert_sync}; use core::pin::Pin; use futures_io::AsyncWrite; diff --git a/src/lib.rs b/src/lib.rs index 2635f1f1..4578a625 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -152,146 +152,13 @@ )] #![cfg_attr(not(all), allow(unused))] -#[cfg(any( - feature = "bzip2", - feature = "flate2", - feature = "lzma", - feature = "lz4" -))] -use std::convert::TryInto; - #[macro_use] mod macros; -mod codec; #[cfg(feature = "futures-io")] pub mod futures; #[cfg(feature = "tokio")] pub mod tokio; -mod unshared; -mod util; - -#[cfg(feature = "brotli")] -pub mod brotli; -#[cfg(feature = "lz4")] -pub mod lz4; -#[cfg(feature = "zstd")] -pub mod zstd; - -/// Level of compression data should be compressed with. -#[non_exhaustive] -#[derive(Clone, Copy, Debug)] -pub enum Level { - /// Fastest quality of compression, usually produces bigger size. - Fastest, - - /// Best quality of compression, usually produces the smallest size. - Best, - - /// Default quality of compression defined by the selected compression algorithm. - Default, - - /// Precise quality based on the underlying compression algorithms' qualities. The - /// interpretation of this depends on the algorithm chosen and the specific implementation - /// backing it. Qualities are implicitly clamped to the algorithm's maximum. - Precise(i32), -} - -impl Level { - #[cfg(feature = "brotli")] - fn into_brotli( - self, - mut params: ::brotli::enc::backward_references::BrotliEncoderParams, - ) -> ::brotli::enc::backward_references::BrotliEncoderParams { - match self { - Self::Fastest => params.quality = 0, - Self::Best => params.quality = 11, - Self::Precise(quality) => params.quality = quality.clamp(0, 11), - Self::Default => (), - } - - params - } - - #[cfg(feature = "bzip2")] - fn into_bzip2(self) -> bzip2::Compression { - let fastest = bzip2::Compression::fast(); - let best = bzip2::Compression::best(); - - match self { - Self::Fastest => fastest, - Self::Best => best, - Self::Precise(quality) => bzip2::Compression::new( - quality - .try_into() - .unwrap_or(0) - .clamp(fastest.level(), best.level()), - ), - Self::Default => bzip2::Compression::default(), - } - } - - #[cfg(feature = "flate2")] - fn into_flate2(self) -> flate2::Compression { - let fastest = flate2::Compression::fast(); - let best = flate2::Compression::best(); - let none = flate2::Compression::none(); - - match self { - Self::Fastest => fastest, - Self::Best => best, - Self::Precise(quality) => flate2::Compression::new( - quality - .try_into() - .unwrap_or(0) - .clamp(none.level(), best.level()), - ), - Self::Default => flate2::Compression::default(), - } - } - - #[cfg(feature = "zstd")] - fn into_zstd(self) -> i32 { - let (fastest, best) = libzstd::compression_level_range().into_inner(); - - // NOTE: zstd's "fastest" level is -131072 which can create outputs larger than inputs. - // This library chooses a "fastest" level which has a more-or-less equivalent compression - // ratio to gzip's fastest mode. We still allow precise levels to go negative. - // See discussion in https://github.com/Nullus157/async-compression/issues/352 - const OUR_FASTEST: i32 = 1; - - match self { - Self::Fastest => OUR_FASTEST, - Self::Best => best, - Self::Precise(quality) => quality.clamp(fastest, best), - Self::Default => libzstd::DEFAULT_COMPRESSION_LEVEL, - } - } - - #[cfg(feature = "lzma")] - fn into_xz2(self) -> u32 { - match self { - Self::Fastest => 0, - Self::Best => 9, - Self::Precise(quality) => quality.try_into().unwrap_or(0).min(9), - Self::Default => 5, - } - } - - #[cfg(feature = "lz4")] - fn into_lz4( - self, - mut preferences: ::lz4::liblz4::LZ4FPreferences, - ) -> ::lz4::liblz4::LZ4FPreferences { - let level = match self { - Self::Fastest => 0, - Self::Best => 12, - Self::Precise(quality) => quality.try_into().unwrap_or(0).min(12), - Self::Default => 0, - }; - - preferences.compression_level = level; - preferences - } -} +pub use compression_codecs as codec; +pub use compression_core as core; diff --git a/src/macros.rs b/src/macros.rs index cbd161f1..0be7b55b 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -18,7 +18,7 @@ macro_rules! algos { #[cfg(feature = $algo_s)] $encoder<$inner> { pub fn new(inner: $inner) -> Self { - Self::with_quality(inner, crate::Level::Default) + Self::with_quality(inner, crate::core::Level::Default) } } @@ -42,26 +42,24 @@ macro_rules! algos { ($($mod:ident)::+ <$inner:ident>) => { algos!(@algo brotli ["brotli"] BrotliDecoder BrotliEncoder <$inner> { @enc - pub fn with_quality(inner: $inner, level: crate::Level) -> Self { - let params = brotli::enc::backward_references::BrotliEncoderParams::default(); - + pub fn with_quality(inner: $inner, level: crate::core::Level) -> Self { + // let params = brotli::enc::backward_references::BrotliEncoderParams::default(); + let params = crate::codec::brotli::params::EncoderParams::default(); + let params = params.quality(level); Self { inner: crate::$($mod::)+generic::Encoder::new( inner, - crate::codec::BrotliEncoder::new(level.into_brotli(params)), + crate::codec::BrotliEncoder::new(params), ), } } /// Creates a new encoder, using the specified compression level and parameters, which /// will read uncompressed data from the given stream and emit a compressed stream. - pub fn with_quality_and_params( + pub fn with_params( inner: $inner, - level: crate::Level, - params: crate::brotli::EncoderParams, + params: crate::codec::brotli::params::EncoderParams, ) -> Self { - let params = level.into_brotli(params.as_brotli()); - Self { inner: crate::$($mod::)+generic::Encoder::new( inner, @@ -76,11 +74,12 @@ macro_rules! algos { algos!(@algo bzip2 ["bzip2"] BzDecoder BzEncoder <$inner> { @enc - pub fn with_quality(inner: $inner, level: crate::Level) -> Self { + pub fn with_quality(inner: $inner, level: crate::core::Level) -> Self { + let params = crate::codec::bzip2::params::Bzip2EncoderParams::from(level); Self { inner: crate::$($mod::)+generic::Encoder::new( inner, - crate::codec::BzEncoder::new(level.into_bzip2(), 0), + crate::codec::BzEncoder::new(params, 0), ), } } @@ -90,11 +89,13 @@ macro_rules! algos { algos!(@algo deflate ["deflate"] DeflateDecoder DeflateEncoder <$inner> { @enc - pub fn with_quality(inner: $inner, level: crate::Level) -> Self { + pub fn with_quality(inner: $inner, level: crate::core::Level) -> Self { + let mut params = crate::codec::flate::params::FlateEncoderParams::from(level); + Self { inner: crate::$($mod::)+generic::Encoder::new( inner, - crate::codec::DeflateEncoder::new(level.into_flate2()), + crate::codec::DeflateEncoder::new(params), ), } } @@ -119,11 +120,12 @@ macro_rules! algos { algos!(@algo gzip ["gzip"] GzipDecoder GzipEncoder <$inner> { @enc - pub fn with_quality(inner: $inner, level: crate::Level) -> Self { + pub fn with_quality(inner: $inner, level: crate::core::Level) -> Self { + let params = crate::codec::flate::params::FlateEncoderParams::from(level); Self { inner: crate::$($mod::)+generic::Encoder::new( inner, - crate::codec::GzipEncoder::new(level.into_flate2()), + crate::codec::GzipEncoder::new(params), ), } } @@ -133,11 +135,12 @@ macro_rules! algos { algos!(@algo zlib ["zlib"] ZlibDecoder ZlibEncoder <$inner> { @enc - pub fn with_quality(inner: $inner, level: crate::Level) -> Self { + pub fn with_quality(inner: $inner, level: crate::core::Level) -> Self { + let params = crate::codec::flate::params::FlateEncoderParams::from(level); Self { inner: crate::$($mod::)+generic::Encoder::new( inner, - crate::codec::ZlibEncoder::new(level.into_flate2()), + crate::codec::ZlibEncoder::new(params), ), } } @@ -158,11 +161,12 @@ macro_rules! algos { algos!(@algo zstd ["zstd"] ZstdDecoder ZstdEncoder <$inner> { @enc - pub fn with_quality(inner: $inner, level: crate::Level) -> Self { + pub fn with_quality(inner: $inner, level: crate::core::Level) -> Self { + let params = crate::codec::zstd::params::CParameter::quality(level); Self { inner: crate::$($mod::)+generic::Encoder::new( inner, - crate::codec::ZstdEncoder::new(level.into_zstd()), + crate::codec::ZstdEncoder::new(params), ), } } @@ -175,15 +179,16 @@ macro_rules! algos { /// Panics if this function is called with a [`CParameter::nb_workers()`] parameter and /// the `zstdmt` crate feature is _not_ enabled. /// - /// [`CParameter::nb_workers()`]: crate::zstd::CParameter + /// [`CParameter::nb_workers()`]: crate::codec::zstd::params::CParameter // // TODO: remove panic note on next breaking release, along with `CParameter::nb_workers` // change - pub fn with_quality_and_params(inner: $inner, level: crate::Level, params: &[crate::zstd::CParameter]) -> Self { + pub fn with_quality_and_params(inner: $inner, level: crate::core::Level, params: &[crate::codec::zstd::params::CParameter]) -> Self { + let level = crate::codec::zstd::params::CParameter::quality(level); Self { inner: crate::$($mod::)+generic::Encoder::new( inner, - crate::codec::ZstdEncoder::new_with_params(level.into_zstd(), params), + crate::codec::ZstdEncoder::new_with_params(level, params), ), } } @@ -198,11 +203,12 @@ macro_rules! algos { /// # Errors /// /// Returns error when `dictionary` is not valid. - pub fn with_dict(inner: $inner, level: crate::Level, dictionary: &[u8]) -> ::std::io::Result { + pub fn with_dict(inner: $inner, level: crate::core::Level, dictionary: &[u8]) -> ::std::io::Result { + let level = crate::codec::zstd::params::CParameter::quality(level); Ok(Self { inner: crate::$($mod::)+generic::Encoder::new( inner, - crate::codec::ZstdEncoder::new_with_dict(level.into_zstd(), dictionary)?, + crate::codec::ZstdEncoder::new_with_dict(level, dictionary)?, ), }) } @@ -210,7 +216,7 @@ macro_rules! algos { { @dec /// Creates a new decoder, using the specified parameters, which will read compressed /// data from the given stream and emit a decompressed stream. - pub fn with_params(inner: $inner, params: &[crate::zstd::DParameter]) -> Self { + pub fn with_params(inner: $inner, params: &[crate::codec::zstd::params::DParameter]) -> Self { Self { inner: crate::$($mod::)+generic::Decoder::new( inner, @@ -244,11 +250,11 @@ macro_rules! algos { algos!(@algo xz ["xz"] XzDecoder XzEncoder <$inner> { @enc - pub fn with_quality(inner: $inner, level: crate::Level) -> Self { + pub fn with_quality(inner: $inner, level: crate::core::Level) -> Self { Self { inner: crate::$($mod::)+generic::Encoder::new( inner, - crate::codec::XzEncoder::new(level.into_xz2()), + crate::codec::XzEncoder::new(level), ), } } @@ -257,11 +263,11 @@ macro_rules! algos { /// /// Note that flushing will severely impact multi-threaded performance. #[cfg(feature = "xz-parallel")] - pub fn parallel(inner: $inner, level: crate::Level, threads: std::num::NonZeroU32) -> Self { + pub fn parallel(inner: $inner, level: crate::core::Level, threads: std::num::NonZeroU32) -> Self { Self { inner: crate::$($mod::)+generic::Encoder::new( inner, - crate::codec::XzEncoder::parallel(threads, level.into_xz2()), + crate::codec::XzEncoder::parallel(threads, level), ), } } @@ -284,10 +290,12 @@ macro_rules! algos { /// Creates a new multi-threaded decoder. #[cfg(feature = "xz-parallel")] pub fn parallel(read: $inner, threads: std::num::NonZeroU32) -> Self { + use std::convert::TryInto; + Self { inner: crate::$($mod::)+generic::Decoder::new( read, - crate::codec::XzDecoder::parallel(threads, u64::MAX), + crate::codec::XzDecoder::parallel(threads, usize::MAX.try_into().unwrap()), ), } } @@ -312,12 +320,11 @@ macro_rules! algos { algos!(@algo lzma ["lzma"] LzmaDecoder LzmaEncoder <$inner> { @enc - pub fn with_quality(inner: $inner, level: crate::Level) -> Self { + pub fn with_quality(inner: $inner, level: crate::core::Level) -> Self { + let encoder = crate::codec::LzmaEncoder::new(level); + let inner = crate::$($mod::)+generic::Encoder::new(inner, encoder); Self { - inner: crate::$($mod::)+generic::Encoder::new( - inner, - crate::codec::LzmaEncoder::new(level.into_xz2()), - ), + inner } } } @@ -342,18 +349,19 @@ macro_rules! algos { algos!(@algo lz4 ["lz4"] Lz4Decoder Lz4Encoder <$inner> { @enc - pub fn with_quality(inner: $inner, level: crate::Level) -> Self { - Self::with_quality_and_params(inner, level, crate::lz4::EncoderParams::default()) + pub fn with_quality(inner: $inner, level: crate::core::Level) -> Self { + Self::with_quality_and_params(inner, level, crate::codec::lz4::params::EncoderParams::default()) } /// Creates a new encoder, using the specified compression level and parameters, which /// will read uncompressed data from the given stream and emit a compressed stream. pub fn with_quality_and_params( inner: $inner, - level: crate::Level, - params: crate::lz4::EncoderParams, + level: crate::core::Level, + mut params: crate::codec::lz4::params::EncoderParams, ) -> Self { - let encoder = crate::codec::Lz4Encoder::new(level.into_lz4(params.as_lz4())); + let params = params.level(level); + let encoder = crate::codec::Lz4Encoder::new(params); let cap = encoder.buffer_size(); Self { inner: crate::$($mod::)+generic::Encoder::with_capacity( diff --git a/src/tokio/bufread/generic/decoder.rs b/src/tokio/bufread/generic/decoder.rs index c4fdb985..29c3e0ee 100644 --- a/src/tokio/bufread/generic/decoder.rs +++ b/src/tokio/bufread/generic/decoder.rs @@ -1,12 +1,12 @@ +use crate::codec::Decode; +use crate::core::util::PartialBuffer; use core::{ pin::Pin, task::{Context, Poll}, }; -use std::io::{IoSlice, Result}; - -use crate::{codec::Decode, util::PartialBuffer}; use futures_core::ready; use pin_project_lite::pin_project; +use std::io::{IoSlice, Result}; use tokio::io::{AsyncBufRead, AsyncRead, AsyncWrite, ReadBuf}; #[derive(Debug)] diff --git a/src/tokio/bufread/generic/encoder.rs b/src/tokio/bufread/generic/encoder.rs index 44ab05bb..b83af5f5 100644 --- a/src/tokio/bufread/generic/encoder.rs +++ b/src/tokio/bufread/generic/encoder.rs @@ -1,12 +1,12 @@ +use crate::codec::Encode; +use crate::core::util::PartialBuffer; use core::{ pin::Pin, task::{Context, Poll}, }; -use std::io::{IoSlice, Result}; - -use crate::{codec::Encode, util::PartialBuffer}; use futures_core::ready; use pin_project_lite::pin_project; +use std::io::{IoSlice, Result}; use tokio::io::{AsyncBufRead, AsyncRead, AsyncWrite, ReadBuf}; #[derive(Debug)] diff --git a/src/tokio/bufread/macros/decoder.rs b/src/tokio/bufread/macros/decoder.rs index a3c36ac1..d4663f20 100644 --- a/src/tokio/bufread/macros/decoder.rs +++ b/src/tokio/bufread/macros/decoder.rs @@ -14,13 +14,21 @@ macro_rules! decoder { impl<$inner: tokio::io::AsyncBufRead> $name<$inner> { /// Creates a new decoder which will read compressed data from the given stream and - /// emit a uncompressed stream. + /// emit an uncompressed stream. pub fn new(read: $inner) -> $name<$inner> { $name { inner: crate::tokio::bufread::Decoder::new(read, crate::codec::$name::new()), } } + /// Creates a new decoder with the given codec, which will read compressed data from the given stream and + /// emit an uncompressed stream. + pub fn with_codec(read: $inner, codec: crate::codec::$name) -> $name<$inner> { + $name { + inner: crate::tokio::bufread::Decoder::new(read, codec) + } + } + $($($inherent_methods)*)* } @@ -112,7 +120,7 @@ macro_rules! decoder { const _: () = { fn _assert() { - use crate::util::{_assert_send, _assert_sync}; + use crate::core::util::{_assert_send, _assert_sync}; use core::pin::Pin; use tokio::io::AsyncBufRead; diff --git a/src/tokio/bufread/macros/encoder.rs b/src/tokio/bufread/macros/encoder.rs index f43f0631..5f489cc4 100644 --- a/src/tokio/bufread/macros/encoder.rs +++ b/src/tokio/bufread/macros/encoder.rs @@ -15,10 +15,18 @@ macro_rules! encoder { impl<$inner: tokio::io::AsyncBufRead> $name<$inner> { $( /// Creates a new encoder which will read uncompressed data from the given stream - /// and emit a compressed stream. + /// and emit an compressed stream. /// $($inherent_methods)* )* + + /// Creates a new encoder with the given codec, which will read uncompressed data from the given stream + /// and emit an compressed stream. + pub fn with_codec(read: $inner, codec: crate::codec::$name) -> $name<$inner> { + $name { + inner: crate::tokio::bufread::Encoder::new(read, codec) + } + } } impl<$inner> $name<$inner> { @@ -102,7 +110,7 @@ macro_rules! encoder { const _: () = { fn _assert() { - use crate::util::{_assert_send, _assert_sync}; + use crate::core::util::{_assert_send, _assert_sync}; use core::pin::Pin; use tokio::io::AsyncBufRead; diff --git a/src/tokio/write/generic/decoder.rs b/src/tokio/write/generic/decoder.rs index 075c1ec0..4f8408c5 100644 --- a/src/tokio/write/generic/decoder.rs +++ b/src/tokio/write/generic/decoder.rs @@ -1,16 +1,13 @@ +use crate::codec::Decode; +use crate::core::util::PartialBuffer; +use crate::tokio::write::{AsyncBufWrite, BufWriter}; +use futures_core::ready; +use pin_project_lite::pin_project; use std::{ io, pin::Pin, task::{Context, Poll}, }; - -use crate::{ - codec::Decode, - tokio::write::{AsyncBufWrite, BufWriter}, - util::PartialBuffer, -}; -use futures_core::ready; -use pin_project_lite::pin_project; use tokio::io::{AsyncBufRead, AsyncRead, AsyncWrite, ReadBuf}; #[derive(Debug)] diff --git a/src/tokio/write/generic/encoder.rs b/src/tokio/write/generic/encoder.rs index 6e39bfb3..dcb8eac0 100644 --- a/src/tokio/write/generic/encoder.rs +++ b/src/tokio/write/generic/encoder.rs @@ -1,16 +1,13 @@ +use crate::codec::Encode; +use crate::core::util::PartialBuffer; +use crate::tokio::write::{AsyncBufWrite, BufWriter}; +use futures_core::ready; +use pin_project_lite::pin_project; use std::{ io, pin::Pin, task::{Context, Poll}, }; - -use crate::{ - codec::Encode, - tokio::write::{AsyncBufWrite, BufWriter}, - util::PartialBuffer, -}; -use futures_core::ready; -use pin_project_lite::pin_project; use tokio::io::{AsyncBufRead, AsyncRead, AsyncWrite, ReadBuf}; #[derive(Debug)] diff --git a/src/tokio/write/macros/decoder.rs b/src/tokio/write/macros/decoder.rs index bc3f1541..91883752 100644 --- a/src/tokio/write/macros/decoder.rs +++ b/src/tokio/write/macros/decoder.rs @@ -13,7 +13,7 @@ macro_rules! decoder { } impl<$inner: tokio::io::AsyncWrite> $name<$inner> { - /// Creates a new decoder which will take in compressed data and write it uncompressed + /// Creates a new decoder which will take in compressed data and write it, uncompressed, /// to the given stream. pub fn new(read: $inner) -> $name<$inner> { $name { @@ -21,6 +21,14 @@ macro_rules! decoder { } } + /// Creates a new decoder which will take in compressed data and write it, uncompressed, + /// to the given stream. + pub fn with_codec(read: $inner, codec: crate::codec::$name) -> $name<$inner> { + $name { + inner: crate::tokio::write::Decoder::new(read, codec) + } + } + $($($inherent_methods)*)* } @@ -106,7 +114,7 @@ macro_rules! decoder { const _: () = { fn _assert() { - use crate::util::{_assert_send, _assert_sync}; + use crate::core::util::{_assert_send, _assert_sync}; use core::pin::Pin; use tokio::io::AsyncWrite; diff --git a/src/tokio/write/macros/encoder.rs b/src/tokio/write/macros/encoder.rs index a2a25e63..310ddc5c 100644 --- a/src/tokio/write/macros/encoder.rs +++ b/src/tokio/write/macros/encoder.rs @@ -19,6 +19,14 @@ macro_rules! encoder { /// $($inherent_methods)* )* + + /// Creates a new encoder with the given codec, which will take in uncompressed data and write it, + /// compressed, to the given stream. + pub fn with_codec(read: $inner, codec: crate::codec::$name) -> $name<$inner> { + $name { + inner: crate::tokio::write::Encoder::new(read, codec) + } + } } impl<$inner> $name<$inner> { @@ -103,7 +111,7 @@ macro_rules! encoder { const _: () = { fn _assert() { - use crate::util::{_assert_send, _assert_sync}; + use crate::core::util::{_assert_send, _assert_sync}; use core::pin::Pin; use tokio::io::AsyncWrite; diff --git a/tests/brotli.rs b/tests/brotli.rs index eaaa45e2..d032d3e8 100644 --- a/tests/brotli.rs +++ b/tests/brotli.rs @@ -1,4 +1,11 @@ +use compression_codecs::brotli::params::EncoderParams; + #[macro_use] mod utils; test_cases!(brotli); + +#[test] +pub fn brotli_params() { + let _ = EncoderParams::default(); +} diff --git a/tests/proptest.rs b/tests/proptest.rs index 8e181ddd..ba66b1f7 100644 --- a/tests/proptest.rs +++ b/tests/proptest.rs @@ -1,4 +1,4 @@ -use async_compression::Level; +use compression_core::Level; use ::proptest::{ arbitrary::any, diff --git a/tests/utils/mod.rs b/tests/utils/mod.rs index 67bafc6b..22312f87 100644 --- a/tests/utils/mod.rs +++ b/tests/utils/mod.rs @@ -12,7 +12,7 @@ pub mod algos; pub mod impls; pub use self::{input_stream::InputStream, track_closed::TrackClosed, track_eof::TrackEof}; -pub use async_compression::Level; +pub use compression_core::Level; pub use futures::{executor::block_on, pin_mut, stream::Stream}; pub use std::{future::Future, io::Result, iter::FromIterator, pin::Pin}; diff --git a/tests/zstd-window-size.rs b/tests/zstd-window-size.rs index 7b5e6dec..3d1b90c7 100644 --- a/tests/zstd-window-size.rs +++ b/tests/zstd-window-size.rs @@ -1,6 +1,6 @@ #![cfg(not(windows))] -use async_compression::zstd::DParameter; +use compression_codecs::zstd::params::DParameter; use tokio::io::AsyncWriteExt as _; #[tokio::test]