Improve Zstd binary provided (#93)

* sync

* feat(cli): clap compress progress

* fmt(clippy): apply is_multiple_of changes

* cli: split bin/zstd.rs into multiple files

* docs: update changelog for clap cli revision

* feat(cli): change compression ratio display and improve time display

* refactor: migrate to cargo workspace

---------

Co-authored-by: arc <zleyyij@users.noreply.github.com>

Author: zleyyij

.github/workflows/ci.yml

@@ -12,13 +12,15 @@ jobs:
 
         - name: Install stable toolchain
           uses: dtolnay/rust-toolchain@stable
+          with:
+            components: clippy
         - name: Install cargo-hack
           uses: taiki-e/install-action@v2
           with:
             tool: cargo-hack
-        - run: cargo hack check --feature-powerset --exclude-features rustc-dep-of-std
-        - run: cargo hack clippy --feature-powerset --exclude-features rustc-dep-of-std
-        - run: cargo hack test --feature-powerset --exclude-features rustc-dep-of-std
+        - run: cargo hack check --workspace --feature-powerset --exclude-features rustc-dep-of-std
+        - run: cargo hack clippy --workspace --feature-powerset --exclude-features rustc-dep-of-std
+        - run: cargo hack test --workspace --feature-powerset --exclude-features rustc-dep-of-std
 
   nightly-stuff:
     name: nightly clippy, format and miri tests
@@ -33,8 +35,8 @@ jobs:
           components: rustfmt, clippy, miri
 
       - run: cargo +nightly fmt --all -- --check
-      - run: cargo +nightly clippy --no-default-features -- -D warnings
-      - run: cargo +nightly clippy -- -D warnings
+      - run: cargo +nightly clippy --workspace --no-default-features -- -D warnings
+      - run: cargo +nightly clippy --workspace -- -D warnings
       - run: cargo +nightly miri test ringbuffer
       - run: cargo +nightly miri test short_Writer
 

Cargo.toml

@@ -1,60 +1,3 @@
-[package]
-name = "ruzstd"
-version = "0.8.1"
-authors = ["Moritz Borcherding <moritz.borcherding@web.de>"]
-edition = "2018"
-license = "MIT"
-homepage = "https://github.com/KillingSpark/zstd-rs"
-repository = "https://github.com/KillingSpark/zstd-rs"
-description = "A decoder for the zstd compression format"
-exclude = ["decodecorpus_files/*", "dict_tests/*", "fuzz_decodecorpus/*"]
-readme = "Readme.md"
-keywords = ["zstd", "zstandard", "decompression"]
-categories = ["compression"]
-
-[package.metadata.docs.rs]
-all-features = true
-rustdoc-args = ["--cfg", "docsrs"]
-
-[dependencies]
-twox-hash = { version = "2.0", default-features = false, features = ["xxhash64"], optional = true }
-
-# Internal feature, only used when building as part of libstd, not part of the
-# stable interface of this crate.
-compiler_builtins = { version = "0.1.2", optional = true }
-core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" }
-alloc = { version = "1.0.0", optional = true, package = "rustc-std-workspace-alloc" }
-fastrand = "2.3.0"
-
-
-[dev-dependencies]
-criterion = "0.5"
-rand = { version = "0.8.5", features = ["small_rng"] }
-zstd =  { version = "0.13.2", features = ["zstdmt"]}
-
-[features]
-default = ["hash", "std"]
-hash = ["dep:twox-hash"]
-fuzz_exports = []
-std = []
-dict_builder = ["std"]
-
-# Internal feature, only used when building as part of libstd, not part of the
-# stable interface of this crate.
-rustc-dep-of-std = ["dep:compiler_builtins", "dep:core", "dep:alloc"]
-
-[[bench]]
-name = "decode_all"
-harness = false
-
-[[bin]]
-name = "zstd"
-required-features = ["std"]
-
-[[bin]]
-name = "zstd_stream"
-required-features = ["std"]
-
-[[bin]]
-name = "zstd_dict"
-required-features = ["std", "dict_builder"]
+[workspace]
+resolver = "3"
+members = ["ruzstd", "cli"]

Changelog.md

@@ -4,6 +4,8 @@ This document records the changes made between versions, starting with version 0
 
 # After 0.8.0 (Current)
 * The compressor now includes a `content_checksum` when the `hash` feature is enabled
+* Dictionary generation has been added
+* The CLI has been refactored to use `clap`
 
 # After 0.7.3
 * Add initial compression support

cli/Cargo.toml

@@ -0,0 +1,29 @@
+
+[package]
+name = "ruzstd-cli"
+version = "0.8.1"
+authors = ["Moritz Borcherding <moritz.borcherding@web.de>"]
+edition = "2018"
+license = "MIT"
+homepage = "https://github.com/KillingSpark/zstd-rs"
+repository = "https://github.com/KillingSpark/zstd-rs"
+description = "A command line interface for the `ruzstd` zstd implementation"
+readme = "../Readme.md"
+keywords = ["zstd", "zstandard", "decompression"]
+categories = ["compression"]
+
+[dependencies]
+# Used for the command line binary only
+clap = { version = "4.5.46", features = ["derive"]}
+color-eyre = { version = "0.6.5" }
+tracing = { version = "0.1.41" }
+indicatif = { version = "0.18.0" }
+tracing-indicatif = { version = "0.3.13" }
+console = { version = "0.16.1" }
+tracing-subscriber = { version = "0.3.20" }
+ruzstd = { path = "../ruzstd", features = ["std"] }
+
+[dev-dependencies]
+criterion = "0.5"
+rand = { version = "0.8.5", features = ["small_rng"] }
+zstd = "0.13.2"

cli/src/main.rs

@@ -0,0 +1,182 @@
+extern crate ruzstd;
+mod progress;
+use progress::ProgressMonitor;
+
+use std::fs::File;
+use std::io::BufReader;
+use std::os::unix::fs::MetadataExt;
+use std::path::Path;
+use std::path::PathBuf;
+
+use progress::fmt_size;
+
+use clap::{Parser, Subcommand};
+use color_eyre::eyre::{ContextCompat, WrapErr};
+use ruzstd::encoding::CompressionLevel;
+use tracing::info;
+use tracing_indicatif::IndicatifLayer;
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
+
+#[derive(Parser)]
+#[command(version, about)]
+struct Cli {
+    #[command(subcommand)]
+    command: Option<Commands>,
+}
+
+// TODO: implement a dictionary creation command, and a command for benchmarking
+#[derive(Subcommand)]
+enum Commands {
+    /// Compress a single file. If no output file is specified,
+    /// output will be written to <INPUT_FILE>.zst
+    Compress {
+        /// File to compress
+        input_file: PathBuf,
+        /// Where the compressed file is written
+        /// [default: <INPUT_FILE>.zst]
+        output_file: Option<PathBuf>,
+        /// How thoroughly the file should be compressed. A higher level will take
+        /// more time to compress but result in a smaller file, and vice versa.
+        ///
+        /// - 0: Uncompressed
+        /// - 1: Fastest
+        /// - 2: Default
+        /// - 3: Better
+        /// - 4: Best
+        #[arg(
+            short,
+            long,
+            value_name = "COMPRESSION_LEVEL",
+            default_value_t = 2,
+            verbatim_doc_comment
+        )]
+        level: u8,
+    },
+    Decompress {
+        /// .zst archive to decompress
+        input_file: PathBuf,
+        /// Where the compressed file is written
+        /// [default: <ARCHIVE_NAME>]
+        output_file: Option<PathBuf>,
+    },
+}
+
+fn main() -> color_eyre::Result<()> {
+    // Process CLI arguments
+    let cli = Cli::parse();
+    // Initialize logging (with indicatif integration)
+    let indicatif_layer = IndicatifLayer::new();
+    tracing_subscriber::registry()
+        .with(
+            tracing_subscriber::fmt::layer()
+                .with_writer(indicatif_layer.get_stderr_writer())
+                .without_time(),
+        )
+        .with(indicatif_layer)
+        .init();
+
+    let command: Commands = cli.command.wrap_err("no subcommand provided").unwrap();
+    match command {
+        Commands::Compress {
+            input_file,
+            output_file,
+            level,
+        } => {
+            let output_file = output_file.unwrap_or_else(|| add_extension(&input_file, ".zst"));
+            compress(input_file, output_file, level)?;
+        }
+        Commands::Decompress {
+            input_file,
+            output_file,
+        } => {
+            let output_file = output_file.unwrap_or(
+                input_file
+                    .file_stem()
+                    .expect("input has a file name")
+                    .into(),
+            );
+            decompress(input_file, output_file)?;
+        }
+    }
+    Ok(())
+}
+
+fn compress(input: PathBuf, output: PathBuf, level: u8) -> color_eyre::Result<()> {
+    info!("compressing {input:?} to {output:?}");
+    let compression_level: ruzstd::encoding::CompressionLevel = match level {
+        0 => CompressionLevel::Uncompressed,
+        1 => CompressionLevel::Fastest,
+        2 => CompressionLevel::Default,
+        3 => CompressionLevel::Better,
+        4 => CompressionLevel::Best,
+        _ => {
+            unimplemented!("unsupported compression level: {}", level);
+        }
+    };
+    let source_file = File::open(input).wrap_err("failed to open input file")?;
+    let source_size = source_file.metadata()?.len() as usize;
+    let buffered_source = BufReader::new(source_file);
+    let encoder_input = ProgressMonitor::new(buffered_source, source_size);
+    let output: File = File::create(output).wrap_err("failed to open output file for writing")?;
+
+    ruzstd::encoding::compress(encoder_input, &output, compression_level);
+    let compressed_size = output.metadata()?.len();
+    let compression_ratio = compressed_size as f64 / source_size as f64 * 100.0;
+    info!(
+        "{} ——> {} ({compression_ratio:.2}%)",
+        fmt_size(source_size as f64),
+        fmt_size(compressed_size as f64)
+    );
+    Ok(())
+}
+
+fn decompress(input: PathBuf, output: PathBuf) -> color_eyre::Result<()> {
+    info!("extracting {input:?} to {output:?}");
+    let source_file = File::open(input).wrap_err("failed to open input file")?;
+    let source_size = source_file.metadata()?.size() as usize;
+    let buffered_source = BufReader::new(source_file);
+    let decoder_input = ProgressMonitor::new(buffered_source, source_size);
+    let mut output: File =
+        File::create(output).wrap_err("failed to open output file for writing")?;
+
+    let mut decoder = ruzstd::decoding::StreamingDecoder::new(decoder_input)?;
+
+    std::io::copy(&mut decoder, &mut output)?;
+
+    info!(
+        "inflated {} ——> {}",
+        fmt_size(source_size as f64),
+        fmt_size(output.metadata()?.len() as f64),
+    );
+    Ok(())
+}
+
+/// A temporary utility function that appends a file extension
+/// to the provided path buf.
+///
+/// Pending removal when our MSRV reaches 1.91 so we can use
+///
+/// <https://doc.rust-lang.org/std/path/struct.PathBuf.html#method.add_extension>
+fn add_extension<P: AsRef<Path>>(path: &Path, extension: P) -> PathBuf {
+    let mut output = path.to_path_buf().into_os_string();
+    output.push(extension.as_ref().as_os_str());
+
+    output.into()
+}
+
+#[cfg(test)]
+mod tests {
+    use std::path::PathBuf;
+
+    use crate::add_extension;
+
+    #[test]
+    fn extension_added() {
+        let filename = PathBuf::from("README.md");
+        assert_eq!(
+            add_extension(&filename, ".zst"),
+            PathBuf::from("README.md.zst")
+        );
+    }
+}