Implement rudimentary compression support (#65)

Adds support for a basic encoder that just packs the input into raw blocks

Author: zleyyij

Cargo.toml

@@ -24,6 +24,7 @@ alloc = { version = "1.0.0", optional = true, package = "rustc-std-workspace-all
 [dev-dependencies]
 criterion = "0.5"
 rand = { version = "0.8.5", features = ["small_rng"] }
+zstd = "0.13.2"
 
 [features]
 default = ["hash", "std"]

Readme.md

@@ -6,17 +6,22 @@
 
 # What is this
 
-A feature-complete decoder for the zstd compression format as defined in: [This document](https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md).
+A pure Rust implementation of the Zstandard compression algorithm, as defined in [this document](https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md).
 
-It is NOT a compressor. I don't plan on implementing that part either, at least not in the near future. (If someone is motivated enough I will of course accept a pull-request!)
+This crate contains a fully operational implementation of the decompression portion of the standard.
 
-This crate might look like it is not active, this is because there isn't really anything to do anymore, unless a bug is found or a new API feature is requested. I will of course respond to and look into issues!
+*Work has started on a compressor, but it has not reached a point where the compressor provides any real function.* (CONTRIBUTORS WELCOME)
+
+This crate is currently actively maintained.
 
 # Current Status
 
 Feature complete on the decoder side. In terms of speed it is still behind the original C implementation which has a rust binding located [here](https://github.com/gyscos/zstd-rs).
 
-Actively maintained but no new features currently planned. If you have suggestions please open an issue and I'll consider it.
+On the compression side:
+- [x] Support for generating raw, uncompressed frames
+- [ ] Support for generating RLE compressed blocks
+- [ ] Support for generating compressed blocks at any compression level
 
 ## Speed
 

benches/decode_all.rs

@@ -3,7 +3,7 @@ use ruzstd::FrameDecoder;
 
 fn criterion_benchmark(c: &mut Criterion) {
     let mut fr = FrameDecoder::new();
-    let mut target_slice = &mut vec![0u8;1024 * 1024 * 200];
+    let mut target_slice = &mut vec![0u8; 1024 * 1024 * 200];
     let src = include_bytes!("../decodecorpus_files/z000033.zst");
 
     c.bench_function("decode_all_slice", |b| {

fuzz/Cargo.toml

@@ -24,6 +24,10 @@ members = ["."]
 name = "decode"
 path = "fuzz_targets/decode.rs"
 
+[[bin]]
+name = "encode"
+path = "fuzz_targets/encode.rs"
+
 [[bin]]
 name = "interop"
 path = "fuzz_targets/interop.rs"

fuzz/fuzz_targets/encode.rs

@@ -0,0 +1,11 @@
+#![no_main]
+#[macro_use] extern crate libfuzzer_sys;
+extern crate ruzstd;
+use ruzstd::encoding::{FrameCompressor, CompressionLevel};
+
+fuzz_target!(|data: &[u8]| {
+    let mut content = data;
+    let mut compressor = FrameCompressor::new(data, CompressionLevel::Uncompressed);
+    let mut output = Vec::new();
+    compressor.compress(&mut output);
+});

fuzz/fuzz_targets/interop.rs

@@ -31,16 +31,42 @@ fn encode_zstd(data: &[u8]) -> Result<Vec<u8>, std::io::Error> {
     zstd::stream::encode_all(std::io::Cursor::new(data), 3)
 }
 
+fn encode_ruzstd_uncompressed(data: &mut dyn std::io::Read) -> Vec<u8> {
+    let mut input = Vec::new();
+    data.read_to_end(&mut input).unwrap();
+    let mut compressor = ruzstd::encoding::FrameCompressor::new(&input, ruzstd::encoding::CompressionLevel::Uncompressed);
+    let mut output = Vec::new();
+    compressor.compress(&mut output);
+    output
+}
+
+fn decode_zstd(data: &[u8]) -> Result<Vec<u8>, std::io::Error> {
+    let mut output = Vec::new();
+    zstd::stream::copy_decode(data, &mut output)?;
+    Ok(output)
+}
+
 fuzz_target!(|data: &[u8]| {
+    // Decoding
     let compressed = encode_zstd(data).unwrap();
     let decoded = decode_ruzstd(&mut compressed.as_slice());
     let decoded2 = decode_ruzstd_writer(&mut compressed.as_slice());
     assert!(
         decoded == data,
-        "Decoded data did not match the original input"
+        "Decoded data did not match the original input during decompression"
     );
     assert_eq!(
         decoded2, data,
-        "Decoded data did not match the original input"
+        "Decoded data did not match the original input during decompression"
+    );
+
+    // Encoding
+    // Uncompressed encoding
+    let mut input = data;
+    let compressed = encode_ruzstd_uncompressed(&mut input);
+    let decoded = decode_zstd(&compressed).unwrap();
+    assert_eq!(
+        decoded, data,
+        "Decoded data did not match the original input during compression"
     );
 });

src/decoding/block_decoder.rs

@@ -454,6 +454,8 @@ impl BlockDecoder {
         Ok(())
     }
 
+    /// Reads 3 bytes from the provided reader and returns
+    /// the deserialized header and the number of bytes read.
     pub fn read_block_header(
         &mut self,
         mut r: impl Read,