have decompress actually decompress arbitrary bytes

Author: folkertdev

.github/workflows/checks.yaml

@@ -236,12 +236,10 @@ jobs:
         include:
           - fuzz_target: decompress
             corpus: "bzip2-files/compressed"
-            features: '--no-default-features --features="disable-checksum"'
-            flags: fuzz-decompress
+            features: '--no-default-features --features="disable-checksum,keep-invalid-in-corpus"'
           - fuzz_target: compress
             corpus: ""
             features: ''
-            flags: fuzz-compress
     steps:
       - name: Checkout sources
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
@@ -259,11 +257,13 @@ jobs:
       - name: Download custom decompression corpus
         if: ${{ contains(matrix.corpus, 'bzip2-files') }}
         run: |
-          wget https://github.com/trifectatechfoundation/compression-corpus/releases/download/2025-04-14-151155/bzip2-files.zip
+          wget https://github.com/trifectatechfoundation/compression-corpus/releases/download/2025-04-24-180855/bzip2-files.zip
           unzip bzip2-files.zip -d bzip2-files
       - name: Run `cargo fuzz`
         env:
           RUST_BACKTRACE: "1"
+          # prevents `cargo fuzz coverage` from rebuilding everything
+          RUSTFLAGS: "-C instrument-coverage"
         run: |
           cargo fuzz run ${{matrix.features}} ${{matrix.fuzz_target}} ${{matrix.corpus}} -- -max_total_time=20
       - name: Fuzz codecov
@@ -273,7 +273,7 @@ jobs:
               target/$(rustc --print host-tuple)/coverage/$(rustc --print host-tuple)/release/${{matrix.fuzz_target}} \
               -instr-profile=fuzz/coverage/${{matrix.fuzz_target}}/coverage.profdata \
               --format=lcov \
-              -ignore-filename-regex="\.cargo|\.rustup|fuzz/src|test-libbz2" > lcov.info
+              -ignore-filename-regex="\.cargo|\.rustup|fuzz_targets|test-libbz2" > lcov.info
       - name: List the corpus
         if: ${{ contains(matrix.corpus, 'bzip2-files') }}
         run: |
@@ -283,7 +283,7 @@ jobs:
         with:
           files: ./lcov.info
           fail_ci_if_error: false
-          flags: ${{ matrix.flags }}
+          flags: fuzz-${{ matrix.fuzz_target }}
           token: ${{ secrets.CODECOV_TOKEN }}
           name: fuzz
 

fuzz/Cargo.toml

@@ -16,6 +16,7 @@ default = ["rust-allocator"]
 c-allocator = ["libbz2-rs-sys/c-allocator"]
 rust-allocator = ["libbz2-rs-sys/rust-allocator"]
 disable-checksum = ["libbz2-rs-sys/__internal-fuzz-disable-checksum"]
+keep-invalid-in-corpus = [] # For code coverage (on CI), we want to keep inputs that triggered the error branches
 
 [dependencies.libfuzzer-sys]
 version = "0.4"
@@ -64,3 +65,9 @@ name = "compress"
 path = "fuzz_targets/compress.rs"
 test = false
 doc = false
+
+[[bin]]
+name = "end_to_end"
+path = "fuzz_targets/end_to_end.rs"
+test = false
+doc = false

fuzz/fuzz_targets/decompress.rs

@@ -1,52 +1,69 @@
 #![no_main]
-use libbz2_rs_sys::BZ_OK;
-use libfuzzer_sys::fuzz_target;
 
-fn decompress_help(input: &[u8]) -> Vec<u8> {
-    let mut dest_vec = vec![0u8; 1 << 16];
+use libbz2_rs_sys::{bz_stream, BZ2_bzDecompress, BZ2_bzDecompressEnd, BZ2_bzDecompressInit};
+use libfuzzer_sys::{fuzz_target, Corpus};
 
-    let mut dest_len = dest_vec.len() as _;
-    let dest = dest_vec.as_mut_ptr();
+fuzz_target!(|input: &[u8]| -> Corpus { decompress_help(input) });
 
-    let source = input.as_ptr();
-    let source_len = input.len() as _;
+fn decompress_help(source: &[u8]) -> Corpus {
+    let mut strm: bz_stream = bz_stream::zeroed();
 
-    let err = unsafe { test_libbz2_rs_sys::decompress_rs(dest, &mut dest_len, source, source_len) };
+    // Pick either small or fast based on a byte of input data.
+    let small = source.get(source.len() / 2).map_or(false, |v| v % 2 == 0);
 
-    if err != BZ_OK {
-        panic!("error {:?}", err);
-    }
-
-    dest_vec.truncate(dest_len as usize);
+    let ret = unsafe { BZ2_bzDecompressInit(&mut strm, 0, small as _) };
+    assert_eq!(ret, libbz2_rs_sys::BZ_OK);
 
-    dest_vec
-}
+    // Small enough to hit interesting cases, but large enough to hit the fast path
+    let chunk_size = 16;
 
-fuzz_target!(|data: Vec<u8>| {
-    let mut length = 8 * 1024;
-    let mut deflated = vec![0; length as usize];
-
-    let error = unsafe {
-        test_libbz2_rs_sys::compress_c(
-            deflated.as_mut_ptr().cast(),
-            &mut length,
-            data.as_ptr().cast(),
-            data.len() as _,
-            9,
-        )
+    // For code coverage (on CI), we want to keep inputs that triggered the error
+    // branches, to get an accurate picture of what error paths we actually hit.
+    //
+    // It helps that on CI we start with a corpus of valid files: a mutation of such an
+    // input is not a sequence of random bytes, but rather quite close to correct and
+    // hence likely to hit interesting error conditions.
+    let invalid_input = if cfg!(feature = "keep-invalid-in-corpus") {
+        Corpus::Keep
+    } else {
+        Corpus::Reject
     };
 
-    assert_eq!(error, BZ_OK);
+    let mut output = vec![0u8; source.len()];
+    let output_len = output.len() as _;
 
-    deflated.truncate(length as _);
+    strm.next_out = output.as_mut_ptr().cast::<core::ffi::c_char>();
+    strm.avail_out = output_len;
 
-    let output = decompress_help(&deflated);
+    for chunk in source.chunks(chunk_size) {
+        strm.next_in = chunk.as_ptr() as *mut core::ffi::c_char;
+        strm.avail_in = chunk.len() as _;
 
-    if output != data {
-        let path = std::env::temp_dir().join("deflate.txt");
-        std::fs::write(&path, &data).unwrap();
-        eprintln!("saved input file to {path:?}");
+        match unsafe { BZ2_bzDecompress(&mut strm) } {
+            libbz2_rs_sys::BZ_STREAM_END => {
+                break;
+            }
+            libbz2_rs_sys::BZ_OK => {
+                continue;
+            }
+            libbz2_rs_sys::BZ_FINISH_OK | libbz2_rs_sys::BZ_OUTBUFF_FULL => {
+                let add_space: u32 = Ord::max(1024, output.len().try_into().unwrap());
+                output.resize(output.len() + add_space as usize, 0);
+
+                // If resize() reallocates, it may have moved in memory.
+                strm.next_out = output.as_mut_ptr().cast::<core::ffi::c_char>();
+                strm.avail_out += add_space;
+            }
+            _ => {
+                unsafe { BZ2_bzDecompressEnd(&mut strm) };
+                return invalid_input;
+            }
+        }
     }
 
-    assert_eq!(output, data);
-});
+    let err = unsafe { BZ2_bzDecompressEnd(&mut strm) };
+    match err {
+        libbz2_rs_sys::BZ_OK => Corpus::Keep,
+        _ => invalid_input,
+    }
+}

fuzz/fuzz_targets/end_to_end.rs

@@ -0,0 +1,52 @@
+#![no_main]
+use libbz2_rs_sys::BZ_OK;
+use libfuzzer_sys::fuzz_target;
+
+fn decompress_help(input: &[u8]) -> Vec<u8> {
+    let mut dest_vec = vec![0u8; 1 << 16];
+
+    let mut dest_len = dest_vec.len() as _;
+    let dest = dest_vec.as_mut_ptr();
+
+    let source = input.as_ptr();
+    let source_len = input.len() as _;
+
+    let err = unsafe { test_libbz2_rs_sys::decompress_rs(dest, &mut dest_len, source, source_len) };
+
+    if err != BZ_OK {
+        panic!("error {:?}", err);
+    }
+
+    dest_vec.truncate(dest_len as usize);
+
+    dest_vec
+}
+
+fuzz_target!(|data: Vec<u8>| {
+    let mut length = 8 * 1024;
+    let mut deflated = vec![0; length as usize];
+
+    let error = unsafe {
+        test_libbz2_rs_sys::compress_c(
+            deflated.as_mut_ptr().cast(),
+            &mut length,
+            data.as_ptr().cast(),
+            data.len() as _,
+            9,
+        )
+    };
+
+    assert_eq!(error, BZ_OK);
+
+    deflated.truncate(length as _);
+
+    let output = decompress_help(&deflated);
+
+    if output != data {
+        let path = std::env::temp_dir().join("deflate.txt");
+        std::fs::write(&path, &data).unwrap();
+        eprintln!("saved input file to {path:?}");
+    }
+
+    assert_eq!(output, data);
+});