clearly describe how to perform fuzzing, and make a fuzz/ workspace

- add publish = false to fuzz subcrates
- move fuzzing to a subdirectory in order to share a workspace
- break out separate shards for specific named features, but only for the top-level workspace

Author: cosmicexplorer

.github/workflows/ci.yaml

@@ -22,13 +22,12 @@ jobs:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
     strategy:
       matrix:
+        workspace: ['Cargo.toml', 'fuzz/Cargo.toml']
         os: [ubuntu-latest, macOS-latest, windows-latest]
         rustalias: [stable, nightly, msrv]
         feature_flag:
         - "--all-features"
         - "--no-default-features"
-        - "--no-default-features --features deflate-flate2-zlib-rs"
-        - "--no-default-features --features deflate-zopfli"
         - ""
         include:
         - rustalias: stable
@@ -37,24 +36,34 @@ jobs:
           rust: '1.83'
         - rustalias: nightly
           rust: nightly
+        # Break out a separate test shard for specific dependencies on their own.
+        - feature_flag: "--no-default-features --features deflate-flate2-zlib-rs"
+          workspace: 'Cargo.toml'
+        - feature_flag: "--no-default-features --features deflate-zopfli"
+          workspace: 'Cargo.toml'
     name: 'Build and test ${{ matrix.feature_flag }}: ${{ matrix.os }}, ${{ matrix.rustalias }}'
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v5
     - run: rustup toolchain add ${{ matrix.rust }} && rustup default ${{ matrix.rust }}
 
-    - run: cargo check --all ${{ matrix.feature_flag }} --bins --examples
-    - run: cargo test --all ${{ matrix.feature_flag }}
+    - run: cargo check --manifest-path ${{ github.workspace }}/${{ matrix.workspace }} --all ${{ matrix.feature_flag }} --bins --examples
+    - run: cargo test --manifest-path ${{ github.workspace }}/${{ matrix.workspace }} --all ${{ matrix.feature_flag }}
   miri:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
     strategy:
       matrix:
+        workspace: ['Cargo.toml', 'fuzz/Cargo.toml']
         feature_flag:
         - "--all-features"
         - "--no-default-features"
-        - "--no-default-features --features deflate-flate2-zlib-rs"
-        - "--no-default-features --features deflate-zopfli"
         - ""
+        include:
+          # Break out a separate test shard for specific dependencies on their own.
+        - feature_flag: "--no-default-features --features deflate-flate2-zlib-rs"
+          workspace: 'Cargo.toml'
+        - feature_flag: "--no-default-features --features deflate-zopfli"
+          workspace: 'Cargo.toml'
     name: 'Miri ${{ matrix.feature_flag }}'
     runs-on: ubuntu-latest
     steps:
@@ -64,18 +73,21 @@ jobs:
     - run: rustup toolchain add --force-non-host stable-s390x-unknown-linux-gnu
     - run: rustup target add s390x-unknown-linux-gnu --toolchain stable-s390x-unknown-linux-gnu
     - run: rustup component add --toolchain nightly-x86_64-unknown-linux-gnu miri
-    - run: cargo +nightly miri test --target s390x-unknown-linux-gnu --all ${{ matrix.feature_flag }} --bins --examples
+    - run: cargo +nightly miri test --manifest-path ${{ github.workspace }}/${{ matrix.workspace }} --target s390x-unknown-linux-gnu --all ${{ matrix.feature_flag }} --bins --examples
   cargo_semver:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
     strategy:
       matrix:
+        # Only do semver checks on the released library.
+        workspace: ['Cargo.toml']
         feature_group: ["all-features", "default-features", "only-explicit-features"]
     name: 'Semver checks: ${{ matrix.feature_group }}'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
       - uses: obi1kenobi/cargo-semver-checks-action@v2
         with:
+          manifest-path: ${{ github.workspace }}/${{ matrix.workspace }}
           feature-group: ${{ matrix.feature_group }}
   cargo_fmt:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
@@ -86,38 +98,41 @@ jobs:
       - run: rustup toolchain add nightly && rustup default nightly && rustup component add rustfmt
       - name: fmt
         run: cargo fmt --all -- --check
-      - name: fmt fuzz_read
-        run: cargo fmt --manifest-path fuzz_read/Cargo.toml -- --check
-      - name: fmt fuzz_write
-        run: cargo fmt --manifest-path fuzz_write/Cargo.toml -- --check
+      - name: fmt fuzz
+        run: cargo fmt --all --manifest-path ${{ github.workspace }}/fuzz/Cargo.toml -- --check
 
   check_minimal_versions:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
+    strategy:
+      matrix:
+        # Only check minimal versions for the released library.
+        workspace: ['Cargo.toml']
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v5
       - run: rustup toolchain add nightly && rustup default nightly
 
       - name: resolve minimal versions
-        run: cargo -Z minimal-versions update
+        run: cargo -Z minimal-versions update --manifest-path ${{ github.workspace }}/${{ matrix.workspace }}
       - name: check
-        run: cargo check --all-features
+        run: cargo check --all-features --manifest-path ${{ github.workspace }}/${{ matrix.workspace }}
       - name: test
-        run: cargo test --all-features
+        run: cargo test --all-features --manifest-path ${{ github.workspace }}/${{ matrix.workspace }}
 
   style_and_docs:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
     strategy:
       matrix:
+        workspace: ['Cargo.toml', 'fuzz/Cargo.toml']
         feature_flag: ["--all-features", "--no-default-features", ""]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v5
     - run: rustup toolchain add nightly && rustup default nightly && rustup component add clippy
 
-    - run: cargo clippy --all-targets ${{ matrix.feature_flag }} -- -D warnings
-    - run: cargo doc --no-deps ${{ matrix.feature_flag }}
+    - run: cargo clippy --workspace ${{ matrix.feature_flag }} --manifest-path ${{ github.workspace }}/${{ matrix.workspace }} -- -D warnings
+    - run: cargo doc --no-deps --workspace ${{ matrix.feature_flag }} --manifest-path ${{ github.workspace }}/${{ matrix.workspace }}
 
   fuzz_read:
     runs-on: ubuntu-latest
@@ -136,16 +151,16 @@ jobs:
       - name: cargo afl system-config
         run: cargo afl system-config
       - name: clippy
-        run: cargo afl clippy --all-features --manifest-path ${{ github.workspace }}/fuzz_read/Cargo.toml -- -D warnings
+        run: cargo afl clippy --all-features -p fuzz_read --manifest-path ${{ github.workspace }}/fuzz/Cargo.toml -- -D warnings
       - name: compile fuzz
-        run: cargo afl build --all-features --manifest-path ${{ github.workspace }}/fuzz_read/Cargo.toml
+        run: cargo afl build --all-features -p fuzz_read --manifest-path ${{ github.workspace }}/fuzz/Cargo.toml
       - name: run fuzz
         timeout-minutes: 130
-        run: cargo afl fuzz -i ${{ github.workspace }}/fuzz_read/in -o out -V 7200 -- ${{ github.workspace }}/fuzz_read/target/debug/fuzz_read
+        run: cargo afl fuzz -i ${{ github.workspace }}/fuzz/read/in -o out -V 7200 -- ${{ github.workspace }}/fuzz/target/debug/fuzz_read
       - name: Minimize corpus
-        run: cargo afl cmin -i out/default/queue -o out_cmin -- ${{ github.workspace }}/fuzz_read/target/debug/fuzz_read
+        run: cargo afl cmin -i out/default/queue -o out_cmin -- ${{ github.workspace }}/fuzz/target/debug/fuzz_read
       - name: Report coverage
-        run: cargo afl showmap -C -i out -o map -- ${{ github.workspace }}/fuzz_read/target/debug/fuzz_read
+        run: cargo afl showmap -C -i out -o map -- ${{ github.workspace }}/fuzz/target/debug/fuzz_read
       - run: sudo apt install rename
         if: always()
       - name: Rename files
@@ -188,14 +203,14 @@ jobs:
       - name: cargo afl system-config
         run: cargo afl system-config
       - name: clippy
-        run: cargo afl clippy --no-default-features --manifest-path ${{ github.workspace }}/fuzz_read/Cargo.toml -- -D warnings
+        run: cargo afl clippy --no-default-features -p fuzz_read --manifest-path ${{ github.workspace }}/fuzz/Cargo.toml -- -D warnings
       - name: compile fuzz
-        run: cargo afl build --manifest-path ${{ github.workspace }}/fuzz_read/Cargo.toml
+        run: cargo afl build -p fuzz_read --manifest-path ${{ github.workspace }}/fuzz/Cargo.toml
       - name: run fuzz
         timeout-minutes: 130
-        run: cargo afl fuzz -i ${{ github.workspace }}/fuzz_read/in -o out -V 7200 -- ${{ github.workspace }}/fuzz_read/target/debug/fuzz_read
+        run: cargo afl fuzz -i ${{ github.workspace }}/fuzz/read/in -o out -V 7200 -- ${{ github.workspace }}/fuzz/target/debug/fuzz_read
       - name: Report coverage
-        run: cargo afl showmap -C -i out -o map -- ${{ github.workspace }}/fuzz_read/target/debug/fuzz_read
+        run: cargo afl showmap -C -i out -o map -- ${{ github.workspace }}/fuzz/target/debug/fuzz_read
       - run: sudo apt install rename
         if: always()
       - name: Rename files
@@ -233,16 +248,16 @@ jobs:
       - name: cargo afl system-config
         run: cargo afl system-config
       - name: clippy
-        run: cargo afl clippy --all-features --manifest-path ${{ github.workspace }}/fuzz_write/Cargo.toml -- -D warnings
+        run: cargo afl clippy --all-features -p fuzz_write --manifest-path ${{ github.workspace }}/fuzz/Cargo.toml -- -D warnings
       - name: compile fuzz
-        run: cargo afl build --all-features --manifest-path ${{ github.workspace }}/fuzz_write/Cargo.toml
+        run: cargo afl build --all-features -p fuzz_write --manifest-path ${{ github.workspace }}/fuzz/Cargo.toml
       - name: run fuzz
         timeout-minutes: 130
-        run: cargo afl fuzz -i ${{ github.workspace }}/fuzz_write/in -o out -V 7200 -x ${{ github.workspace }}/fuzz_write/fuzz.dict -- ${{ github.workspace }}/fuzz_write/target/debug/fuzz_write
+        run: cargo afl fuzz -i ${{ github.workspace }}/fuzz/write/in -o out -V 7200 -x ${{ github.workspace }}/fuzz/write/fuzz.dict -- ${{ github.workspace }}/fuzz/target/debug/fuzz_write
       - name: Minimize corpus
-        run: cargo afl cmin -i out/default/queue -o out_cmin -- ${{ github.workspace }}/fuzz_write/target/debug/fuzz_write
+        run: cargo afl cmin -i out/default/queue -o out_cmin -- ${{ github.workspace }}/fuzz/target/debug/fuzz_write
       - name: Report coverage
-        run: cargo afl showmap -C -i out -o map -- ${{ github.workspace }}/fuzz_write/target/debug/fuzz_write
+        run: cargo afl showmap -C -i out -o map -- ${{ github.workspace }}/fuzz/target/debug/fuzz_write
       - run: sudo apt install rename
         if: always()
       - name: Rename files
@@ -285,14 +300,14 @@ jobs:
       - name: cargo afl system-config
         run: cargo afl system-config
       - name: clippy
-        run: cargo afl clippy --no-default-features --manifest-path ${{ github.workspace }}/fuzz_write/Cargo.toml -- -D warnings
+        run: cargo afl clippy --no-default-features -p fuzz_write --manifest-path ${{ github.workspace }}/fuzz/Cargo.toml -- -D warnings
       - name: compile fuzz
-        run: cargo afl build --all-features --manifest-path ${{ github.workspace }}/fuzz_write/Cargo.toml
+        run: cargo afl build --all-features -p fuzz_write --manifest-path ${{ github.workspace }}/fuzz/Cargo.toml
       - name: run fuzz
         timeout-minutes: 130
-        run: cargo afl fuzz -i ${{ github.workspace }}/fuzz_write/in -o out -V 7200 -x ${{ github.workspace }}/fuzz_write/fuzz.dict -- ${{ github.workspace }}/fuzz_write/target/debug/fuzz_write
+        run: cargo afl fuzz -i ${{ github.workspace }}/fuzz/write/in -o out -V 7200 -x ${{ github.workspace }}/fuzz/write/fuzz.dict -- ${{ github.workspace }}/fuzz/target/debug/fuzz_write
       - name: Report coverage
-        run: cargo afl showmap -C -i out -o map -- ${{ github.workspace }}/fuzz_write/target/debug/fuzz_write
+        run: cargo afl showmap -C -i out -o map -- ${{ github.workspace }}/fuzz/target/debug/fuzz_write
       - run: sudo apt install rename
         if: always()
       - name: Rename files

Cargo.toml

@@ -13,6 +13,7 @@ keywords = ["zip", "archive", "compression"]
 # Any change to rust-version must be reflected also in `README.md` and `.github/workflows/ci.yaml`.
 # The MSRV policy is documented in `README.md`.
 rust-version = "1.83.0"
+categories = ["compression", "filesystem", "parser-implementations"]
 description = """
 Library to support the reading and writing of zip files.
 """
@@ -23,10 +24,8 @@ exclude = ["tests/**", "examples/**", ".github/**", "fuzz_read/**", "fuzz_write/
 all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
-[workspace.dependencies]
-time = { version = "0.3.37", default-features = false }
-
 [dependencies]
+arbitrary = { version = "1.4", features = ["derive"], optional = true }
 aes = { version = "0.8", optional = true }
 bzip2 = { version = "0.6.0", optional = true }
 chrono = { version = "^0.4.27", optional = true }
@@ -42,7 +41,7 @@ nt-time = { version = "0.10.6", default-features = false, optional = true }
 ppmd-rust = { version = "1.2", optional = true }
 pbkdf2 = { version = "0.12", optional = true }
 sha1 = { version = "0.10", optional = true }
-time = { workspace = true, optional = true, features = [
+time = { version = "0.3.37", default-features = false, optional = true, features = [
     "std",
 ] }
 zeroize = { version = "1.8", optional = true, features = ["zeroize_derive"] }
@@ -52,19 +51,17 @@ deflate64 = { version = "0.1.9", optional = true }
 lzma-rust2 = { version = "0.13", optional = true, default-features = false, features = ["std", "encoder", "optimization", "xz"] }
 bitstream-io =  { version = "4.5.0", optional = true }
 
-[target.'cfg(fuzzing)'.dependencies]
-arbitrary = { version = "1.4.1", features = ["derive"] }
-
 [dev-dependencies]
 bencher = "0.1.5"
 getrandom = { version = "0.3.1", features = ["wasm_js", "std"] }
 walkdir = "2.5"
-time = { workspace = true, features = ["formatting", "macros"] }
+time = { version = "0.3", features = ["formatting", "macros"] }
 anyhow = "1.0.95"
 clap = { version = "=4.4.18", features = ["derive"] }
 tempfile = "3.15"
 
 [features]
+arbitrary = ["dep:arbitrary"]
 aes-crypto = ["dep:aes", "dep:constant_time_eq", "hmac", "pbkdf2", "sha1", "getrandom", "zeroize"]
 chrono = ["dep:chrono"]
 _deflate-any = []

README.md

@@ -1,14 +1,12 @@
 zip
-========
+===
 
 [![Build Status](https://github.com/zip-rs/zip2/actions/workflows/ci.yaml/badge.svg)](https://github.com/Pr0methean/zip/actions?query=branch%3Amaster+workflow%3ACI)
 [![Crates.io version](https://img.shields.io/crates/v/zip.svg)](https://crates.io/crates/zip)
 
 [Documentation](https://docs.rs/zip/latest/zip/)
 
-Info
-----
-
+# Info
 
 A zip library for rust which supports reading and writing of simple ZIP files. Formerly hosted at
 https://github.com/zip-rs/zip2.
@@ -28,8 +26,7 @@ Currently unsupported zip extensions:
 
 * Multi-disk
 
-Features
---------
+# Features
 
 The features available are:
 
@@ -53,17 +50,15 @@ The features available are:
 
 By default `aes-crypto`, `bzip2`, `deflate`, `deflate64`, `lzma`, `ppmd`, `time`, `xz` and `zstd` are enabled.
 
-MSRV
-----
+# MSRV
 
 Our current Minimum Supported Rust Version is **1.83**. When adding features,
 we will follow these guidelines:
 
 - We will always support a minor Rust version that has been stable for at least 6 months.
 - Any change to the MSRV will be accompanied with a **minor** version bump.
 
-Examples
---------
+# Examples
 
 See the [examples directory](examples) for:
 
@@ -74,10 +69,9 @@ See the [examples directory](examples) for:
 * How to read a zip from the standard input.
 * How to append a directory to an existing archive
 
-Fuzzing
--------
+# Fuzzing
 
-Fuzzing support is through [cargo afl](https://rust-fuzz.github.io/book/afl.html). To install cargo afl:
+Fuzzing support is through [`cargo afl`](https://rust-fuzz.github.io/book/afl/tutorial.html). To install `cargo afl`:
 
 ```bash
 cargo install cargo-afl
@@ -86,13 +80,65 @@ cargo install cargo-afl
 To start fuzzing zip extraction:
 
 ```bash
-cargo +nightly afl build --all-features --manifest-path fuzz_read/Cargo.toml
-cargo +nightly afl run fuzz_read/target/debug/fuzz_read
+mkdir -vp fuzz-read-out
+cargo afl build --manifest-path=fuzz/Cargo.toml --all-features -p fuzz_read
+# Curated input corpus:
+cargo afl fuzz -i fuzz/read/in -o fuzz-read-out fuzz/target/debug/fuzz_read
+# Test data files:
+cargo afl fuzz -i tests/data -e zip -o fuzz-read-out fuzz/target/debug/fuzz_read
 ```
 
 To start fuzzing zip creation:
 
 ```bash
-cargo +nightly afl build --all-features --manifest-path fuzz_write/Cargo.toml
-cargo +nightly afl run fuzz_write/target/debug/fuzz_write
+mkdir -vp fuzz-write-out
+cargo afl build --manifest-path=fuzz/Cargo.toml --all-features -p fuzz_write
+# Curated input corpus and dictionary schema:
+cargo afl fuzz -x fuzz/write/fuzz.dict -i fuzz/write/in -o fuzz-write-out fuzz/target/debug/fuzz_write
+```
+
+## Fuzzing stdio
+
+The read and write fuzzers can also receive input over stdin for one-off validation. Note here that the fuzzers can be configured to build in support for DEFLATE, or not:
+```bash
+# Success, no output:
+cargo run --manifest-path=fuzz/Cargo.toml --quiet --all-features -p fuzz_read <tests/data/deflate64.zip
+# Error, without deflate64 support:
+cargo run --manifest-path=fuzz/Cargo.toml --quiet -p fuzz_read <tests/data/deflate64.zip
+
+thread 'main' (537304) panicked at fuzz_read/src/main.rs:40:36:
+called `Result::unwrap()` on an `Err` value: UnsupportedArchive("Compression method not supported")
+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
+```
+
+The zip creation fuzzer will try to print out a description of the kind of input it translated the input bytes into:
+```bash
+# This is an empty input case:
+<fuzz/write/in/id-000000,time-0,execs-0,orig-0011743621118ab6c5278ffbb8fd14bddd8369ee.min \
+  cargo run --manifest-path=fuzz/Cargo.toml --quiet --all-features -p fuzz_write
+# This input was translated into one or more test cases:
+<fuzz/write/in/id-000000,time-0,execs-0,orig-0011743621118ab6c5278ffbb8fd14bddd8369ee.min \
+  cargo run --manifest-path=fuzz/Cargo.toml --quiet -p fuzz_write
+writer.start_file_from_path("", FileOptions { compression_method: Stored, compression_level: None, last_modified_time: DateTime::from_date_and_time(2048, 1, 1, 0, 0, 0)?, permissions: None, large_file: false, encrypt_with: None, extended_options: ExtendedFileOptions {extra_data: vec![].into(), central_extra_data: vec![].into()}, alignment: 0 })?;
+writer.write_all(&[])?;
+writer
+let _ = writer.finish_into_readable()?;
+```
+
+The zip creation fuzzer uses [`arbitrary::Unstructured`](https://docs.rs/arbitrary/latest/arbitrary/struct.Unstructured.html) to convert bytes over stdin to random inputs, so it can be triggered with other sources of random input:
+```bash
+# Usually, the random input is translated into zero test cases:
+head -c50 /dev/random | cargo run --manifest-path=fuzz/Cargo.toml --quiet --all-features -p fuzz_write
+# Sometimes, one or more test cases are generated and successfully evaluated:
+head -c50 /dev/random | cargo run --manifest-path=fuzz/Cargo.toml --quiet --all-features -p fuzz_write
+writer.set_raw_comment([20, 202])?;
+let mut writer = ZipWriter::new_append(writer.finish()?)?;
+let sub_writer = {
+let mut initial_junk = Cursor::new(vec![106]);
+initial_junk.seek(SeekFrom::End(0))?;
+                          let mut writer = ZipWriter::new(initial_junk);
+writer
+};
+writer.merge_archive(sub_writer.finish_into_readable()?)?;
+let mut writer = ZipWriter::new_append(writer.finish()?)?;
 ```