fuzzing.yml

name: Fuzzing

# spell-checker:ignore fuzzer dtolnay Swatinem

on:
  pull_request:
  push:
    branches:
      - '*'

permissions:
  contents: read # to fetch code (actions/checkout)

# End the current execution if there is a new changeset in the PR.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  uufuzz-examples:
    name: Build and test uufuzz examples
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v5
      with:
        persist-credentials: false
    - uses: dtolnay/rust-toolchain@stable
    - uses: Swatinem/rust-cache@v2
      with:
        shared-key: "uufuzz-cache-key"
        cache-directories: "fuzz/target"
    - name: Build uufuzz library
      run: |
        cd fuzz/uufuzz
        cargo build --release
    - name: Run uufuzz tests
      run: |
        cd fuzz/uufuzz
        cargo test --lib
    - name: Build and run uufuzz examples
      run: |
        cd fuzz/uufuzz
        echo "Building all examples..."
        cargo build --examples --release

        # Run all examples except integration_testing (which has FD issues in CI)
        for example in examples/*.rs; do
          example_name=$(basename "$example" .rs)
          if [ "$example_name" != "integration_testing" ]; then
            cargo run --example "$example_name" --release
          fi
        done

  fuzz-build:
    name: Build the fuzzers
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v5
      with:
        persist-credentials: false
    - uses: dtolnay/rust-toolchain@nightly
    - name: Install `cargo-fuzz`
      run: cargo install cargo-fuzz
    - uses: Swatinem/rust-cache@v2
      with:
        shared-key: "cargo-fuzz-cache-key"
        cache-directories: "fuzz/target"
    - name: Run `cargo-fuzz build`
      run: cargo +nightly fuzz build

  fuzz-run:
    needs: fuzz-build
    name: Fuzz
    runs-on: ubuntu-latest
    timeout-minutes: 5
    env:
      RUN_FOR: 60
    strategy:
      matrix:
        test-target:
          - { name: fuzz_test, should_pass: true }
          # https://github.com/uutils/coreutils/issues/5311
          - { name: fuzz_date, should_pass: false }
          - { name: fuzz_expr, should_pass: true }
          - { name: fuzz_printf, should_pass: true }
          - { name: fuzz_echo, should_pass: true }
          - { name: fuzz_seq, should_pass: false }
          - { name: fuzz_sort, should_pass: false }
          - { name: fuzz_wc, should_pass: false }
          - { name: fuzz_cut, should_pass: false }
          - { name: fuzz_split, should_pass: false }
          - { name: fuzz_tr, should_pass: false }
          - { name: fuzz_env, should_pass: false }
          - { name: fuzz_cksum, should_pass: false }
          - { name: fuzz_parse_glob, should_pass: true }
          - { name: fuzz_parse_size, should_pass: true }
          - { name: fuzz_parse_time, should_pass: true }
          - { name: fuzz_seq_parse_number, should_pass: true }
          - { name: fuzz_non_utf8_paths, should_pass: true }

    steps:
    - uses: actions/checkout@v5
      with:
        persist-credentials: false
    - uses: dtolnay/rust-toolchain@nightly
    - name: Install `cargo-fuzz`
      run: cargo install cargo-fuzz
    - uses: Swatinem/rust-cache@v2
      with:
        shared-key: "cargo-fuzz-cache-key"
        cache-directories: "fuzz/target"
    - name: Restore Cached Corpus
      uses: actions/cache/restore@v4
      with:
        key: corpus-cache-${{ matrix.test-target.name }}
        path: |
          fuzz/corpus/${{ matrix.test-target.name }}
    - name: Run ${{ matrix.test-target.name }} for XX seconds
      id: run_fuzzer
      shell: bash
      continue-on-error: ${{ !matrix.test-target.name.should_pass }}
      run: |
        mkdir -p fuzz/stats
        STATS_FILE="fuzz/stats/${{ matrix.test-target.name }}.txt"
        cargo +nightly fuzz run ${{ matrix.test-target.name }} -- -max_total_time=${{ env.RUN_FOR }} -timeout=${{ env.RUN_FOR }} -detect_leaks=0 -print_final_stats=1 2>&1 | tee "$STATS_FILE"

        # Extract key stats from the output
        if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then
          RUNS=$(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}')
          echo "runs=$RUNS" >> "$GITHUB_OUTPUT"
        else
          echo "runs=unknown" >> "$GITHUB_OUTPUT"
        fi

        if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then
          EXEC_RATE=$(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}')
          echo "exec_rate=$EXEC_RATE" >> "$GITHUB_OUTPUT"
        else
          echo "exec_rate=unknown" >> "$GITHUB_OUTPUT"
        fi

        if grep -q "stat::new_units_added" "$STATS_FILE"; then
          NEW_UNITS=$(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}')
          echo "new_units=$NEW_UNITS" >> "$GITHUB_OUTPUT"
        else
          echo "new_units=unknown" >> "$GITHUB_OUTPUT"
        fi

        # Save should_pass value to file for summary job to use
        echo "${{ matrix.test-target.should_pass }}" > "fuzz/stats/${{ matrix.test-target.name }}.should_pass"

        # Print stats to job output for immediate visibility
        echo "----------------------------------------"
        echo "FUZZING STATISTICS FOR ${{ matrix.test-target.name }}"
        echo "----------------------------------------"
        echo "Runs:           $(grep -q "stat::number_of_executed_units" "$STATS_FILE" && grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}' || echo "unknown")"
        echo "Execution Rate: $(grep -q "stat::average_exec_per_sec" "$STATS_FILE" && grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}' || echo "unknown") execs/sec"
        echo "New Units:      $(grep -q "stat::new_units_added" "$STATS_FILE" && grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}' || echo "unknown")"
        echo "Expected:       ${{ matrix.test-target.name.should_pass }}"
        if grep -q "SUMMARY: " "$STATS_FILE"; then
          echo "Status:         $(grep "SUMMARY: " "$STATS_FILE" | head -1)"
        else
          echo "Status:         Completed"
        fi
        echo "----------------------------------------"

        # Add summary to GitHub step summary
        echo "### Fuzzing Results for ${{ matrix.test-target.name }}" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "| Metric | Value |" >> $GITHUB_STEP_SUMMARY
        echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY

        if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then
          echo "| Runs | $(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY
        fi

        if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then
          echo "| Execution Rate | $(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}') execs/sec |" >> $GITHUB_STEP_SUMMARY
        fi

        if grep -q "stat::new_units_added" "$STATS_FILE"; then
          echo "| New Units | $(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY
        fi

        echo "| Should pass | ${{ matrix.test-target.should_pass }} |" >> $GITHUB_STEP_SUMMARY

        if grep -q "SUMMARY: " "$STATS_FILE"; then
          echo "| Status | $(grep "SUMMARY: " "$STATS_FILE" | head -1) |" >> $GITHUB_STEP_SUMMARY
        else
          echo "| Status | Completed |" >> $GITHUB_STEP_SUMMARY
        fi

        echo "" >> $GITHUB_STEP_SUMMARY
    - name: Save Corpus Cache
      uses: actions/cache/save@v4
      with:
        key: corpus-cache-${{ matrix.test-target.name }}
        path: |
          fuzz/corpus/${{ matrix.test-target.name }}
    - name: Upload Stats
      uses: actions/upload-artifact@v5
      with:
        name: fuzz-stats-${{ matrix.test-target.name }}
        path: |
          fuzz/stats/${{ matrix.test-target.name }}.txt
          fuzz/stats/${{ matrix.test-target.name }}.should_pass
        retention-days: 5
  fuzz-summary:
    needs: fuzz-run
    name: Fuzzing Summary
    runs-on: ubuntu-latest
    if: always()
    steps:
      - uses: actions/checkout@v5
        with:
          persist-credentials: false
      - name: Download all stats
        uses: actions/download-artifact@v6
        with:
          path: fuzz/stats-artifacts
          pattern: fuzz-stats-*
          merge-multiple: true
      - name: Prepare stats directory
        run: |
          mkdir -p fuzz/stats
          # Debug: List content of stats-artifacts directory
          echo "Contents of stats-artifacts directory:"
          find fuzz/stats-artifacts -type f | sort

          # Extract files from the artifact directories - handle nested directories
          find fuzz/stats-artifacts -type f -name "*.txt" -exec cp {} fuzz/stats/ \;
          find fuzz/stats-artifacts -type f -name "*.should_pass" -exec cp {} fuzz/stats/ \;

          # Debug information
          echo "Contents of stats directory after extraction:"
          ls -la fuzz/stats/
          echo "Contents of should_pass files (if any):"
          cat fuzz/stats/*.should_pass 2>/dev/null || echo "No should_pass files found"
      - name: Generate Summary
        run: |
          echo "# Fuzzing Summary" > fuzzing_summary.md
          echo "" >> fuzzing_summary.md
          echo "| Target | Runs | Exec/sec | New Units | Should pass | Status |" >> fuzzing_summary.md
          echo "|--------|------|----------|-----------|-------------|--------|" >> fuzzing_summary.md

          TOTAL_RUNS=0
          TOTAL_NEW_UNITS=0

          for stat_file in fuzz/stats/*.txt; do
            TARGET=$(basename "$stat_file" .txt)
            SHOULD_PASS_FILE="${stat_file%.*}.should_pass"

            # Get expected status
            if [ -f "$SHOULD_PASS_FILE" ]; then
              EXPECTED=$(cat "$SHOULD_PASS_FILE")
            else
              EXPECTED="unknown"
            fi

            # Extract runs
            if grep -q "stat::number_of_executed_units" "$stat_file"; then
              RUNS=$(grep "stat::number_of_executed_units" "$stat_file" | awk '{print $2}')
              TOTAL_RUNS=$((TOTAL_RUNS + RUNS))
            else
              RUNS="unknown"
            fi

            # Extract execution rate
            if grep -q "stat::average_exec_per_sec" "$stat_file"; then
              EXEC_RATE=$(grep "stat::average_exec_per_sec" "$stat_file" | awk '{print $2}')
            else
              EXEC_RATE="unknown"
            fi

            # Extract new units added
            if grep -q "stat::new_units_added" "$stat_file"; then
              NEW_UNITS=$(grep "stat::new_units_added" "$stat_file" | awk '{print $2}')
              if [[ "$NEW_UNITS" =~ ^[0-9]+$ ]]; then
                TOTAL_NEW_UNITS=$((TOTAL_NEW_UNITS + NEW_UNITS))
              fi
            else
              NEW_UNITS="unknown"
            fi

            # Extract status
            if grep -q "SUMMARY: " "$stat_file"; then
              STATUS=$(grep "SUMMARY: " "$stat_file" | head -1)
            else
              STATUS="Completed"
            fi

            echo "| $TARGET | $RUNS | $EXEC_RATE | $NEW_UNITS | $EXPECTED | $STATUS |" >> fuzzing_summary.md
          done

          echo "" >> fuzzing_summary.md
          echo "## Overall Statistics" >> fuzzing_summary.md
          echo "" >> fuzzing_summary.md
          echo "- **Total runs:** $TOTAL_RUNS" >> fuzzing_summary.md
          echo "- **Total new units discovered:** $TOTAL_NEW_UNITS" >> fuzzing_summary.md
          echo "- **Average execution rate:** $(grep -h "stat::average_exec_per_sec" fuzz/stats/*.txt | awk '{sum += $2; count++} END {if (count > 0) print sum/count " execs/sec"; else print "unknown"}')" >> fuzzing_summary.md

          # Add count by expected status
          echo "- **Tests expected to pass:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "true")" >> fuzzing_summary.md
          echo "- **Tests expected to fail:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "false")" >> fuzzing_summary.md

          # Write to GitHub step summary
          cat fuzzing_summary.md >> $GITHUB_STEP_SUMMARY
      - name: Show Summary
        run: |
          cat fuzzing_summary.md
      - name: Upload Summary
        uses: actions/upload-artifact@v5
        with:
          name: fuzzing-summary
          path: fuzzing_summary.md
          retention-days: 5