Skip to content

Commit 083b6d5

Browse files
cakebakercakebaker
authored
Merge pull request #7747 from sylvestre/fuzz4
fuzzing: add a summary and enable printf
2 parents c90bca4 + 5e9e67e commit 083b6d5

File tree

1 file changed

+191
-3
lines changed

1 file changed

+191
-3
lines changed

‎.github/workflows/fuzzing.yml‎

Lines changed: 191 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ jobs:
3636

3737
fuzz-run:
3838
needs: fuzz-build
39-
name: Run the fuzzers
39+
name: Fuzz
4040
runs-on: ubuntu-latest
4141
timeout-minutes: 5
4242
env:
@@ -48,7 +48,7 @@ jobs:
4848
# https://github.com/uutils/coreutils/issues/5311
4949
- { name: fuzz_date, should_pass: false }
5050
- { name: fuzz_expr, should_pass: true }
51-
- { name: fuzz_printf, should_pass: false }
51+
- { name: fuzz_printf, should_pass: true }
5252
- { name: fuzz_echo, should_pass: true }
5353
- { name: fuzz_seq, should_pass: false }
5454
- { name: fuzz_sort, should_pass: false }
@@ -81,13 +81,201 @@ jobs:
8181
path: |
8282
fuzz/corpus/${{ matrix.test-target.name }}
8383
- name: Run ${{ matrix.test-target.name }} for XX seconds
84+
id: run_fuzzer
8485
shell: bash
8586
continue-on-error: ${{ !matrix.test-target.name.should_pass }}
8687
run: |
87-
cargo +nightly fuzz run ${{ matrix.test-target.name }} -- -max_total_time=${{ env.RUN_FOR }} -timeout=${{ env.RUN_FOR }} -detect_leaks=0
88+
mkdir -p fuzz/stats
89+
STATS_FILE="fuzz/stats/${{ matrix.test-target.name }}.txt"
90+
cargo +nightly fuzz run ${{ matrix.test-target.name }} -- -max_total_time=${{ env.RUN_FOR }} -timeout=${{ env.RUN_FOR }} -detect_leaks=0 -print_final_stats=1 2>&1 | tee "$STATS_FILE"
91+
92+
# Extract key stats from the output
93+
if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then
94+
RUNS=$(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}')
95+
echo "runs=$RUNS" >> "$GITHUB_OUTPUT"
96+
else
97+
echo "runs=unknown" >> "$GITHUB_OUTPUT"
98+
fi
99+
100+
if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then
101+
EXEC_RATE=$(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}')
102+
echo "exec_rate=$EXEC_RATE" >> "$GITHUB_OUTPUT"
103+
else
104+
echo "exec_rate=unknown" >> "$GITHUB_OUTPUT"
105+
fi
106+
107+
if grep -q "stat::new_units_added" "$STATS_FILE"; then
108+
NEW_UNITS=$(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}')
109+
echo "new_units=$NEW_UNITS" >> "$GITHUB_OUTPUT"
110+
else
111+
echo "new_units=unknown" >> "$GITHUB_OUTPUT"
112+
fi
113+
114+
# Save should_pass value to file for summary job to use
115+
echo "${{ matrix.test-target.should_pass }}" > "fuzz/stats/${{ matrix.test-target.name }}.should_pass"
116+
117+
# Print stats to job output for immediate visibility
118+
echo "----------------------------------------"
119+
echo "FUZZING STATISTICS FOR ${{ matrix.test-target.name }}"
120+
echo "----------------------------------------"
121+
echo "Runs: $(grep -q "stat::number_of_executed_units" "$STATS_FILE" && grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}' || echo "unknown")"
122+
echo "Execution Rate: $(grep -q "stat::average_exec_per_sec" "$STATS_FILE" && grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}' || echo "unknown") execs/sec"
123+
echo "New Units: $(grep -q "stat::new_units_added" "$STATS_FILE" && grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}' || echo "unknown")"
124+
echo "Expected: ${{ matrix.test-target.name.should_pass }}"
125+
if grep -q "SUMMARY: " "$STATS_FILE"; then
126+
echo "Status: $(grep "SUMMARY: " "$STATS_FILE" | head -1)"
127+
else
128+
echo "Status: Completed"
129+
fi
130+
echo "----------------------------------------"
131+
132+
# Add summary to GitHub step summary
133+
echo "### Fuzzing Results for ${{ matrix.test-target.name }}" >> $GITHUB_STEP_SUMMARY
134+
echo "" >> $GITHUB_STEP_SUMMARY
135+
echo "| Metric | Value |" >> $GITHUB_STEP_SUMMARY
136+
echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY
137+
138+
if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then
139+
echo "| Runs | $(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY
140+
fi
141+
142+
if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then
143+
echo "| Execution Rate | $(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}') execs/sec |" >> $GITHUB_STEP_SUMMARY
144+
fi
145+
146+
if grep -q "stat::new_units_added" "$STATS_FILE"; then
147+
echo "| New Units | $(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY
148+
fi
149+
150+
echo "| Should pass | ${{ matrix.test-target.should_pass }} |" >> $GITHUB_STEP_SUMMARY
151+
152+
if grep -q "SUMMARY: " "$STATS_FILE"; then
153+
echo "| Status | $(grep "SUMMARY: " "$STATS_FILE" | head -1) |" >> $GITHUB_STEP_SUMMARY
154+
else
155+
echo "| Status | Completed |" >> $GITHUB_STEP_SUMMARY
156+
fi
157+
158+
echo "" >> $GITHUB_STEP_SUMMARY
88159
- name: Save Corpus Cache
89160
uses: actions/cache/save@v4
90161
with:
91162
key: corpus-cache-${{ matrix.test-target.name }}
92163
path: |
93164
fuzz/corpus/${{ matrix.test-target.name }}
165+
- name: Upload Stats
166+
uses: actions/upload-artifact@v4
167+
with:
168+
name: fuzz-stats-${{ matrix.test-target.name }}
169+
path: |
170+
fuzz/stats/${{ matrix.test-target.name }}.txt
171+
fuzz/stats/${{ matrix.test-target.name }}.should_pass
172+
retention-days: 5
173+
fuzz-summary:
174+
needs: fuzz-run
175+
name: Fuzzing Summary
176+
runs-on: ubuntu-latest
177+
if: always()
178+
steps:
179+
- uses: actions/checkout@v4
180+
with:
181+
persist-credentials: false
182+
- name: Download all stats
183+
uses: actions/download-artifact@v4
184+
with:
185+
path: fuzz/stats-artifacts
186+
pattern: fuzz-stats-*
187+
merge-multiple: true
188+
- name: Prepare stats directory
189+
run: |
190+
mkdir -p fuzz/stats
191+
# Debug: List content of stats-artifacts directory
192+
echo "Contents of stats-artifacts directory:"
193+
find fuzz/stats-artifacts -type f | sort
194+
195+
# Extract files from the artifact directories - handle nested directories
196+
find fuzz/stats-artifacts -type f -name "*.txt" -exec cp {} fuzz/stats/ \;
197+
find fuzz/stats-artifacts -type f -name "*.should_pass" -exec cp {} fuzz/stats/ \;
198+
199+
# Debug information
200+
echo "Contents of stats directory after extraction:"
201+
ls -la fuzz/stats/
202+
echo "Contents of should_pass files (if any):"
203+
cat fuzz/stats/*.should_pass 2>/dev/null || echo "No should_pass files found"
204+
- name: Generate Summary
205+
run: |
206+
echo "# Fuzzing Summary" > fuzzing_summary.md
207+
echo "" >> fuzzing_summary.md
208+
echo "| Target | Runs | Exec/sec | New Units | Should pass | Status |" >> fuzzing_summary.md
209+
echo "|--------|------|----------|-----------|-------------|--------|" >> fuzzing_summary.md
210+
211+
TOTAL_RUNS=0
212+
TOTAL_NEW_UNITS=0
213+
214+
for stat_file in fuzz/stats/*.txt; do
215+
TARGET=$(basename "$stat_file" .txt)
216+
SHOULD_PASS_FILE="${stat_file%.*}.should_pass"
217+
218+
# Get expected status
219+
if [ -f "$SHOULD_PASS_FILE" ]; then
220+
EXPECTED=$(cat "$SHOULD_PASS_FILE")
221+
else
222+
EXPECTED="unknown"
223+
fi
224+
225+
# Extract runs
226+
if grep -q "stat::number_of_executed_units" "$stat_file"; then
227+
RUNS=$(grep "stat::number_of_executed_units" "$stat_file" | awk '{print $2}')
228+
TOTAL_RUNS=$((TOTAL_RUNS + RUNS))
229+
else
230+
RUNS="unknown"
231+
fi
232+
233+
# Extract execution rate
234+
if grep -q "stat::average_exec_per_sec" "$stat_file"; then
235+
EXEC_RATE=$(grep "stat::average_exec_per_sec" "$stat_file" | awk '{print $2}')
236+
else
237+
EXEC_RATE="unknown"
238+
fi
239+
240+
# Extract new units added
241+
if grep -q "stat::new_units_added" "$stat_file"; then
242+
NEW_UNITS=$(grep "stat::new_units_added" "$stat_file" | awk '{print $2}')
243+
if [[ "$NEW_UNITS" =~ ^[0-9]+$ ]]; then
244+
TOTAL_NEW_UNITS=$((TOTAL_NEW_UNITS + NEW_UNITS))
245+
fi
246+
else
247+
NEW_UNITS="unknown"
248+
fi
249+
250+
# Extract status
251+
if grep -q "SUMMARY: " "$stat_file"; then
252+
STATUS=$(grep "SUMMARY: " "$stat_file" | head -1)
253+
else
254+
STATUS="Completed"
255+
fi
256+
257+
echo "| $TARGET | $RUNS | $EXEC_RATE | $NEW_UNITS | $EXPECTED | $STATUS |" >> fuzzing_summary.md
258+
done
259+
260+
echo "" >> fuzzing_summary.md
261+
echo "## Overall Statistics" >> fuzzing_summary.md
262+
echo "" >> fuzzing_summary.md
263+
echo "- **Total runs:** $TOTAL_RUNS" >> fuzzing_summary.md
264+
echo "- **Total new units discovered:** $TOTAL_NEW_UNITS" >> fuzzing_summary.md
265+
echo "- **Average execution rate:** $(grep -h "stat::average_exec_per_sec" fuzz/stats/*.txt | awk '{sum += $2; count++} END {if (count > 0) print sum/count " execs/sec"; else print "unknown"}')" >> fuzzing_summary.md
266+
267+
# Add count by expected status
268+
echo "- **Tests expected to pass:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "true")" >> fuzzing_summary.md
269+
echo "- **Tests expected to fail:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "false")" >> fuzzing_summary.md
270+
271+
# Write to GitHub step summary
272+
cat fuzzing_summary.md >> $GITHUB_STEP_SUMMARY
273+
- name: Show Summary
274+
run: |
275+
cat fuzzing_summary.md
276+
- name: Upload Summary
277+
uses: actions/upload-artifact@v4
278+
with:
279+
name: fuzzing-summary
280+
path: fuzzing_summary.md
281+
retention-days: 5

0 commit comments

Comments
 (0)