Skip to content

Commit 30150e3

Browse files
sylvestresylvestre
committed
fuzzing: get the summary data
1 parent 22c4846 commit 30150e3

File tree

1 file changed

+188
-2
lines changed

1 file changed

+188
-2
lines changed

‎.github/workflows/fuzzing.yml

Lines changed: 188 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,13 +81,199 @@ jobs:
8181
path: |
8282
fuzz/corpus/${{ matrix.test-target.name }}
8383
- name: Run ${{ matrix.test-target.name }} for XX seconds
84+
id: run_fuzzer
8485
shell: bash
85-
continue-on-error: ${{ !matrix.test-target.name.should_pass }}
86+
continue-on-error: ${{ !matrix.test-target.should_pass }}
8687
run: |
87-
cargo +nightly fuzz run ${{ matrix.test-target.name }} -- -max_total_time=${{ env.RUN_FOR }} -timeout=${{ env.RUN_FOR }} -detect_leaks=0
88+
mkdir -p fuzz/stats
89+
STATS_FILE="fuzz/stats/${{ matrix.test-target.name }}.txt"
90+
cargo +nightly fuzz run ${{ matrix.test-target.name }} -- -max_total_time=${{ env.RUN_FOR }} -timeout=${{ env.RUN_FOR }} -detect_leaks=0 -print_final_stats=1 2>&1 | tee "$STATS_FILE"
91+
92+
# Extract key stats from the output
93+
if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then
94+
RUNS=$(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}')
95+
echo "runs=$RUNS" >> "$GITHUB_OUTPUT"
96+
else
97+
echo "runs=unknown" >> "$GITHUB_OUTPUT"
98+
fi
99+
100+
if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then
101+
EXEC_RATE=$(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}')
102+
echo "exec_rate=$EXEC_RATE" >> "$GITHUB_OUTPUT"
103+
else
104+
echo "exec_rate=unknown" >> "$GITHUB_OUTPUT"
105+
fi
106+
107+
if grep -q "stat::new_units_added" "$STATS_FILE"; then
108+
NEW_UNITS=$(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}')
109+
echo "new_units=$NEW_UNITS" >> "$GITHUB_OUTPUT"
110+
else
111+
echo "new_units=unknown" >> "$GITHUB_OUTPUT"
112+
fi
113+
114+
# Save should_pass value to file for summary job to use
115+
echo "${{ matrix.test-target.should_pass }}" > "fuzz/stats/${{ matrix.test-target.name }}.should_pass"
116+
117+
# Print stats to job output for immediate visibility
118+
echo "----------------------------------------"
119+
echo "FUZZING STATISTICS FOR ${{ matrix.test-target.name }}"
120+
echo "----------------------------------------"
121+
echo "Runs: $(grep -q "stat::number_of_executed_units" "$STATS_FILE" && grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}' || echo "unknown")"
122+
echo "Execution Rate: $(grep -q "stat::average_exec_per_sec" "$STATS_FILE" && grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}' || echo "unknown") execs/sec"
123+
echo "New Units: $(grep -q "stat::new_units_added" "$STATS_FILE" && grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}' || echo "unknown")"
124+
echo "Expected: ${{ matrix.test-target.should_pass }}"
125+
if grep -q "SUMMARY: " "$STATS_FILE"; then
126+
echo "Status: $(grep "SUMMARY: " "$STATS_FILE" | head -1)"
127+
else
128+
echo "Status: Completed"
129+
fi
130+
echo "----------------------------------------"
131+
132+
# Add summary to GitHub step summary
133+
echo "### Fuzzing Results for ${{ matrix.test-target.name }}" >> $GITHUB_STEP_SUMMARY
134+
echo "" >> $GITHUB_STEP_SUMMARY
135+
echo "| Metric | Value |" >> $GITHUB_STEP_SUMMARY
136+
echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY
137+
138+
if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then
139+
echo "| Runs | $(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY
140+
fi
141+
142+
if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then
143+
echo "| Execution Rate | $(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}') execs/sec |" >> $GITHUB_STEP_SUMMARY
144+
fi
145+
146+
if grep -q "stat::new_units_added" "$STATS_FILE"; then
147+
echo "| New Units | $(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY
148+
fi
149+
150+
echo "| Expected | ${{ matrix.test-target.should_pass }} |" >> $GITHUB_STEP_SUMMARY
151+
152+
if grep -q "SUMMARY: " "$STATS_FILE"; then
153+
echo "| Status | $(grep "SUMMARY: " "$STATS_FILE" | head -1) |" >> $GITHUB_STEP_SUMMARY
154+
else
155+
echo "| Status | Completed |" >> $GITHUB_STEP_SUMMARY
156+
fi
157+
158+
echo "" >> $GITHUB_STEP_SUMMARY
88159
- name: Save Corpus Cache
89160
uses: actions/cache/save@v4
90161
with:
91162
key: corpus-cache-${{ matrix.test-target.name }}
92163
path: |
93164
fuzz/corpus/${{ matrix.test-target.name }}
165+
- name: Upload Stats
166+
uses: actions/upload-artifact@v4
167+
with:
168+
name: fuzz-stats-${{ matrix.test-target.name }}
169+
path: |
170+
fuzz/stats/${{ matrix.test-target.name }}.txt
171+
fuzz/stats/${{ matrix.test-target.name }}.should_pass
172+
retention-days: 5
173+
fuzz-summary:
174+
needs: fuzz-run
175+
name: Fuzzing Summary
176+
runs-on: ubuntu-latest
177+
if: always()
178+
steps:
179+
- uses: actions/checkout@v4
180+
with:
181+
persist-credentials: false
182+
- name: Download all stats
183+
uses: actions/download-artifact@v4
184+
with:
185+
path: fuzz/stats-artifacts
186+
pattern: fuzz-stats-*
187+
merge-multiple: true
188+
- name: Prepare stats directory
189+
run: |
190+
mkdir -p fuzz/stats
191+
# Extract files from the artifact directories
192+
for dir in fuzz/stats-artifacts/fuzz-stats-*; do
193+
if [ -d "$dir" ]; then
194+
cp "$dir"/* fuzz/stats/
195+
fi
196+
done
197+
# Debug information
198+
echo "Contents of stats directory:"
199+
ls -la fuzz/stats/
200+
echo "Contents of should_pass files:"
201+
cat fuzz/stats/*.should_pass 2>/dev/null || echo "No should_pass files found"
202+
- name: Generate Summary
203+
run: |
204+
echo "# Fuzzing Summary" > fuzzing_summary.md
205+
echo "" >> fuzzing_summary.md
206+
echo "| Target | Runs | Exec/sec | New Units | Expected | Status |" >> fuzzing_summary.md
207+
echo "|--------|------|----------|-----------|----------|--------|" >> fuzzing_summary.md
208+
209+
TOTAL_RUNS=0
210+
TOTAL_NEW_UNITS=0
211+
212+
for stat_file in fuzz/stats/*.txt; do
213+
TARGET=$(basename "$stat_file" .txt)
214+
SHOULD_PASS_FILE="${stat_file%.*}.should_pass"
215+
216+
# Get expected status
217+
if [ -f "$SHOULD_PASS_FILE" ]; then
218+
EXPECTED=$(cat "$SHOULD_PASS_FILE")
219+
else
220+
EXPECTED="unknown"
221+
fi
222+
223+
# Extract runs
224+
if grep -q "stat::number_of_executed_units" "$stat_file"; then
225+
RUNS=$(grep "stat::number_of_executed_units" "$stat_file" | awk '{print $2}')
226+
TOTAL_RUNS=$((TOTAL_RUNS + RUNS))
227+
else
228+
RUNS="unknown"
229+
fi
230+
231+
# Extract execution rate
232+
if grep -q "stat::average_exec_per_sec" "$stat_file"; then
233+
EXEC_RATE=$(grep "stat::average_exec_per_sec" "$stat_file" | awk '{print $2}')
234+
else
235+
EXEC_RATE="unknown"
236+
fi
237+
238+
# Extract new units added
239+
if grep -q "stat::new_units_added" "$stat_file"; then
240+
NEW_UNITS=$(grep "stat::new_units_added" "$stat_file" | awk '{print $2}')
241+
if [[ "$NEW_UNITS" =~ ^[0-9]+$ ]]; then
242+
TOTAL_NEW_UNITS=$((TOTAL_NEW_UNITS + NEW_UNITS))
243+
fi
244+
else
245+
NEW_UNITS="unknown"
246+
fi
247+
248+
# Extract status
249+
if grep -q "SUMMARY: " "$stat_file"; then
250+
STATUS=$(grep "SUMMARY: " "$stat_file" | head -1)
251+
else
252+
STATUS="Completed"
253+
fi
254+
255+
echo "| $TARGET | $RUNS | $EXEC_RATE | $NEW_UNITS | $EXPECTED | $STATUS |" >> fuzzing_summary.md
256+
done
257+
258+
echo "" >> fuzzing_summary.md
259+
echo "## Overall Statistics" >> fuzzing_summary.md
260+
echo "" >> fuzzing_summary.md
261+
echo "- **Total runs:** $TOTAL_RUNS" >> fuzzing_summary.md
262+
echo "- **Total new units discovered:** $TOTAL_NEW_UNITS" >> fuzzing_summary.md
263+
echo "- **Average execution rate:** $(grep -h "stat::average_exec_per_sec" fuzz/stats/*.txt | awk '{sum += $2; count++} END {if (count > 0) print sum/count " execs/sec"; else print "unknown"}')" >> fuzzing_summary.md
264+
265+
# Add count by expected status
266+
echo "- **Tests expected to pass:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "true")" >> fuzzing_summary.md
267+
echo "- **Tests expected to fail:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "false")" >> fuzzing_summary.md
268+
269+
# Write to GitHub step summary
270+
cat fuzzing_summary.md >> $GITHUB_STEP_SUMMARY
271+
- name: Show Summary
272+
run: |
273+
cat fuzzing_summary.md
274+
- name: Upload Summary
275+
uses: actions/upload-artifact@v4
276+
with:
277+
name: fuzzing-summary
278+
path: fuzzing_summary.md
279+
retention-days: 5

0 commit comments

Comments
 (0)