Skip to content

Commit 0c96717

Browse files
MileriusMilerius
authored
Merge pull request #8 from Milerius/feat/improve-benchmarks
feat(bench): hardware performance counters for benchmarks
2 parents b1dedd0 + dfa9159 commit 0c96717

File tree

18 files changed

+1921
-26
lines changed

18 files changed

+1921
-26
lines changed

‎.github/scripts/bench-report.sh‎

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# Generate a comparison-style markdown benchmark report.
5+
# Usage: bench-report.sh <linux-json> <macos-json>
6+
# Outputs markdown to stdout.
7+
8+
linux_json="${1:?usage: bench-report.sh <linux-json> <macos-json>}"
9+
macos_json="${2:?usage: bench-report.sh <linux-json> <macos-json>}"
10+
11+
# Normalize workload names into (impl, pattern, element) and extract metrics.
12+
# Output: JSON array of {impl, pattern, element, ns_per_op, ...}
13+
normalize='
14+
[.results[] | {
15+
workload,
16+
impl: (
17+
if .workload | startswith("spsc/inline/") then "mantis/inline"
18+
elif .workload | startswith("copy/") then "mantis/copy"
19+
elif .workload | startswith("general/") then "mantis/general"
20+
elif .workload | startswith("spsc/rtrb/") then "rtrb"
21+
elif .workload | startswith("spsc/crossbeam/") then "crossbeam"
22+
else "other"
23+
end
24+
),
25+
pattern: (
26+
if (.workload | test("single_item/|single/")) then "single"
27+
elif (.workload | test("burst_100/|burst/100/")) then "burst_100"
28+
elif (.workload | test("burst_1000/|burst/1000/")) then "burst_1000"
29+
elif (.workload | test("batch/100/")) then "batch_100"
30+
elif (.workload | test("batch/1000/")) then "batch_1000"
31+
elif (.workload | test("full_drain/")) then "full_drain"
32+
else "other"
33+
end
34+
),
35+
element: (.workload | split("/") | last),
36+
ns_per_op: (.ns_per_op | . * 100 | round / 100),
37+
cycles: (.cycles_per_op // null | if . == null then null elif . < 0.1 then 0 else (. * 10 | round / 10) end),
38+
insns: (.instructions_per_op // null | if . == null then null else round end),
39+
bmiss: (.branch_misses_per_op // null | if . == null then null elif . < 0.1 then 0 else (. * 10 | round / 10) end)
40+
}]
41+
'
42+
43+
# Build a comparison table for a given pattern.
44+
# Args: $1=json_file $2=pattern $3=title
45+
render_comparison() {
46+
local json="$1" pattern="$2" title="$3"
47+
local data impls elements
48+
49+
data=$(jq -r --arg p "$pattern" "$normalize | map(select(.pattern == \$p))" "$json")
50+
elements=$(echo "$data" | jq -r '[.[].element] | unique | .[]')
51+
impls=$(echo "$data" | jq -r '[.[].impl] | unique | .[]')
52+
53+
# Skip if no data for this pattern
54+
if [ -z "$elements" ]; then
55+
return
56+
fi
57+
58+
# Build header
59+
local header="| Element |"
60+
local separator="|:--------|"
61+
for impl in $impls; do
62+
header="$header $impl |"
63+
separator="$separator------:|"
64+
done
65+
66+
echo "#### $title"
67+
echo ""
68+
echo "$header"
69+
echo "$separator"
70+
71+
# Build rows
72+
for elem in $elements; do
73+
local row="| \`$elem\` |"
74+
# Find the best (lowest) ns/op for this element
75+
local best
76+
best=$(echo "$data" | jq -r --arg e "$elem" \
77+
'[.[] | select(.element == $e) | .ns_per_op] | min')
78+
79+
for impl in $impls; do
80+
local cell
81+
cell=$(echo "$data" | jq -r --arg i "$impl" --arg e "$elem" \
82+
'.[] | select(.impl == $i and .element == $e) | .ns_per_op // empty' 2>/dev/null)
83+
if [ -z "$cell" ]; then
84+
row="$row - |"
85+
else
86+
# Bold the best value with trophy emoji
87+
if [ "$cell" = "$best" ]; then
88+
row="$row **${cell}** 🏆 |"
89+
else
90+
row="$row $cell |"
91+
fi
92+
fi
93+
done
94+
echo "$row"
95+
done
96+
echo ""
97+
}
98+
99+
# Build an insns/op comparison table for a given pattern.
100+
render_insns_comparison() {
101+
local json="$1" pattern="$2" title="$3"
102+
local data impls elements has_any
103+
104+
data=$(jq -r --arg p "$pattern" "$normalize | map(select(.pattern == \$p))" "$json")
105+
elements=$(echo "$data" | jq -r '[.[].element] | unique | .[]')
106+
impls=$(echo "$data" | jq -r '[.[].impl] | unique | .[]')
107+
108+
# Check if any insns data exists
109+
has_any=$(echo "$data" | jq -r '[.[].insns | select(. != null)] | length')
110+
if [ "$has_any" = "0" ] || [ -z "$elements" ]; then
111+
return
112+
fi
113+
114+
local header="| Element |"
115+
local separator="|:--------|"
116+
for impl in $impls; do
117+
header="$header $impl |"
118+
separator="$separator------:|"
119+
done
120+
121+
echo "#### $title"
122+
echo ""
123+
echo "$header"
124+
echo "$separator"
125+
126+
for elem in $elements; do
127+
local row="| \`$elem\` |"
128+
local best
129+
best=$(echo "$data" | jq -r --arg e "$elem" \
130+
'[.[] | select(.element == $e) | .insns | select(. != null)] | min // empty')
131+
132+
for impl in $impls; do
133+
local cell
134+
cell=$(echo "$data" | jq -r --arg i "$impl" --arg e "$elem" \
135+
'.[] | select(.impl == $i and .element == $e) | .insns // empty' 2>/dev/null)
136+
if [ -z "$cell" ] || [ "$cell" = "null" ]; then
137+
row="$row - |"
138+
elif [ -n "$best" ] && [ "$cell" = "$best" ]; then
139+
row="$row **${cell}** 🏆 |"
140+
else
141+
row="$row $cell |"
142+
fi
143+
done
144+
echo "$row"
145+
done
146+
echo ""
147+
}
148+
149+
# Render full detailed table in a collapsible section.
150+
render_full_table() {
151+
local json="$1"
152+
153+
echo "| Workload | ns/op | p50 | p99 | cycles | insns | bmiss | l1d | llc |"
154+
echo "|:---------|------:|----:|----:|-------:|------:|------:|----:|----:|"
155+
156+
jq -r '.results[] |
157+
[
158+
.workload,
159+
(.ns_per_op | . * 100 | round / 100 | tostring),
160+
(.p50_ns | . * 10 | round / 10 | tostring),
161+
(.p99_ns | . * 10 | round / 10 | tostring),
162+
(.cycles_per_op // null | if . == null then "-" elif . < 0.1 then "<0.1" else (. * 10 | round / 10 | tostring) end),
163+
(.instructions_per_op // null | if . == null then "-" else (. | round | tostring) end),
164+
(.branch_misses_per_op // null | if . == null then "-" elif . < 0.1 then "<0.1" else (. * 10 | round / 10 | tostring) end),
165+
(.l1_misses_per_op // null | if . == null then "-" elif . == 0 then "0" else (. * 10 | round / 10 | tostring) end),
166+
(.llc_misses_per_op // null | if . == null then "-" elif . == 0 then "0" else (. * 10 | round / 10 | tostring) end)
167+
] | "| " + join(" | ") + " |"
168+
' "$json"
169+
}
170+
171+
# Render one platform section.
172+
render_platform() {
173+
local json="$1" label="$2"
174+
local cpu arch compiler
175+
176+
if [ ! -f "$json" ]; then
177+
echo "*${label} benchmark results not available.*"
178+
echo ""
179+
return
180+
fi
181+
182+
cpu=$(jq -r '.cpu' "$json")
183+
arch=$(jq -r '.arch' "$json")
184+
compiler=$(jq -r '.compiler' "$json")
185+
186+
echo "**CPU:** \`${cpu}\` | **Arch:** \`${arch}\` | **Compiler:** \`${compiler}\`"
187+
echo ""
188+
189+
echo "##### Latency (ns/op, lower is better)"
190+
echo ""
191+
render_comparison "$json" "single" "Single Push+Pop"
192+
render_comparison "$json" "burst_100" "Burst 100"
193+
render_comparison "$json" "burst_1000" "Burst 1000"
194+
render_comparison "$json" "batch_100" "Batch 100"
195+
render_comparison "$json" "batch_1000" "Batch 1000"
196+
render_comparison "$json" "full_drain" "Full Drain"
197+
198+
echo "##### Instructions per Op (lower is better)"
199+
echo ""
200+
render_insns_comparison "$json" "single" "Single Push+Pop"
201+
render_insns_comparison "$json" "burst_100" "Burst 100"
202+
203+
echo "<details>"
204+
echo "<summary>Full results (all fields)</summary>"
205+
echo ""
206+
render_full_table "$json"
207+
echo ""
208+
echo "</details>"
209+
}
210+
211+
commit_sha="${GITHUB_SHA:-$(git rev-parse --short HEAD)}"
212+
213+
cat <<HEADER
214+
## Benchmark Report
215+
216+
<sub>Commit: \`${commit_sha}\`</sub>
217+
218+
HEADER
219+
220+
echo "<details open>"
221+
echo "<summary><strong>Linux</strong></summary>"
222+
echo ""
223+
render_platform "$linux_json" "Linux"
224+
echo "</details>"
225+
echo ""
226+
227+
echo "<details open>"
228+
echo "<summary><strong>macOS</strong></summary>"
229+
echo ""
230+
render_platform "$macos_json" "macOS"
231+
echo "</details>"

‎.github/workflows/bench.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ jobs:
3636
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
3737
with:
3838
prefix-key: "bench-${{ steps.cpu.outputs.hash }}"
39+
- name: Allow perf_event_open (Linux)
40+
if: runner.os == 'Linux'
41+
run: sudo sysctl -w kernel.perf_event_paranoid=1
3942
- name: Run SPSC benchmarks
4043
run: cargo +nightly bench --bench spsc --all-features
4144
env:
@@ -63,3 +66,38 @@ jobs:
6366
path: |
6467
target/bench-report-spsc.json
6568
bench-output.json
69+
70+
report:
71+
name: Post benchmark report
72+
needs: benchmark
73+
if: always() && github.event_name == 'pull_request'
74+
runs-on: ubuntu-latest
75+
permissions:
76+
pull-requests: write
77+
steps:
78+
- name: Checkout PR
79+
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
80+
with:
81+
persist-credentials: false
82+
- name: Download Linux results
83+
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4
84+
with:
85+
name: benchmark-results-ubuntu-latest
86+
path: results/linux
87+
continue-on-error: true
88+
- name: Download macOS results
89+
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4
90+
with:
91+
name: benchmark-results-macos-latest
92+
path: results/macos
93+
continue-on-error: true
94+
- name: Generate markdown report
95+
run: |
96+
linux_json="results/linux/target/bench-report-spsc.json"
97+
macos_json="results/macos/target/bench-report-spsc.json"
98+
bash .github/scripts/bench-report.sh "$linux_json" "$macos_json" > report.md
99+
- name: Post PR comment
100+
uses: marocchino/sticky-pull-request-comment@52423e01640425a022ef5fd42c6fb5f633a02728 # v2.9.1
101+
with:
102+
header: benchmark-report
103+
path: report.md

‎Cargo.lock‎

Lines changed: 66 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,4 @@ cfg-if = "1"
5151
criterion = { version = "0.8", features = ["html_reports"] }
5252
bolero = "0.11"
5353
libc = "0.2.169"
54+
perf-event2 = "0.7"

‎crates/bench/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ repository.workspace = true
1010
[features]
1111
default = []
1212
asm = ["mantis-platform/asm"]
13+
perf-counters = ["mantis-platform/perf-counters"]
1314
bench-contenders = ["dep:rtrb", "dep:crossbeam-queue"]
1415

1516
[dependencies]

0 commit comments

Comments
 (0)