Skip to content

Commit 28c052c

Browse files
yug49yug49
committed
feat: add Bencher CI integration for performance tracking
1 parent 7ec2a3d commit 28c052c

File tree

3 files changed

+214
-0
lines changed

3 files changed

+214
-0
lines changed

‎.github/workflows/benchmarks.yml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: Benchmarks
2+
3+
# Runs benchmarks on-demand (manual dispatch) to track performance
4+
# and detect regressions in the Event Scanner.
5+
6+
on:
7+
workflow_dispatch:
8+
inputs:
9+
benchmark:
10+
description: "Benchmark to run"
11+
required: true
12+
default: "all"
13+
type: choice
14+
options:
15+
- all
16+
- historic_scanning
17+
- latest_events_scanning
18+
19+
permissions:
20+
contents: read
21+
checks: write
22+
23+
env:
24+
CARGO_TERM_COLOR: always
25+
26+
jobs:
27+
benchmark:
28+
name: Run Benchmarks
29+
runs-on: ubuntu-latest
30+
31+
steps:
32+
- name: Harden the runner (Audit all outbound calls)
33+
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
34+
with:
35+
egress-policy: audit
36+
37+
- name: Checkout repository
38+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
39+
40+
- name: Install Rust toolchain
41+
uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2
42+
43+
- name: Install Foundry (Anvil)
44+
uses: foundry-rs/foundry-toolchain@v1
45+
with:
46+
version: stable
47+
48+
- name: Install Bencher CLI
49+
uses: bencherdev/bencher@main
50+
51+
- name: Run benchmarks and track with Bencher
52+
env:
53+
BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
54+
BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT }}
55+
run: |
56+
# Determine which benchmarks to run
57+
if [ "${{ inputs.benchmark }}" = "all" ]; then
58+
BENCH_CMD="cargo bench --manifest-path benches/Cargo.toml"
59+
else
60+
BENCH_CMD="cargo bench --manifest-path benches/Cargo.toml --bench ${{ inputs.benchmark }}"
61+
fi
62+
63+
bencher run \
64+
--project "$BENCHER_PROJECT" \
65+
--token "$BENCHER_API_TOKEN" \
66+
--branch main \
67+
--testbed ubuntu-latest \
68+
--threshold-measure latency \
69+
--threshold-test t_test \
70+
--threshold-max-sample-size 64 \
71+
--threshold-upper-boundary 0.99 \
72+
--thresholds-reset \
73+
--err \
74+
--adapter rust_criterion \
75+
--github-actions "${{ secrets.GITHUB_TOKEN }}" \
76+
$BENCH_CMD

‎.github/workflows/pr_benchmarks_run.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: Run PR Benchmarks
2+
3+
# This workflow runs benchmarks on pull requests and caches the results.
4+
# For security, benchmark results are uploaded as artifacts and tracked
5+
# by a separate workflow (pr_benchmarks_track.yml) that has access to secrets.
6+
7+
on:
8+
pull_request:
9+
types: [opened, reopened, synchronize]
10+
paths:
11+
# Only run benchmarks if relevant code changes
12+
- "src/**"
13+
- "benches/**"
14+
- "Cargo.toml"
15+
- "Cargo.lock"
16+
17+
permissions:
18+
contents: read
19+
20+
env:
21+
CARGO_TERM_COLOR: always
22+
23+
jobs:
24+
benchmark_pr:
25+
name: Run PR Benchmarks
26+
runs-on: ubuntu-latest
27+
28+
steps:
29+
- name: Harden the runner (Audit all outbound calls)
30+
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
31+
with:
32+
egress-policy: audit
33+
34+
- name: Checkout repository
35+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
36+
37+
- name: Install Rust toolchain
38+
uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2
39+
40+
- name: Install Foundry (Anvil)
41+
uses: foundry-rs/foundry-toolchain@v1
42+
with:
43+
version: stable
44+
45+
- name: Run benchmarks
46+
run: |
47+
cargo bench --manifest-path benches/Cargo.toml -- --noplot 2>&1 | tee benchmark_results.txt
48+
49+
- name: Upload benchmark results
50+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
51+
with:
52+
name: benchmark_results.txt
53+
path: ./benchmark_results.txt
54+
55+
- name: Upload PR event
56+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
57+
with:
58+
name: event.json
59+
path: ${{ github.event_path }}

‎.github/workflows/pr_benchmarks_track.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
name: Track PR Benchmarks
2+
3+
# This workflow tracks benchmark results from pull requests using Bencher.
4+
# It runs after the pr_benchmarks_run.yml workflow completes successfully.
5+
# This separation is required for security when handling fork PRs, as secrets
6+
# are not available in the context of fork PR workflows.
7+
8+
on:
9+
workflow_run:
10+
workflows: [Run PR Benchmarks]
11+
types: [completed]
12+
13+
permissions:
14+
contents: read
15+
pull-requests: write
16+
17+
env:
18+
BENCHMARK_RESULTS: benchmark_results.txt
19+
PR_EVENT: event.json
20+
21+
jobs:
22+
track_pr_benchmarks:
23+
name: Track PR Benchmarks with Bencher
24+
if: github.event.workflow_run.conclusion == 'success'
25+
runs-on: ubuntu-latest
26+
27+
steps:
28+
- name: Harden the runner (Audit all outbound calls)
29+
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
30+
with:
31+
egress-policy: audit
32+
33+
- name: Download benchmark results
34+
uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8
35+
with:
36+
name: ${{ env.BENCHMARK_RESULTS }}
37+
run_id: ${{ github.event.workflow_run.id }}
38+
39+
- name: Download PR event
40+
uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8
41+
with:
42+
name: ${{ env.PR_EVENT }}
43+
run_id: ${{ github.event.workflow_run.id }}
44+
45+
- name: Export PR event data
46+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
47+
with:
48+
script: |
49+
const fs = require('fs');
50+
const prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, { encoding: 'utf8' }));
51+
core.exportVariable('PR_NUMBER', prEvent.number);
52+
core.exportVariable('PR_HEAD', prEvent.pull_request.head.ref);
53+
core.exportVariable('PR_HEAD_SHA', prEvent.pull_request.head.sha);
54+
core.exportVariable('PR_BASE', prEvent.pull_request.base.ref);
55+
core.exportVariable('PR_BASE_SHA', prEvent.pull_request.base.sha);
56+
57+
- name: Install Bencher CLI
58+
uses: bencherdev/bencher@main
59+
60+
- name: Track PR benchmarks with Bencher
61+
env:
62+
BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
63+
BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT }}
64+
run: |
65+
bencher run \
66+
--project "$BENCHER_PROJECT" \
67+
--token "$BENCHER_API_TOKEN" \
68+
--branch "$PR_HEAD" \
69+
--hash "$PR_HEAD_SHA" \
70+
--start-point "$PR_BASE" \
71+
--start-point-hash "$PR_BASE_SHA" \
72+
--start-point-clone-thresholds \
73+
--start-point-reset \
74+
--testbed ubuntu-latest \
75+
--err \
76+
--adapter rust_criterion \
77+
--github-actions "${{ secrets.GITHUB_TOKEN }}" \
78+
--ci-number "$PR_NUMBER" \
79+
--file "$BENCHMARK_RESULTS"

0 commit comments

Comments
 (0)