Skip to content

Commit 4789eab

Browse files
MaurUppiMaurUppi
committed
ci: add dns benchmark suite profiles and baseline strategies
1 parent 16a5d5c commit 4789eab

File tree

5 files changed

+377
-88
lines changed

5 files changed

+377
-88
lines changed

‎.github/workflows/dns-benchmark-compare.yml‎

Lines changed: 68 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ on:
99
- "go.mod"
1010
- "go.sum"
1111
- "scripts/ci/dns-benchmark-compare.sh"
12+
- "scripts/ci/dns-benchmark-suite-runner.sh"
13+
- "scripts/ci/dns-benchmark-suites.sh"
14+
- "scripts/ci/benchmarks/**"
1215
- ".github/workflows/dns-benchmark-compare.yml"
1316
workflow_dispatch:
1417
inputs:
@@ -20,10 +23,26 @@ on:
2023
description: "Head ref to benchmark"
2124
required: false
2225
default: "HEAD"
23-
bench_filter:
24-
description: "Regex to select benchmark names"
26+
base_strategy:
27+
description: "Base commit strategy: merge-base or exact"
2528
required: false
26-
default: "^Benchmark(AsyncCache|SingleflightOverhead|HighQpsScenario|RealisticDnsQuery|DnsController_Singleflight|PipelinedConn_Concurrent|PipelinedConn_Sequential)$"
29+
default: "merge-base"
30+
suite_profile:
31+
description: "Suite profile: quick or dns-module"
32+
required: false
33+
default: "dns-module"
34+
suite_list:
35+
description: "Optional comma-separated suite list. Overrides profile."
36+
required: false
37+
default: ""
38+
bench_count:
39+
description: "go test -count"
40+
required: false
41+
default: "3"
42+
bench_time:
43+
description: "go test -benchtime"
44+
required: false
45+
default: "200ms"
2746

2847
permissions:
2948
contents: read
@@ -34,7 +53,7 @@ jobs:
3453
name: DNS Bench Compare
3554
runs-on: ubuntu-22.04
3655
continue-on-error: true
37-
timeout-minutes: 30
56+
timeout-minutes: 45
3857

3958
steps:
4059
- name: Checkout
@@ -63,100 +82,63 @@ jobs:
6382
echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
6483
6584
- name: Run Benchmark Compare
66-
env:
67-
BENCH_FILTER: ${{ github.event.inputs.bench_filter }}
6885
run: |
6986
set -euo pipefail
7087
chmod +x scripts/ci/dns-benchmark-compare.sh
88+
chmod +x scripts/ci/dns-benchmark-suite-runner.sh
7189
mkdir -p bench-artifacts
7290
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
7391
BASE_REF="origin/${{ github.base_ref }}"
7492
HEAD_REF="${{ github.sha }}"
93+
BASE_STRATEGY="merge-base"
94+
SUITE_PROFILE="dns-module"
95+
SUITE_LIST=""
96+
BENCH_COUNT="3"
97+
BENCH_TIME="200ms"
7598
else
7699
BASE_REF="${{ github.event.inputs.base_ref }}"
77100
HEAD_REF="${{ github.event.inputs.head_ref }}"
101+
BASE_STRATEGY="${{ github.event.inputs.base_strategy }}"
102+
SUITE_PROFILE="${{ github.event.inputs.suite_profile }}"
103+
SUITE_LIST="${{ github.event.inputs.suite_list }}"
104+
BENCH_COUNT="${{ github.event.inputs.bench_count }}"
105+
BENCH_TIME="${{ github.event.inputs.bench_time }}"
78106
fi
79-
overall=0
80-
run_suite() {
81-
suite="$1"
82-
pkg="$2"
83-
filter="$3"
84-
exclude_files="${4:-}"
85-
suite_dir="bench-artifacts/${suite}"
86-
mkdir -p "$suite_dir"
87-
set +e
88-
BENCH_PACKAGE="$pkg" \
89-
BENCH_FILTER="$filter" \
90-
BENCH_OVERLAY_DIR="$PWD/scripts/ci/benchmarks" \
91-
BENCH_EXCLUDE_TEST_FILES="$exclude_files" \
92-
ARTIFACT_DIR="$PWD/$suite_dir" \
93-
./scripts/ci/dns-benchmark-compare.sh "$BASE_REF" "$HEAD_REF" \
94-
2>&1 | tee "$suite_dir/run.log"
95-
status=${PIPESTATUS[0]}
96-
set -e
97-
if [[ $status -ne 0 && ! -f "$suite_dir/report.md" ]]; then
98-
{
99-
echo "## DNS Benchmark Compare (${suite})"
100-
echo
101-
echo "- Status: failed"
102-
echo "- Exit code: \`$status\`"
103-
echo "- Base ref: \`$BASE_REF\`"
104-
echo "- Head ref: \`$HEAD_REF\`"
105-
echo "- Package: \`$pkg\`"
106-
echo
107-
echo "### Failure Log (tail)"
108-
echo
109-
echo '```text'
110-
tail -n 200 "$suite_dir/run.log" || true
111-
echo '```'
112-
} > "$suite_dir/report.md"
113-
fi
114-
echo "$status" > "$suite_dir/status.txt"
115-
if [[ $status -ne 0 ]]; then
116-
overall=1
117-
fi
118-
}
119107
120-
run_suite "control_dns" \
121-
"./control" \
122-
"^Benchmark(AsyncCache|SingleflightOverhead|HighQpsScenario|RealisticDnsQuery|DnsController_Singleflight|PipelinedConn_Concurrent|PipelinedConn_Sequential)$" \
123-
"control/tcp_test.go,control/tcp_splice_bench_test.go"
124-
125-
run_suite "component_dns" \
126-
"./component/dns" \
127-
"^BenchmarkUpstreamResolver_GetUpstream_(Serial|Parallel)$"
128-
129-
{
130-
echo "## DNS Benchmark Compare"
131-
echo
132-
echo "- Base ref: \`$BASE_REF\`"
133-
echo "- Head ref: \`$HEAD_REF\`"
134-
echo
135-
echo "### Suite Status"
136-
echo
137-
control_status="$(cat bench-artifacts/control_dns/status.txt 2>/dev/null || echo 1)"
138-
comp_status="$(cat bench-artifacts/component_dns/status.txt 2>/dev/null || echo 1)"
139-
echo "- control_dns: $([[ \"$control_status\" == \"0\" ]] && echo 'success' || echo 'failed')"
140-
echo "- component_dns: $([[ \"$comp_status\" == \"0\" ]] && echo 'success' || echo 'failed')"
141-
echo
142-
echo "### control_dns"
143-
echo
144-
if [[ -f bench-artifacts/control_dns/report.md ]]; then
145-
cat bench-artifacts/control_dns/report.md
146-
else
147-
echo "_No report generated_"
148-
fi
149-
echo
150-
echo "### component_dns"
151-
echo
152-
if [[ -f bench-artifacts/component_dns/report.md ]]; then
153-
cat bench-artifacts/component_dns/report.md
154-
else
155-
echo "_No report generated_"
156-
fi
157-
} > bench-artifacts/report.md
158-
159-
exit $overall
108+
set +e
109+
DNS_BENCH_PROFILE="$SUITE_PROFILE" \
110+
DNS_BENCH_SUITES="$SUITE_LIST" \
111+
BENCH_COUNT="$BENCH_COUNT" \
112+
BENCH_TIME="$BENCH_TIME" \
113+
BASE_COMMIT_STRATEGY="$BASE_STRATEGY" \
114+
ARTIFACT_DIR="$PWD/bench-artifacts" \
115+
./scripts/ci/dns-benchmark-suite-runner.sh "$BASE_REF" "$HEAD_REF" \
116+
2>&1 | tee bench-artifacts/run.log
117+
status=${PIPESTATUS[0]}
118+
set -e
119+
120+
if [[ $status -ne 0 && ! -f bench-artifacts/report.md ]]; then
121+
{
122+
echo "## DNS Benchmark Compare"
123+
echo
124+
echo "- Status: failed"
125+
echo "- Exit code: \`$status\`"
126+
echo "- Base ref: \`$BASE_REF\`"
127+
echo "- Head ref: \`$HEAD_REF\`"
128+
echo "- Base strategy: \`$BASE_STRATEGY\`"
129+
echo "- Suite profile: \`$SUITE_PROFILE\`"
130+
if [[ -n "$SUITE_LIST" ]]; then
131+
echo "- Suite selection: \`$SUITE_LIST\`"
132+
fi
133+
echo
134+
echo "### Failure Log (tail)"
135+
echo
136+
echo '```text'
137+
tail -n 200 bench-artifacts/run.log || true
138+
echo '```'
139+
} > bench-artifacts/report.md
140+
fi
141+
exit $status
160142
161143
- name: Upload Benchmark Artifacts
162144
if: always()

‎scripts/ci/dns-benchmark-compare.sh‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ cd "$REPO_ROOT"
2525

2626
BASE_REF="${1:-${BASE_REF:-origin/main}}"
2727
HEAD_REF="${2:-${HEAD_REF:-HEAD}}"
28+
BASE_COMMIT_STRATEGY="${BASE_COMMIT_STRATEGY:-merge-base}"
2829
BENCH_PACKAGE="${BENCH_PACKAGE:-./control}"
2930
BENCH_FILTER="${BENCH_FILTER:-^Benchmark(AsyncCache|SingleflightOverhead|HighQpsScenario|RealisticDnsQuery|DnsController_Singleflight|PipelinedConn_Concurrent|PipelinedConn_Sequential)$}"
3031
BENCH_COUNT="${BENCH_COUNT:-3}"
@@ -71,7 +72,17 @@ resolve_ref() {
7172
resolve_ref "$BASE_REF" || die "cannot resolve base ref: $BASE_REF"
7273
resolve_ref "$HEAD_REF" || die "cannot resolve head ref: $HEAD_REF"
7374

74-
BASE_COMMIT="$(git merge-base "$BASE_REF" "$HEAD_REF")"
75+
case "$BASE_COMMIT_STRATEGY" in
76+
merge-base)
77+
BASE_COMMIT="$(git merge-base "$BASE_REF" "$HEAD_REF")"
78+
;;
79+
exact)
80+
BASE_COMMIT="$(git rev-parse "$BASE_REF")"
81+
;;
82+
*)
83+
die "unsupported BASE_COMMIT_STRATEGY: $BASE_COMMIT_STRATEGY (expected merge-base|exact)"
84+
;;
85+
esac
7586
HEAD_COMMIT="$(git rev-parse "$HEAD_REF")"
7687

7788
log "base ref: $BASE_REF ($BASE_COMMIT)"
@@ -217,7 +228,8 @@ REPORT_MD="$ARTIFACT_DIR/report.md"
217228
echo "## DNS Benchmark Compare"
218229
echo
219230
echo "- Base ref: \`$BASE_REF\`"
220-
echo "- Base commit (merge-base): \`$BASE_COMMIT\`"
231+
echo "- Base strategy: \`$BASE_COMMIT_STRATEGY\`"
232+
echo "- Base commit: \`$BASE_COMMIT\`"
221233
echo "- Head ref: \`$HEAD_REF\`"
222234
echo "- Head commit: \`$HEAD_COMMIT\`"
223235
echo "- Package: \`$BENCH_PACKAGE\`"

‎scripts/ci/dns-benchmark-framework.md‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# DNS Benchmark Framework
2+
3+
This framework compares DNS-related benchmarks between a base ref and a head ref.
4+
5+
## Goals
6+
7+
- Cover key DNS module behavior with grouped benchmark suites.
8+
- Allow repeatable baseline comparisons for any DNS-related commit/PR.
9+
- Keep CI non-blocking while still publishing actionable perf reports.
10+
11+
## Suite Profiles
12+
13+
- `quick`
14+
- `control_core_flow`
15+
- `component_upstream_hotpath`
16+
- `dns-module` (default in PR CI)
17+
- `control_core_flow`
18+
- `control_singleflight_scale`
19+
- `control_cache_hotpath`
20+
- `control_cache_structures`
21+
- `component_upstream_hotpath`
22+
23+
Suite definitions live in `scripts/ci/dns-benchmark-suites.sh`.
24+
25+
## Base Strategies
26+
27+
- `merge-base` (default): compare `merge-base(base_ref, head_ref)` vs `head_ref`.
28+
- `exact`: compare `base_ref` commit directly vs `head_ref`.
29+
30+
Use `exact` when you want strict "commit A vs commit B" comparisons.
31+
32+
## Local Usage
33+
34+
```bash
35+
chmod +x scripts/ci/dns-benchmark-compare.sh scripts/ci/dns-benchmark-suite-runner.sh
36+
37+
# Profile-based run
38+
DNS_BENCH_PROFILE=dns-module \
39+
BASE_COMMIT_STRATEGY=merge-base \
40+
ARTIFACT_DIR=bench-artifacts \
41+
./scripts/ci/dns-benchmark-suite-runner.sh origin/main HEAD
42+
43+
# Explicit suite list and exact baseline
44+
DNS_BENCH_SUITES=component_upstream_hotpath,control_singleflight_scale \
45+
BASE_COMMIT_STRATEGY=exact \
46+
ARTIFACT_DIR=bench-artifacts \
47+
./scripts/ci/dns-benchmark-suite-runner.sh 5268be5 594f449
48+
```
49+
50+
## GitHub Actions
51+
52+
Workflow: `.github/workflows/dns-benchmark-compare.yml`
53+
54+
- Pull requests: runs `dns-module` profile against `origin/<base>` and PR head.
55+
- Manual dispatch: supports `base_ref`, `head_ref`, `base_strategy`, `suite_profile`, `suite_list`, `bench_count`, and `bench_time`.
56+
57+
All runs publish:
58+
59+
- Aggregated report: `bench-artifacts/report.md`
60+
- Per-suite reports and raw benchmark outputs
61+
- PR comment with the latest report

0 commit comments

Comments
 (0)