|
1 | 1 | name: bench |
2 | 2 |
|
3 | | -# Build-engine benchmark. MANUAL TRIGGER ONLY, and that is a design decision: |
| 3 | +# Build-engine benchmark. Runs on changes under `bench/` and on demand. |
| 4 | +# |
| 5 | +# WHY IT IS PATH-SCOPED RATHER THAN ON EVERY PUSH: |
4 | 6 | # |
5 | 7 | # * it is heavy — a full matrix compiles the same fixture six ways per platform |
6 | 8 | # * it is noisy — cloud runners are shared, and the CPU model changes under you |
7 | 9 | # * it asserts nothing — no threshold, no pass/fail on timings |
8 | 10 | # |
9 | | -# Attaching it to every PR would drown the signal it exists to produce, and a |
10 | | -# timing threshold on a shared runner turns normal variance into red crosses that |
11 | | -# people learn to ignore. Results are uploaded as artifacts; comparing them is a |
12 | | -# human act. |
| 11 | +# So it fires when the SUITE itself changes, where the question "did I break the |
| 12 | +# harness / did this shift the numbers" is actually being asked, and stays off |
| 13 | +# every unrelated PR. A timing threshold on a shared runner would turn normal |
| 14 | +# variance into red crosses people learn to ignore, so there is none: results |
| 15 | +# are uploaded as artifacts and comparing them is a human act. |
| 16 | +# |
| 17 | +# The matrix runs platforms in parallel and `fail-fast: false`, because one |
| 18 | +# platform missing an engine must not cancel the data from the others. |
13 | 19 | # |
14 | 20 | # See bench/README.md for the measurement contract before quoting any number. |
15 | 21 |
|
16 | 22 | on: |
| 23 | + # Changes to the suite itself — including its own tests and the project |
| 24 | + # descriptions it measures. Not `paths: ['**']`: the point is to fire where |
| 25 | + # the numbers can move, not on every commit. |
| 26 | + push: |
| 27 | + paths: |
| 28 | + - 'bench/**' |
| 29 | + - '.github/workflows/bench.yml' |
| 30 | + pull_request: |
| 31 | + paths: |
| 32 | + - 'bench/**' |
| 33 | + - '.github/workflows/bench.yml' |
17 | 34 | workflow_dispatch: |
18 | 35 | inputs: |
19 | 36 | engines: |
|
73 | 90 | shell: bash |
74 | 91 | run: | |
75 | 92 | set -euo pipefail |
76 | | - want="${{ inputs.platforms }}" |
| 93 | + # `inputs.*` is empty on a push/pull_request trigger, so every input needs |
| 94 | + # a fallback here — an empty `platforms` would otherwise plan an empty |
| 95 | + # matrix and the job would silently do nothing. |
| 96 | + want="${{ inputs.platforms || 'linux,macos,windows' }}" |
77 | 97 | entries=() |
78 | 98 | case ",$want," in *,linux,*) entries+=('{"os":"ubuntu-24.04","name":"linux"}');; esac |
79 | 99 | case ",$want," in *,macos,*) entries+=('{"os":"macos-14","name":"macos"}');; esac |
@@ -137,16 +157,19 @@ jobs: |
137 | 157 | # positive number. Passing raw numbers unconditionally would make every |
138 | 158 | # run's size an accident of this file rather than a named, comparable |
139 | 159 | # workload — and --preset must come first so the overrides still win. |
140 | | - args=( --preset "${{ inputs.preset }}" ) |
141 | | - [ "${{ inputs.units }}" -gt 0 ] 2>/dev/null && args+=( --units "${{ inputs.units }}" ) |
142 | | - [ "${{ inputs.fanin }}" -gt 0 ] 2>/dev/null && args+=( --fanin "${{ inputs.fanin }}" ) |
| 160 | + # Every `inputs.*` needs a fallback: on a push/pull_request trigger |
| 161 | + # they are all EMPTY, and an empty --engines would run nothing while |
| 162 | + # still reporting success. |
| 163 | + args=( --preset "${{ inputs.preset || 'smoke' }}" ) |
| 164 | + [ "${{ inputs.units || 0 }}" -gt 0 ] 2>/dev/null && args+=( --units "${{ inputs.units }}" ) |
| 165 | + [ "${{ inputs.fanin || 0 }}" -gt 0 ] 2>/dev/null && args+=( --fanin "${{ inputs.fanin }}" ) |
143 | 166 | "$BENCH" \ |
144 | | - --engines '${{ inputs.engines }}' \ |
145 | | - --variants '${{ inputs.variants }}' \ |
146 | | - --scenarios '${{ inputs.scenarios }}' \ |
147 | | - --profile '${{ inputs.profile }}' \ |
| 167 | + --engines '${{ inputs.engines || 'mcpp,cmake,xmake,meson,bazel' }}' \ |
| 168 | + --variants '${{ inputs.variants || 'headers,modules,modules-impl' }}' \ |
| 169 | + --scenarios '${{ inputs.scenarios || 'cold,noop,touch-hub,touch-leaf,edit-body,edit-comment' }}' \ |
| 170 | + --profile '${{ inputs.profile || 'release' }}' \ |
148 | 171 | "${args[@]}" \ |
149 | | - --runs '${{ inputs.runs }}' \ |
| 172 | + --runs '${{ inputs.runs || 0 }}' \ |
150 | 173 | --work "$RUNNER_TEMP/bench-work" \ |
151 | 174 | --out "bench-${{ matrix.name }}.json" |
152 | 175 |
|
|
0 commit comments