Compare benchmarks against a pinned release baseline

The benchmark job in test_gpu.yml now fails when a median runtime is
more than 25% slower than the benchmark JSON attached to the release
named by BENCHMARK_BASELINE_TAG (v0.2.0). upload_benchmark from #77
attaches that JSON when a release is published, so bumping the tag
after a release moves the baseline.

A small ubuntu-22.04 job finds the one .json asset on the pinned
release and hands it to the benchmark job as an artifact, because the
GPU runner image was reported in 2025 to lack gh
(actions/runner-images#12398). A tag with no release, or a release
with more than one JSON asset, fails that job and the benchmark job
does not run. A release with no JSON asset only warns, and the run is
recorded without a check.

Release runs skip the lookup and record without comparing, so a
regression or a lookup problem cannot keep upload_benchmark from
attaching results. The plot and upload steps run even when the check
fails.

pytest-benchmark passes benchmarks missing from the baseline without
comparing them, so a follow-up step warns about those and fails when
the run shares no benchmark with the baseline. The machine_info
mismatch warning is shown without failing, because filterwarnings =
error would otherwise abort the session.

Author: Kayvan-Zahiri

.github/workflows/test_gpu.yml

@@ -26,6 +26,11 @@ defaults:
   run:
     shell: bash
 
+env:
+  # Release whose attached benchmark JSON is the regression baseline.
+  # Bump it after publishing a release that has benchmark results attached.
+  BENCHMARK_BASELINE_TAG: v0.2.0
+
 jobs:
   test-gpu:
     strategy:
@@ -55,7 +60,46 @@ jobs:
         run: make test
 
 
+  benchmark-baseline:
+    # Releases record results for upload_benchmark without a regression check
+    if: github.event_name != 'release'
+    # The GPU runner image was reported in 2025 to lack gh (actions/runner-images#12398)
+    runs-on: ubuntu-22.04
+    outputs:
+      found: ${{ steps.download.outputs.found }}
+
+    steps:
+      - name: Download benchmark baseline
+        id: download
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # A tag with no release fails here; a release without a benchmark JSON only warns
+          assets=$(gh release view "$BENCHMARK_BASELINE_TAG" --repo "$GITHUB_REPOSITORY" --json assets --jq '.assets[].name') \
+            || { echo "::error::Could not read the release for BENCHMARK_BASELINE_TAG=$BENCHMARK_BASELINE_TAG"; exit 1; }
+          json=$(grep -E '\.json$' <<< "$assets" || true)
+          count=$(grep -c . <<< "$json" || true)
+          if [ "$count" -gt 1 ]; then
+            echo "::error::Release $BENCHMARK_BASELINE_TAG has more than one JSON asset: $(tr '\n' ' ' <<< "$json")"
+            exit 1
+          elif [ "$count" -eq 1 ]; then
+            gh release download "$BENCHMARK_BASELINE_TAG" --repo "$GITHUB_REPOSITORY" --pattern "$json" --output benchmark-baseline.json
+            echo "found=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "Release $BENCHMARK_BASELINE_TAG has no benchmark JSON"
+          fi
+
+      - name: Upload benchmark baseline
+        if: steps.download.outputs.found == 'true'
+        uses: actions/upload-artifact@v5
+        with:
+          name: benchmark-baseline
+          path: benchmark-baseline.json
+
   benchmark:
+    needs: benchmark-baseline
+    # Runs on releases, where benchmark-baseline is skipped, and when the baseline lookup succeeded
+    if: ${{ !cancelled() && (github.event_name == 'release' || needs.benchmark-baseline.result == 'success') }}
     strategy:
       matrix:
         python-version: ["3.11"]
@@ -79,18 +123,54 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Download benchmark baseline
+        if: needs.benchmark-baseline.outputs.found == 'true'
+        uses: actions/download-artifact@v6
+        with:
+          name: benchmark-baseline
+
+      - name: Warn about a missing benchmark baseline
+        if: github.event_name != 'release' && needs.benchmark-baseline.outputs.found != 'true'
+        run: |
+          msg="Benchmark baseline missing: release $BENCHMARK_BASELINE_TAG has no benchmark JSON, so this run is not checked for regressions."
+          echo "::warning::$msg"
+          echo "$msg" >> "$GITHUB_STEP_SUMMARY"
+
       - name: Run CUDA benchmark
-        run: make benchmark
+        # Without benchmark-baseline.json this records a run without comparing
+        run: make benchmark-compare BENCHMARK_BASELINE=benchmark-baseline.json
+
+      - name: Check which benchmarks the baseline covers
+        if: always() && hashFiles('benchmark-baseline.json') != '' && hashFiles('.benchmarks/*/*.json') != ''
+        run: |
+          uv run --no-sync python - <<'EOF'
+          import glob, json
+
+          def names(file):
+              return {b["fullname"] for b in json.load(open(file))["benchmarks"]}
+
+          current = names(glob.glob(".benchmarks/*/*.json")[0])
+          baseline = names("benchmark-baseline.json")
+          if not current & baseline:
+              print("::error::No benchmark from this run is in the baseline, so nothing was checked for regressions")
+              raise SystemExit(1)
+          if current - baseline:
+              print("::warning::Not in the baseline, so not checked for regressions: " + ", ".join(sorted(current - baseline)))
+          EOF
 
       - name: Plot benchmark (runtime)
+        # Keep the results when the regression check fails the job
+        if: always() && hashFiles('.benchmarks/**') != ''
         run:
           uv run --group compare tests/plot_benchmark.py --output .benchmarks/benchmark.png
 
       - name: Plot benchmark (throughput)
+        if: always() && hashFiles('.benchmarks/**') != ''
         run:
           uv run --group compare tests/plot_benchmark.py --points-per-second --output .benchmarks/benchmark_pps.png
 
       - name: Upload benchmark results
+        if: always() && hashFiles('.benchmarks/**') != ''
         uses: actions/upload-artifact@v5
         with:
           name: benchmark-results

BENCHMARKS.md

@@ -72,6 +72,28 @@ uv run --group compare tests/plot_benchmark.py --points-per-second --output asse
 
 The benchmark job in our CI pipeline ([`test_gpu.yml`](https://github.com/Forest-Neurotech/mach/blob/main/.github/workflows/test_gpu.yml)) automatically runs these benchmarks (on a T4 GPU) across different commits and releases, providing continuous performance monitoring.
 
+### Regression checking
+
+`make benchmark-compare` runs the same benchmarks as `make benchmark` and fails when a median runtime is more than 25% slower than a saved pytest-benchmark JSON file:
+
+```bash
+# Compare against a JSON saved under .benchmarks/ by an earlier run
+make benchmark-compare BENCHMARK_BASELINE=path/to/earlier-run.json
+
+# Use a tighter threshold
+make benchmark-compare BENCHMARK_BASELINE=path/to/earlier-run.json BENCHMARK_REGRESSION_THRESHOLD=median:10%
+```
+
+If the baseline file does not exist, the target records a run without comparing.
+Only benchmarks that also appear in the baseline are checked.
+In CI, the others are listed in a warning, and the job fails if no benchmark from the run is in the baseline.
+pytest-benchmark warns that `machine_info` is different when any field of it differs from the baseline, host name included, and still runs the comparison.
+
+In CI, the baseline is the benchmark JSON attached to the GitHub release named by `BENCHMARK_BASELINE_TAG` in [`test_gpu.yml`](https://github.com/Forest-Neurotech/mach/blob/main/.github/workflows/test_gpu.yml), so every run compares against the same results until the tag is bumped.
+Publishing a release runs the benchmark without the regression check and attaches the results to that release; to make them the new baseline, set `BENCHMARK_BASELINE_TAG` to that release's tag.
+If the pinned release has no benchmark JSON, the benchmark job records its results without a regression check and shows a warning, also written to the job summary.
+If `BENCHMARK_BASELINE_TAG` names no release, the release has more than one JSON asset, or the lookup fails for another reason, the `benchmark-baseline` job fails and the benchmark job does not run.
+
 ## CUDA Optimizations
 
 mach optimizes GPU memory access patterns to improve performance. For those interested in learning more about CUDA optimization, excellent resources include:

Makefile

@@ -80,6 +80,21 @@ benchmark: ## Runs benchmarking comparisons
 	@echo "🚀 Running benchmarking comparisons"
 	uv run --group test --group array --group compare pytest tests -v -s --benchmark-only --benchmark-histogram --benchmark-autosave --benchmark-save-data
 
+# pytest-benchmark JSON to compare against (CI downloads it from a pinned release), and the
+# --benchmark-compare-fail expression that counts as a regression.
+BENCHMARK_BASELINE ?= benchmark-baseline.json
+BENCHMARK_REGRESSION_THRESHOLD ?= median:25%
+
+.PHONY: benchmark-compare
+benchmark-compare: ## Runs benchmarks and fails if they regressed against BENCHMARK_BASELINE
+	@if [ -f "$(BENCHMARK_BASELINE)" ]; then \
+		echo "🚀 Comparing against $(BENCHMARK_BASELINE), failing on a $(BENCHMARK_REGRESSION_THRESHOLD) regression"; \
+		uv run --group test --group array --group compare pytest tests -v -s --benchmark-only --benchmark-histogram --benchmark-autosave --benchmark-save-data --benchmark-compare="$(BENCHMARK_BASELINE)" --benchmark-compare-fail=$(BENCHMARK_REGRESSION_THRESHOLD); \
+	else \
+		echo "No baseline at $(BENCHMARK_BASELINE); recording a run without a regression check"; \
+		$(MAKE) --no-print-directory benchmark; \
+	fi
+
 .PHONY: profile
 profile: ## Runs Python test with simple profiling. Recommend using Nsight Compute or Nsight Systems for more detailed profiling.
 	@echo "Building with CUDA_PROFILE"

pyproject.toml

@@ -217,6 +217,8 @@ filterwarnings = [
     # vbeam warnings
     "ignore:point_position will be overwritten by the scan.:UserWarning",
     "ignore:Both point_position and scan are set. Scan will be used.:UserWarning",
+    # benchmark-compare warns when any baseline machine_info field differs, host name included; show it without failing
+    "default:Benchmark machine_info is different:UserWarning",
 ]
 markers = [
     # see conftest.py for default-addition of cuda marker