ci: add cross-platform benchmark comparison against the branch baseline

Runs the end-to-end benchmarks at the merge-base and at head on the same runner, on
arm64 and x64, and reports per-benchmark deltas. Significance comes from confidence
interval overlap rather than a raw percentage: comparing two runs of identical code
locally produced up to 8.6 percent drift, and one case cleared a 5 percent threshold
with nothing changed.

Author: unsafePtr

.github/scripts/compare-bench.cs

@@ -0,0 +1,221 @@
+// Compares two BenchmarkDotNet artifact directories and reports the per-benchmark delta.
+//
+//   dotnet run .github/scripts/compare-bench.cs -- <baseline-dir> <head-dir> [options]
+//
+//     --threshold N          percent change treated as meaningful once significant (default 10)
+//     --label TEXT           prefix for the report heading and warning annotations
+//     --fail-on-regression   exit non-zero when a benchmark regresses beyond the threshold
+//
+// Significance is decided by confidence-interval overlap, not by a bare percentage. Shared CI
+// runners drift enough that a percentage on means alone invents regressions. If the two intervals
+// overlap, the runs are statistically indistinguishable and the row is reported as noise however
+// large the difference in means looks. The percentage threshold then applies on top of a result that
+// is already significant, filtering out real-but-trivial movement.
+
+using System.Globalization;
+using System.Text;
+using System.Text.Json;
+
+var positional = new List<string>();
+var threshold = 10.0;
+var label = "";
+var failOnRegression = false;
+
+for (var i = 0; i < args.Length; i++)
+{
+    switch (args[i])
+    {
+        case "--threshold":
+            threshold = double.Parse(args[++i], CultureInfo.InvariantCulture);
+            break;
+        case "--label":
+            label = args[++i];
+            break;
+        case "--fail-on-regression":
+            failOnRegression = true;
+            break;
+        default:
+            positional.Add(args[i]);
+            break;
+    }
+}
+
+if (positional.Count < 2)
+{
+    Console.Error.WriteLine("usage: compare-bench.cs <baseline-dir> <head-dir> [--threshold N] [--label TEXT] [--fail-on-regression]");
+    return 2;
+}
+
+var (baseline, baselineFiles) = Load(positional[0]);
+var (head, headFiles) = Load(positional[1]);
+
+if (baseline.Count == 0 || head.Count == 0)
+{
+    Console.WriteLine($"::error::No benchmark results found (baseline: {baseline.Count} records from " +
+                      $"{baselineFiles} files, head: {head.Count} records from {headFiles} files)");
+    return 1;
+}
+
+var rows = new List<string[]>();
+var regressions = new List<(string Name, double Delta)>();
+var improvements = new List<(string Name, double Delta)>();
+
+foreach (var name in baseline.Keys.Union(head.Keys).OrderBy(k => k, StringComparer.Ordinal))
+{
+    var shortName = ShortName(name);
+    var hasBefore = baseline.TryGetValue(name, out var before);
+    var hasAfter = head.TryGetValue(name, out var after);
+
+    if (!hasBefore)
+    {
+        rows.Add([shortName, "-", after.Mean.ToString("F1", CultureInfo.InvariantCulture), "new", ""]);
+        continue;
+    }
+
+    if (!hasAfter)
+    {
+        rows.Add([shortName, before.Mean.ToString("F1", CultureInfo.InvariantCulture), "-", "removed", ""]);
+        continue;
+    }
+
+    var delta = (after.Mean - before.Mean) / before.Mean * 100.0;
+    // Non-overlapping intervals mean the difference exceeds the measured noise.
+    var overlap = !(after.Lower > before.Upper || after.Upper < before.Lower);
+
+    string verdict;
+    if (overlap)
+    {
+        verdict = "noise";
+    }
+    else if (delta > threshold)
+    {
+        verdict = "SLOWER";
+        regressions.Add((shortName, delta));
+    }
+    else if (delta < -threshold)
+    {
+        verdict = "faster";
+        improvements.Add((shortName, delta));
+    }
+    else
+    {
+        verdict = "same";
+    }
+
+    rows.Add([
+        shortName,
+        before.Mean.ToString("F1", CultureInfo.InvariantCulture),
+        after.Mean.ToString("F1", CultureInfo.InvariantCulture),
+        verdict,
+        delta.ToString("+0.0;-0.0", CultureInfo.InvariantCulture) + "%",
+    ]);
+}
+
+var report = new StringBuilder();
+report.AppendLine($"## Benchmark{(label.Length > 0 ? ": " + label : "")}");
+report.AppendLine();
+report.AppendLine("Mean nanoseconds. `noise` means the confidence intervals overlap, so the two runs are "
+                + "statistically indistinguishable regardless of the percentage shown.");
+report.AppendLine();
+report.AppendLine("| Benchmark | Base | Head | Verdict | Delta |");
+report.AppendLine("| --- | ---: | ---: | --- | ---: |");
+foreach (var row in rows)
+{
+    report.AppendLine("| " + string.Join(" | ", row) + " |");
+}
+
+report.AppendLine();
+if (regressions.Count > 0)
+{
+    report.AppendLine($"**{regressions.Count} significant regression(s) over {threshold}%:** "
+                    + string.Join(", ", regressions.Select(r => $"{r.Name} ({r.Delta:+0.0;-0.0}%)")));
+}
+
+if (improvements.Count > 0)
+{
+    report.AppendLine($"**{improvements.Count} significant improvement(s):** "
+                    + string.Join(", ", improvements.Select(r => $"{r.Name} ({r.Delta:+0.0;-0.0}%)")));
+}
+
+if (regressions.Count == 0 && improvements.Count == 0)
+{
+    report.AppendLine("No statistically significant change.");
+}
+
+Console.WriteLine(report.ToString());
+
+var summary = Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY");
+if (!string.IsNullOrEmpty(summary))
+{
+    File.AppendAllText(summary, report.ToString() + Environment.NewLine);
+}
+
+foreach (var (name, delta) in regressions)
+{
+    Console.WriteLine($"::warning::{label} {name} is {delta:+0.0;-0.0}% slower than baseline");
+}
+
+if (regressions.Count > 0 && failOnRegression)
+{
+    Console.WriteLine($"::error::{regressions.Count} benchmark(s) regressed beyond {threshold}%");
+    return 1;
+}
+
+return 0;
+
+static (Dictionary<string, Stat> Results, int FileCount) Load(string directory)
+{
+    var results = new Dictionary<string, Stat>(StringComparer.Ordinal);
+    var files = Directory.GetFiles(directory, "*-report-full-compressed.json", SearchOption.AllDirectories);
+    if (files.Length == 0)
+    {
+        // Older BenchmarkDotNet versions emit the uncompressed name instead.
+        files = Directory.GetFiles(directory, "*-report-full.json", SearchOption.AllDirectories);
+    }
+
+    foreach (var path in files)
+    {
+        // ReadAllText strips the UTF-8 BOM that BenchmarkDotNet writes; JsonDocument would choke on it.
+        using var document = JsonDocument.Parse(File.ReadAllText(path));
+        if (!document.RootElement.TryGetProperty("Benchmarks", out var benchmarks))
+        {
+            continue;
+        }
+
+        foreach (var bench in benchmarks.EnumerateArray())
+        {
+            if (!bench.TryGetProperty("Statistics", out var stats) || stats.ValueKind != JsonValueKind.Object)
+            {
+                continue;
+            }
+
+            var mean = stats.GetProperty("Mean").GetDouble();
+            var lower = mean;
+            var upper = mean;
+            if (stats.TryGetProperty("ConfidenceInterval", out var ci) && ci.ValueKind == JsonValueKind.Object)
+            {
+                lower = ci.GetProperty("Lower").GetDouble();
+                upper = ci.GetProperty("Upper").GetDouble();
+            }
+
+            // Key on DisplayInfo, not FullName. FullName omits the job, so a class carrying two
+            // [SimpleJob] attributes (several here pair Net90 with Net10_0) or a --job argument on the
+            // command line produces multiple records sharing one FullName. Keying on FullName silently
+            // keeps whichever was parsed last, and can pair a ShortRun baseline against a default-job
+            // head -- a plausible-looking number that means nothing.
+            var key = bench.GetProperty("DisplayInfo").GetString()!;
+            results[key] = new Stat(mean, lower, upper);
+        }
+    }
+
+    return (results, files.Length);
+}
+
+// "Class.Method: .NET 10.0(Runtime=.NET 10.0) [Size=32]" -> "Class.Method: .NET 10.0 [Size=32]"
+static string ShortName(string display)
+{
+    var trimmed = System.Text.RegularExpressions.Regex.Replace(display, @"\([^)]*\)", "");
+    return trimmed.Replace("Base58Encoding.Benchmarks.", "").Trim();
+}
+
+readonly record struct Stat(double Mean, double Lower, double Upper);

.github/workflows/benchmark.yml

@@ -0,0 +1,148 @@
+name: benchmark
+
+# Runs the end-to-end benchmarks twice on the SAME runner -- once at a baseline commit, once at the
+# current head -- and reports the per-benchmark delta. Both runs share one physical machine and one
+# job, which is the only meaningful noise control available on shared CI hardware: comparing numbers
+# across two jobs, or against numbers recorded on another day, mostly measures the runners.
+#
+# Intended to be manual: a full matrix run is two revisions x several minutes of compute on two
+# runners, far too expensive to attach to every push. The push trigger below is a temporary
+# bootstrap and is scoped to this workflow's own files.
+#
+# To measure a runner's own noise floor, set baseline_ref to the same commit as the branch: the
+# workflow then compares a revision against itself and everything it reports is drift, not signal.
+
+on:
+  # TEMPORARY: workflow_dispatch only becomes dispatchable once the file is on the default branch, so
+  # this trigger exists purely to validate the workflow before merge. Remove it afterwards -- a full
+  # matrix run per push is far too expensive to keep.
+  push:
+    branches: [ 'perf/widening-multiply-32' ]
+    paths:
+      - '.github/workflows/benchmark.yml'
+      - '.github/scripts/compare-bench.cs'
+  workflow_dispatch:
+    inputs:
+      baseline_ref:
+        description: 'Baseline git ref. Default: merge-base with origin/master (i.e. where this branch started).'
+        required: false
+        type: string
+      filter:
+        description: 'BenchmarkDotNet --filter glob.'
+        required: false
+        default: '*EndToEnd*Benchmark*'
+        type: string
+      threshold:
+        description: 'Percent change treated as meaningful, on top of non-overlapping confidence intervals.'
+        required: false
+        default: '10'
+        type: string
+      fail_on_regression:
+        description: 'Fail the job when a benchmark regresses beyond the threshold.'
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  bench:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04-arm
+            label: arm64
+          - runner: ubuntu-24.04
+            label: x64
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
+    permissions:
+      contents: read
+
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        # Full history: the default baseline is the merge-base with master, which a shallow clone
+        # cannot compute.
+        fetch-depth: 0
+
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v5
+      with:
+        dotnet-version: 10.0.x
+
+    # Record the actual CPU. GitHub's x64 pool mixes Intel Xeon and AMD EPYC, and they differ enough
+    # (AVX-512 presence, cache, clocks) that a delta is only interpretable next to the model name.
+    - name: Host
+      run: |
+        uname -m
+        if command -v lscpu >/dev/null 2>&1; then
+          lscpu | sed -n '1,20p'
+        else
+          sysctl -n machdep.cpu.brand_string hw.ncpu
+        fi
+
+    - name: Resolve baseline
+      id: baseline
+      run: |
+        set -euo pipefail
+        if [ -n "${{ inputs.baseline_ref }}" ]; then
+          BASE=$(git rev-parse "${{ inputs.baseline_ref }}")
+        else
+          git fetch --no-tags origin master
+          BASE=$(git merge-base origin/master HEAD)
+        fi
+        HEAD_SHA=$(git rev-parse HEAD)
+        echo "base=$BASE"       >> "$GITHUB_OUTPUT"
+        echo "head=$HEAD_SHA"   >> "$GITHUB_OUTPUT"
+        echo "Baseline : $BASE ($(git log -1 --format=%s "$BASE"))"
+        echo "Head     : $HEAD_SHA ($(git log -1 --format=%s "$HEAD_SHA"))"
+        if [ "$BASE" = "$HEAD_SHA" ]; then
+          echo "::notice::Baseline and head are the same commit - this run measures the runner's noise floor."
+        fi
+
+    # --launchCount 3 starts three separate processes per benchmark so the reported confidence
+    # interval includes process-to-process variance, not just within-process variance. Without it the
+    # intervals are tight enough (~1.7% margin) that ordinary drift clears them and reads as a real
+    # regression; that was reproduced locally on identical code at +5.4%.
+    #
+    # No --job flag: the benchmark classes already carry [SimpleJob(RuntimeMoniker.Net10_0)], and a
+    # --job argument ADDS a second job rather than replacing it, producing two sets of results per
+    # benchmark under one FullName.
+    - name: Benchmark baseline
+      run: |
+        set -euo pipefail
+        git checkout --quiet --detach ${{ steps.baseline.outputs.base }}
+        dotnet restore src/Base58Encoding.slnx
+        dotnet run --project src/Base58Encoding.Benchmarks/Base58Encoding.Benchmarks.csproj \
+          --configuration Release -- \
+          --filter '${{ inputs.filter || '*EndToEnd*Benchmark*' }}' --launchCount 3 \
+          --exporters json --artifacts "$RUNNER_TEMP/bench-base"
+
+    - name: Benchmark head
+      run: |
+        set -euo pipefail
+        git checkout --quiet --detach ${{ steps.baseline.outputs.head }}
+        dotnet restore src/Base58Encoding.slnx
+        dotnet run --project src/Base58Encoding.Benchmarks/Base58Encoding.Benchmarks.csproj \
+          --configuration Release -- \
+          --filter '${{ inputs.filter || '*EndToEnd*Benchmark*' }}' --launchCount 3 \
+          --exporters json --artifacts "$RUNNER_TEMP/bench-head"
+
+    # A .NET 10 file-based app rather than a shell or python script: the SDK is already set up on the
+    # runner, so this needs no extra toolchain, and it stays in the language of the repo.
+    - name: Compare
+      run: |
+        dotnet run .github/scripts/compare-bench.cs -- \
+          "$RUNNER_TEMP/bench-base" "$RUNNER_TEMP/bench-head" \
+          --threshold '${{ inputs.threshold || '10' }}' \
+          --label '${{ matrix.label }} (${{ matrix.runner }})' \
+          ${{ inputs.fail_on_regression && '--fail-on-regression' || '' }}
+
+    - name: Upload raw results
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: bench-${{ matrix.label }}
+        path: |
+          ${{ runner.temp }}/bench-base/**/*.json
+          ${{ runner.temp }}/bench-head/**/*.json