ci: Allow for multiple icount benchmarks in the same run

We don't actually need this for now, but eventually it would be nice to
run icount benchmarks on multiple targets. Start tagging artifact names
with the architecture, and allow passing `--tag` to `ci-util.py` in
order to retrieve the correct one.

Author: tgross35

.github/workflows/main.yaml

@@ -195,8 +195,14 @@ jobs:
 
   benchmarks:
     name: Benchmarks
-    runs-on: ubuntu-24.04
     timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - target: x86_64-unknown-linux-gnu
+          os: ubuntu-24.04
+    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@master
       with:
@@ -215,12 +221,14 @@ jobs:
         cargo binstall -y iai-callgrind-runner --version "$iai_version"
         sudo apt-get install valgrind
     - uses: Swatinem/rust-cache@v2
+      with:
+        key: ${{ matrix.target }}
 
     - name: Run icount benchmarks
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         PR_NUMBER: ${{ github.event.pull_request.number }}
-      run: ./ci/bench-icount.sh
+      run: ./ci/bench-icount.sh ${{ matrix.target }}
 
     - name: Upload the benchmark baseline
       uses: actions/upload-artifact@v4

ci/bench-icount.sh

@@ -2,10 +2,21 @@
 
 set -eux
 
+target="${1:-}"
+
+if [ -z "$target" ]; then
+    host_target=$(rustc -vV | awk '/^host/ { print $2 }')
+    echo "Defaulted to host target $host_target"
+    target="$host_target"
+fi
+
 iai_home="iai-home"
 
+# Use the arch as a tag to disambiguate artifacts
+tag="$(echo "$target" | cut -d'-' -f1)"
+
 # Download the baseline from master
-./ci/ci-util.py locate-baseline --download --extract
+./ci/ci-util.py locate-baseline --download --extract --tag "$tag"
 
 # Run benchmarks once
 function run_icount_benchmarks() {
@@ -44,6 +55,7 @@ function run_icount_benchmarks() {
         # If this is for a pull request, ignore regressions if specified.
         ./ci/ci-util.py check-regressions --home "$iai_home" --allow-pr-override "$PR_NUMBER"
     else
+        # Disregard regressions after merge
         ./ci/ci-util.py check-regressions --home "$iai_home" || true
     fi
 }
@@ -53,6 +65,6 @@ run_icount_benchmarks --features force-soft-floats -- --save-baseline=softfloat
 run_icount_benchmarks -- --save-baseline=hardfloat
 
 # Name and tar the new baseline
-name="baseline-icount-$(date -u +'%Y%m%d%H%M')-${GITHUB_SHA:0:12}"
+name="baseline-icount-$tag-$(date -u +'%Y%m%d%H%M')-${GITHUB_SHA:0:12}"
 echo "BASELINE_NAME=$name" >>"$GITHUB_ENV"
 tar cJf "$name.tar.xz" "$iai_home"

ci/ci-util.py

@@ -28,11 +28,14 @@
             Calculate a matrix of which functions had source change, print that as
             a JSON object.
 
-        locate-baseline [--download] [--extract]
+        locate-baseline [--download] [--extract] [--tag TAG]
             Locate the most recent benchmark baseline available in CI and, if flags
             specify, download and extract it. Never exits with nonzero status if
             downloading fails.
 
+            `--tag` can be specified to look for artifacts with a specific tag, such as
+            for a specific architecture.
+
             Note that `--extract` will overwrite files in `iai-home`.
 
         check-regressions [--home iai-home] [--allow-pr-override pr_number]
@@ -50,7 +53,7 @@
 GIT = ["git", "-C", REPO_ROOT]
 DEFAULT_BRANCH = "master"
 WORKFLOW_NAME = "CI"  # Workflow that generates the benchmark artifacts
-ARTIFACT_GLOB = "baseline-icount*"
+ARTIFACT_PREFIX = "baseline-icount*"
 # Place this in a PR body to skip regression checks (must be at the start of a line).
 REGRESSION_DIRECTIVE = "ci: allow-regressions"
 # Place this in a PR body to skip extensive tests
@@ -278,13 +281,17 @@ def locate_baseline(flags: list[str]) -> None:
 
     download = False
     extract = False
+    tag = ""
 
     while len(flags) > 0:
         match flags[0]:
             case "--download":
                 download = True
             case "--extract":
                 extract = True
+            case "--tag":
+                tag = flags[1]
+                flags = flags[1:]
             case _:
                 eprint(USAGE)
                 exit(1)
@@ -333,8 +340,10 @@ def locate_baseline(flags: list[str]) -> None:
         eprint("skipping download step")
         return
 
+    artifact_glob = f"{ARTIFACT_PREFIX}{f"-{tag}" if tag else ""}*"
+
     sp.run(
-        ["gh", "run", "download", str(job_id), f"--pattern={ARTIFACT_GLOB}"],
+        ["gh", "run", "download", str(job_id), f"--pattern={artifact_glob}"],
         check=False,
     )
 
@@ -344,7 +353,7 @@ def locate_baseline(flags: list[str]) -> None:
 
     # Find the baseline with the most recent timestamp. GH downloads the files to e.g.
     # `some-dirname/some-dirname.tar.xz`, so just glob the whole thing together.
-    candidate_baselines = glob(f"{ARTIFACT_GLOB}/{ARTIFACT_GLOB}")
+    candidate_baselines = glob(f"{artifact_glob}/{artifact_glob}")
     if len(candidate_baselines) == 0:
         eprint("no possible baseline directories found")
         return