Add configurable tags to runners

CHANGELOG.md

@@ -100,6 +100,37 @@ See `README.md` for more information.
 python -m pytest -m "not long_running"
 ```
 
+### Configurable tags
+
+Runners can have tags attached, and benchmark runs can be started on
+everything with a specific tag. For example, a configuration like:
+
+```toml
+[runners.linux_clang]
+os = "linux"
+arch = "x86_64"
+hostname = "pyperf1"
+env.CC = "clang"
+tags = ["linux", "pyperf1", "clang"]
+
+[runners.linux_gcc]
+os = "linux"
+arch = "x86_64"
+hostname = "pyperf1"
+tags = ["linux", "pyperf1", "gcc"]
+
+[runners.linux2_gcc]
+os = "linux"
+arch = "x86_64"
+hostname = "pyperf2"
+tags = ["linux", "pyperf2", "gcc"]
+```
+
+... will add `tag linux`, `tag pyperf1`, `tag pyperf2`, `tag gcc`
+and `tag clang` to the list of possible machines for benchmark runs.
+Selecting `tag linux` will queue a run for all three runners, and `tag
+pyperf1` only for the first two.
+
 ## v1.8.0
 
 ### bench_runner.toml change

bench_runner/runners.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 
+import collections
 import functools
 import os
 import socket
@@ -34,6 +35,7 @@ def __init__(
         github_runner_name: str | None,
         include_in_all: bool = True,
         plot: dict[str, str] | None = None,
+        tags: list[str] | None = None,
     ):
         self.nickname = nickname
         self.os = os
@@ -48,6 +50,7 @@ def __init__(
         if plot is None:
             plot = {"name": nickname}
         self.plot = PlotConfig(**plot)
+        self.tags = tags
 
     @property
     def name(self) -> str:
@@ -77,6 +80,7 @@ def get_runners(cfgpath: PathLike | None = None) -> list[Runner]:
                 section.get("github_runner_name"),
                 section.get("include_in_all", True),
                 section.get("plot", None),
+                section.get("tags"),
             )
         )
 
@@ -117,3 +121,31 @@ def get_runner_for_hostname(
     if hostname is None:
         hostname = socket.gethostname()
     return get_runners_by_hostname(cfgpath).get(hostname, unknown_runner)
+
+
+def get_tags(cfgpath: PathLike | None = None) -> dict[str, list[Runner]]:
+    d = collections.defaultdict(list)
+    for runner in get_runners(cfgpath):
+        if runner.tags:
+            for tag in runner.tags:
+                d[tag].append(runner)
+    return dict(d)
+
+
+def get_runners_from_nicknames_and_tags(
+    nicknames: list[str], cfgpath: PathLike | None = None
+) -> list[Runner]:
+    result = []
+    tags = get_tags(cfgpath)
+    runners = get_runners_by_nickname(cfgpath)
+    for nickname in nicknames:
+        if nickname.startswith("tag "):
+            tag = nickname.removeprefix("tag ")
+            if tag not in tags:
+                raise ValueError(f"Tag {tag} not found in bench_runner.toml")
+            result.extend(tags[nickname.removeprefix("tag ")])
+        else:
+            if nickname not in runners:
+                raise ValueError(f"Runner {nickname} not found in bench_runner.toml")
+            result.append(runners[nickname])
+    return result

bench_runner/scripts/get_merge_base.py

@@ -9,6 +9,7 @@
 from bench_runner import benchmark_definitions
 from bench_runner import flags as mflags
 from bench_runner import git
+from bench_runner import runners as mrunners
 from bench_runner.result import has_result
 from bench_runner.util import PathLike
 
@@ -40,29 +41,38 @@ def _main(
     if not need_to_run:
         print("ref=xxxxxxx")
         print("need_to_run=false")
+        return
+    merge_base = git.get_git_merge_base(cpython)
+
+    if merge_base is None:
+        print("ref=xxxxxxx")
+        print("need_to_run=false")
+        return
+
+    if machine in ("__really_all", "all"):
+        need_to_run = True
     else:
-        merge_base = git.get_git_merge_base(cpython)
-
-        if merge_base is None:
-            print("ref=xxxxxxx")
-            print("need_to_run=false")
-        else:
-            need_to_run = (
-                machine in ("__really_all", "all")
-                or has_result(
+        machines = [
+            r.nickname for r in mrunners.get_runners_from_nicknames_and_tags([machine])
+        ]
+        for m in machines:
+            if (
+                has_result(
                     Path("results"),
                     merge_base,
-                    machine,
+                    m,
                     pystats,
                     flags,
                     benchmark_definitions.get_benchmark_hash(),
                     progress=False,
                 )
                 is None
-            )
+            ):
+                need_to_run = True
+                break
 
-            print(f"ref={merge_base}")
-            print(f"need_to_run={str(need_to_run).lower()}")
+    print(f"ref={merge_base}")
+    print(f"need_to_run={str(need_to_run).lower()}")
 
 
 def main():

bench_runner/scripts/install.py

@@ -165,6 +165,7 @@ def generate__benchmark(src: Any) -> Any:
             github_env = "$GITHUB_ENV"
         vars = copy.copy(runner.env)
         vars["BENCHMARK_MACHINE_NICKNAME"] = runner.nickname
+        vars["BENCHMARK_RUNNER_NAME"] = runner.name
         setup_environment = {
             "name": "Setup environment",
             "run": LiteralScalarString(
@@ -183,6 +184,11 @@ def generate__benchmark(src: Any) -> Any:
         ]
         if runner.include_in_all:
             machine_clauses.append("inputs.machine == 'all'")
+        if runner.tags:
+            for tag in runner.tags:
+                if "'" in tag:
+                    raise ValueError(f"tag cannot contain `'` (runner {runner.name})")
+                machine_clauses.append(f"inputs.machine == 'tag {tag}'")
         runner_template["if"] = f"${{{{ ({' || '.join(machine_clauses)}) }}}}"
 
         dst["jobs"][f"benchmark-{runner.name}"] = runner_template
@@ -205,11 +211,17 @@ def generate_benchmark(dst: Any) -> Any:
     """
     Generates benchmark.yml from benchmark.src.yml.
 
-    Inserts the list of available machines to the drop-down presented to the
-    user.
+    Inserts the list of tags and available machines to the drop-down
+    presented to the user.
     """
     available_runners = [r for r in runners.get_runners() if r.available]
-    runner_choices = [*[x.name for x in available_runners], "all", "__really_all"]
+    tags = sorted(set(f"tag {g}" for r in available_runners if r.tags for g in r.tags))
+    runner_choices = [
+        *tags,
+        *[x.name for x in available_runners],
+        "all",
+        "__really_all",
+    ]
 
     dst["on"]["workflow_dispatch"]["inputs"]["machine"]["options"] = runner_choices
 
@@ -264,12 +276,10 @@ def generate__weekly(dst: Any) -> Any:
     all_jobs = []
 
     for name, weekly_cfg in weekly.items():
-        for runner_nickname in weekly_cfg.get("runners", []):
-            runner = runners.get_runner_by_nickname(runner_nickname)
-            if runner.nickname == "unknown":
-                raise ValueError(
-                    f"Runner {runner_nickname} not found in bench_runner.toml"
-                )
+        cfg_runners = runners.get_runners_from_nicknames_and_tags(
+            weekly_cfg.get("runners", [])
+        )
+        for runner in cfg_runners:
             weekly_flags = weekly_cfg.get("flags", [])
             job = {
                 "uses": "./.github/workflows/_benchmark.yml",

bench_runner/templates/_benchmark.src.yml

@@ -73,7 +73,9 @@ jobs:
           git gc
       - name: Building Python and running pyperformance
         run: |
-          py workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
+          py workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} `
+              ${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '$env:BENCHMARK_RUNNER_NAME' }} `
+              ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
         run: |
@@ -112,7 +114,9 @@ jobs:
           python-version: "3.11"
       - name: Building Python and running pyperformance
         run: |
-          python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} ${{ inputs.perf && '--perf' || '' }} --run_id ${{ github.run_id }}
+          python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} \
+              ${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '"$BENCHMARK_RUNNER_NAME"' }} \
+              ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} ${{ inputs.perf && '--perf' || '' }} --run_id ${{ github.run_id }}
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
         if: ${{ !inputs.perf }}
@@ -154,7 +158,9 @@ jobs:
           git gc
       - name: Building Python and running pyperformance
         run: |
-          python3 workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
+          python3 workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} \
+              ${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '"$BENCHMARK_RUNNER_NAME"' }} \
+              ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
         run: |

bench_runner/templates/benchmark.src.yml

@@ -68,7 +68,7 @@ jobs:
       - name: Determine base
         id: base
         run: |
-          python -m bench_runner get_merge_base ${{ inputs.benchmark_base }} ${{ inputs.machine }} ${{ inputs.pystats }} ${{ env.flags }} >> $GITHUB_OUTPUT
+          python -m bench_runner get_merge_base ${{ inputs.benchmark_base }} '${{ inputs.machine }}' ${{ inputs.pystats }} ${{ env.flags }} >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
 
   head:
@@ -80,7 +80,7 @@ jobs:
       benchmarks: ${{ inputs.benchmarks }}
       pgo: true
       perf: false
-      force: true
+      force: false
     secrets: inherit
 
   base: