diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d4d2de..fe1de97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,37 @@ See `README.md` for more information. python -m pytest -m "not long_running" ``` +### Configurable tags + +Runners can have tags attached, and benchmark runs can be started on +everything with a specific tag. For example, a configuration like: + +```toml +[runners.linux_clang] +os = "linux" +arch = "x86_64" +hostname = "pyperf1" +env.CC = "clang" +tags = ["linux", "pyperf1", "clang"] + +[runners.linux_gcc] +os = "linux" +arch = "x86_64" +hostname = "pyperf1" +tags = ["linux", "pyperf1", "gcc"] + +[runners.linux2_gcc] +os = "linux" +arch = "x86_64" +hostname = "pyperf2" +tags = ["linux", "pyperf2", "gcc"] +``` + +... will add `tag linux`, `tag pyperf1`, `tag pyperf2`, `tag gcc` +and `tag clang` to the list of possible machines for benchmark runs. +Selecting `tag linux` will queue a run for all three runners, and `tag +pyperf1` only for the first two. + ## v1.8.0 ### bench_runner.toml change diff --git a/bench_runner/runners.py b/bench_runner/runners.py index 4ce9c69..4be71f3 100644 --- a/bench_runner/runners.py +++ b/bench_runner/runners.py @@ -1,6 +1,7 @@ from __future__ import annotations +import collections import functools import os import socket @@ -34,6 +35,7 @@ def __init__( github_runner_name: str | None, include_in_all: bool = True, plot: dict[str, str] | None = None, + tags: list[str] | None = None, ): self.nickname = nickname self.os = os @@ -48,6 +50,7 @@ def __init__( if plot is None: plot = {"name": nickname} self.plot = PlotConfig(**plot) + self.tags = tags @property def name(self) -> str: @@ -77,6 +80,7 @@ def get_runners(cfgpath: PathLike | None = None) -> list[Runner]: section.get("github_runner_name"), section.get("include_in_all", True), section.get("plot", None), + section.get("tags"), ) ) @@ -117,3 +121,31 @@ def get_runner_for_hostname( if hostname is None: hostname = socket.gethostname() return get_runners_by_hostname(cfgpath).get(hostname, unknown_runner) + + +def get_tags(cfgpath: PathLike | None = None) -> dict[str, list[Runner]]: + d = collections.defaultdict(list) + for runner in get_runners(cfgpath): + if runner.tags: + for tag in runner.tags: + d[tag].append(runner) + return dict(d) + + +def get_runners_from_nicknames_and_tags( + nicknames: list[str], cfgpath: PathLike | None = None +) -> list[Runner]: + result = [] + tags = get_tags(cfgpath) + runners = get_runners_by_nickname(cfgpath) + for nickname in nicknames: + if nickname.startswith("tag "): + tag = nickname.removeprefix("tag ") + if tag not in tags: + raise ValueError(f"Tag {tag} not found in bench_runner.toml") + result.extend(tags[nickname.removeprefix("tag ")]) + else: + if nickname not in runners: + raise ValueError(f"Runner {nickname} not found in bench_runner.toml") + result.append(runners[nickname]) + return result diff --git a/bench_runner/scripts/get_merge_base.py b/bench_runner/scripts/get_merge_base.py index 4209d4b..2986e00 100644 --- a/bench_runner/scripts/get_merge_base.py +++ b/bench_runner/scripts/get_merge_base.py @@ -9,6 +9,7 @@ from bench_runner import benchmark_definitions from bench_runner import flags as mflags from bench_runner import git +from bench_runner import runners as mrunners from bench_runner.result import has_result from bench_runner.util import PathLike @@ -40,29 +41,38 @@ def _main( if not need_to_run: print("ref=xxxxxxx") print("need_to_run=false") + return + merge_base = git.get_git_merge_base(cpython) + + if merge_base is None: + print("ref=xxxxxxx") + print("need_to_run=false") + return + + if machine in ("__really_all", "all"): + need_to_run = True else: - merge_base = git.get_git_merge_base(cpython) - - if merge_base is None: - print("ref=xxxxxxx") - print("need_to_run=false") - else: - need_to_run = ( - machine in ("__really_all", "all") - or has_result( + machines = [ + r.nickname for r in mrunners.get_runners_from_nicknames_and_tags([machine]) + ] + for m in machines: + if ( + has_result( Path("results"), merge_base, - machine, + m, pystats, flags, benchmark_definitions.get_benchmark_hash(), progress=False, ) is None - ) + ): + need_to_run = True + break - print(f"ref={merge_base}") - print(f"need_to_run={str(need_to_run).lower()}") + print(f"ref={merge_base}") + print(f"need_to_run={str(need_to_run).lower()}") def main(): diff --git a/bench_runner/scripts/install.py b/bench_runner/scripts/install.py index 8439106..f114f1f 100644 --- a/bench_runner/scripts/install.py +++ b/bench_runner/scripts/install.py @@ -165,6 +165,7 @@ def generate__benchmark(src: Any) -> Any: github_env = "$GITHUB_ENV" vars = copy.copy(runner.env) vars["BENCHMARK_MACHINE_NICKNAME"] = runner.nickname + vars["BENCHMARK_RUNNER_NAME"] = runner.name setup_environment = { "name": "Setup environment", "run": LiteralScalarString( @@ -183,6 +184,11 @@ def generate__benchmark(src: Any) -> Any: ] if runner.include_in_all: machine_clauses.append("inputs.machine == 'all'") + if runner.tags: + for tag in runner.tags: + if "'" in tag: + raise ValueError(f"tag cannot contain `'` (runner {runner.name})") + machine_clauses.append(f"inputs.machine == 'tag {tag}'") runner_template["if"] = f"${{{{ ({' || '.join(machine_clauses)}) }}}}" dst["jobs"][f"benchmark-{runner.name}"] = runner_template @@ -205,11 +211,17 @@ def generate_benchmark(dst: Any) -> Any: """ Generates benchmark.yml from benchmark.src.yml. - Inserts the list of available machines to the drop-down presented to the - user. + Inserts the list of tags and available machines to the drop-down + presented to the user. """ available_runners = [r for r in runners.get_runners() if r.available] - runner_choices = [*[x.name for x in available_runners], "all", "__really_all"] + tags = sorted(set(f"tag {g}" for r in available_runners if r.tags for g in r.tags)) + runner_choices = [ + *tags, + *[x.name for x in available_runners], + "all", + "__really_all", + ] dst["on"]["workflow_dispatch"]["inputs"]["machine"]["options"] = runner_choices @@ -264,12 +276,10 @@ def generate__weekly(dst: Any) -> Any: all_jobs = [] for name, weekly_cfg in weekly.items(): - for runner_nickname in weekly_cfg.get("runners", []): - runner = runners.get_runner_by_nickname(runner_nickname) - if runner.nickname == "unknown": - raise ValueError( - f"Runner {runner_nickname} not found in bench_runner.toml" - ) + cfg_runners = runners.get_runners_from_nicknames_and_tags( + weekly_cfg.get("runners", []) + ) + for runner in cfg_runners: weekly_flags = weekly_cfg.get("flags", []) job = { "uses": "./.github/workflows/_benchmark.yml", diff --git a/bench_runner/templates/_benchmark.src.yml b/bench_runner/templates/_benchmark.src.yml index 6f7cc1a..5f85cdd 100644 --- a/bench_runner/templates/_benchmark.src.yml +++ b/bench_runner/templates/_benchmark.src.yml @@ -73,7 +73,9 @@ jobs: git gc - name: Building Python and running pyperformance run: | - py workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }} + py workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ` + ${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '$env:BENCHMARK_RUNNER_NAME' }} ` + ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }} # Pull again, since another job may have committed results in the meantime - name: Pull benchmarking run: | @@ -112,7 +114,9 @@ jobs: python-version: "3.11" - name: Building Python and running pyperformance run: | - python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} ${{ inputs.perf && '--perf' || '' }} --run_id ${{ github.run_id }} + python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} \ + ${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '"$BENCHMARK_RUNNER_NAME"' }} \ + ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} ${{ inputs.perf && '--perf' || '' }} --run_id ${{ github.run_id }} # Pull again, since another job may have committed results in the meantime - name: Pull benchmarking if: ${{ !inputs.perf }} @@ -154,7 +158,9 @@ jobs: git gc - name: Building Python and running pyperformance run: | - python3 workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }} + python3 workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} \ + ${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '"$BENCHMARK_RUNNER_NAME"' }} \ + ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }} # Pull again, since another job may have committed results in the meantime - name: Pull benchmarking run: | diff --git a/bench_runner/templates/benchmark.src.yml b/bench_runner/templates/benchmark.src.yml index 67ddc03..9fef141 100644 --- a/bench_runner/templates/benchmark.src.yml +++ b/bench_runner/templates/benchmark.src.yml @@ -68,7 +68,7 @@ jobs: - name: Determine base id: base run: | - python -m bench_runner get_merge_base ${{ inputs.benchmark_base }} ${{ inputs.machine }} ${{ inputs.pystats }} ${{ env.flags }} >> $GITHUB_OUTPUT + python -m bench_runner get_merge_base ${{ inputs.benchmark_base }} '${{ inputs.machine }}' ${{ inputs.pystats }} ${{ env.flags }} >> $GITHUB_OUTPUT cat $GITHUB_OUTPUT head: @@ -80,7 +80,7 @@ jobs: benchmarks: ${{ inputs.benchmarks }} pgo: true perf: false - force: true + force: false secrets: inherit base: