Skip to content

Commit cf147dd

Browse files
Yhg1sYhg1s
committed
Add configurable groups.
1 parent aa2ac8c commit cf147dd

File tree

6 files changed

+99
-20
lines changed

6 files changed

+99
-20
lines changed

‎CHANGELOG.md

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,39 @@ See `README.md` for more information.
100100
python -m pytest -m "not long_running"
101101
```
102102

103+
### Configurable groups
104+
105+
Runners can configure which groups they belong to, and benchmark runs can be
106+
started on whole groups. Unlike when using 'all' or individual runners, runs
107+
for which the result already exists are skipped. For example, a
108+
configuration like:
109+
110+
```toml
111+
[runners.linux_clang]
112+
os = "linux"
113+
arch = "x86_64"
114+
hostname = "pyperf1"
115+
env.CC = "clang"
116+
groups = ["linux", "pyperf1", "clang"]
117+
118+
[runners.linux_gcc]
119+
os = "linux"
120+
arch = "x86_64"
121+
hostname = "pyperf1"
122+
groups = ["linux", "pyperf1", "gcc"]
123+
124+
[runners.linux2_gcc]
125+
os = "linux"
126+
arch = "x86_64"
127+
hostname = "pyperf2"
128+
groups = ["linux", "pyperf2", "gcc"]
129+
```
130+
131+
... will add `group linux`, `group pyperf1`, `group pyperf2`, `group gcc`
132+
and `group clang` to the list of possible machines for benchmark runs.
133+
Selecting `group linux` will queue a run for all three runners, and `group
134+
pyperf1` only for the first two.
135+
103136
## v1.8.0
104137

105138
### bench_runner.toml change

‎bench_runner/runners.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from __future__ import annotations
22

33

4+
import collections
45
import functools
56
import os
67
import socket
@@ -34,6 +35,7 @@ def __init__(
3435
github_runner_name: str | None,
3536
include_in_all: bool = True,
3637
plot: dict[str, str] | None = None,
38+
groups: list[str] | None = None,
3739
):
3840
self.nickname = nickname
3941
self.os = os
@@ -48,6 +50,7 @@ def __init__(
4850
if plot is None:
4951
plot = {"name": nickname}
5052
self.plot = PlotConfig(**plot)
53+
self.groups = groups
5154

5255
@property
5356
def name(self) -> str:
@@ -77,6 +80,7 @@ def get_runners(cfgpath: PathLike | None = None) -> list[Runner]:
7780
section.get("github_runner_name"),
7881
section.get("include_in_all", True),
7982
section.get("plot", None),
83+
section.get("groups"),
8084
)
8185
)
8286

@@ -117,3 +121,12 @@ def get_runner_for_hostname(
117121
if hostname is None:
118122
hostname = socket.gethostname()
119123
return get_runners_by_hostname(cfgpath).get(hostname, unknown_runner)
124+
125+
126+
def get_groups(cfgpath: PathLike | None = None) -> dict[str, list[Runner]]:
127+
d = collections.defaultdict(list)
128+
for runner in get_runners(cfgpath):
129+
if runner.groups:
130+
for group in runner.groups:
131+
d[group].append(runner)
132+
return dict(d)

‎bench_runner/scripts/get_merge_base.py

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from bench_runner import benchmark_definitions
1010
from bench_runner import flags as mflags
1111
from bench_runner import git
12+
from bench_runner import runners as mrunners
1213
from bench_runner.result import has_result
1314
from bench_runner.util import PathLike
1415

@@ -40,29 +41,41 @@ def _main(
4041
if not need_to_run:
4142
print("ref=xxxxxxx")
4243
print("need_to_run=false")
43-
else:
44-
merge_base = git.get_git_merge_base(cpython)
44+
return
45+
merge_base = git.get_git_merge_base(cpython)
4546

46-
if merge_base is None:
47-
print("ref=xxxxxxx")
48-
print("need_to_run=false")
47+
if merge_base is None:
48+
print("ref=xxxxxxx")
49+
print("need_to_run=false")
50+
return
51+
52+
if machine in ("__really_all", "all"):
53+
need_to_run = True
54+
else:
55+
if machine.startswith("group "):
56+
group = machine.removeprefix("group ")
57+
groups = mrunners.get_groups()
58+
machines = [r.nickname for r in groups[group]]
4959
else:
50-
need_to_run = (
51-
machine in ("__really_all", "all")
52-
or has_result(
60+
machines = [machine]
61+
for m in machines:
62+
if (
63+
has_result(
5364
Path("results"),
5465
merge_base,
55-
machine,
66+
m,
5667
pystats,
5768
flags,
5869
benchmark_definitions.get_benchmark_hash(),
5970
progress=False,
6071
)
6172
is None
62-
)
73+
):
74+
need_to_run = True
75+
break
6376

64-
print(f"ref={merge_base}")
65-
print(f"need_to_run={str(need_to_run).lower()}")
77+
print(f"ref={merge_base}")
78+
print(f"need_to_run={str(need_to_run).lower()}")
6679

6780

6881
def main():

‎bench_runner/scripts/install.py

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,7 @@ def generate__benchmark(src: Any) -> Any:
165165
github_env = "$GITHUB_ENV"
166166
vars = copy.copy(runner.env)
167167
vars["BENCHMARK_MACHINE_NICKNAME"] = runner.nickname
168+
vars["BENCHMARK_RUNNER_NAME"] = runner.name
168169
setup_environment = {
169170
"name": "Setup environment",
170171
"run": LiteralScalarString(
@@ -183,6 +184,11 @@ def generate__benchmark(src: Any) -> Any:
183184
]
184185
if runner.include_in_all:
185186
machine_clauses.append("inputs.machine == 'all'")
187+
if runner.groups:
188+
for group in runner.groups:
189+
if "'" in group:
190+
raise ValueError(f"group cannot contain `'` (runner {runner.name})")
191+
machine_clauses.append(f"inputs.machine == 'group {group}'")
186192
runner_template["if"] = f"${{{{ ({' || '.join(machine_clauses)}) }}}}"
187193

188194
dst["jobs"][f"benchmark-{runner.name}"] = runner_template
@@ -205,11 +211,19 @@ def generate_benchmark(dst: Any) -> Any:
205211
"""
206212
Generates benchmark.yml from benchmark.src.yml.
207213
208-
Inserts the list of available machines to the drop-down presented to the
209-
user.
214+
Inserts the list of groups and available machines to the drop-down
215+
presented to the user.
210216
"""
211217
available_runners = [r for r in runners.get_runners() if r.available]
212-
runner_choices = [*[x.name for x in available_runners], "all", "__really_all"]
218+
groups = sorted(
219+
set(f"group {g}" for r in available_runners if r.groups for g in r.groups)
220+
)
221+
runner_choices = [
222+
*groups,
223+
*[x.name for x in available_runners],
224+
"all",
225+
"__really_all",
226+
]
213227

214228
dst["on"]["workflow_dispatch"]["inputs"]["machine"]["options"] = runner_choices
215229

‎bench_runner/templates/_benchmark.src.yml

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,9 @@ jobs:
7373
git gc
7474
- name: Building Python and running pyperformance
7575
run: |
76-
py workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
76+
py workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} `
77+
${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '$env:BENCHMARK_RUNNER_NAME' }} `
78+
${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
7779
# Pull again, since another job may have committed results in the meantime
7880
- name: Pull benchmarking
7981
run: |
@@ -112,7 +114,9 @@ jobs:
112114
python-version: "3.11"
113115
- name: Building Python and running pyperformance
114116
run: |
115-
python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} ${{ inputs.perf && '--perf' || '' }} --run_id ${{ github.run_id }}
117+
python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} \
118+
${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '"$BENCHMARK_RUNNER_NAME"' }} \
119+
${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} ${{ inputs.perf && '--perf' || '' }} --run_id ${{ github.run_id }}
116120
# Pull again, since another job may have committed results in the meantime
117121
- name: Pull benchmarking
118122
if: ${{ !inputs.perf }}
@@ -154,7 +158,9 @@ jobs:
154158
git gc
155159
- name: Building Python and running pyperformance
156160
run: |
157-
python3 workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
161+
python3 workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} \
162+
${{ (inputs.machine == 'all' || inputs.machine == '__really_all') && inputs.machine || '"$BENCHMARK_RUNNER_NAME"' }} \
163+
${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
158164
# Pull again, since another job may have committed results in the meantime
159165
- name: Pull benchmarking
160166
run: |

‎bench_runner/templates/benchmark.src.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ jobs:
6868
- name: Determine base
6969
id: base
7070
run: |
71-
python -m bench_runner get_merge_base ${{ inputs.benchmark_base }} ${{ inputs.machine }} ${{ inputs.pystats }} ${{ env.flags }} >> $GITHUB_OUTPUT
71+
python -m bench_runner get_merge_base ${{ inputs.benchmark_base }} '${{ inputs.machine }}' ${{ inputs.pystats }} ${{ env.flags }} >> $GITHUB_OUTPUT
7272
cat $GITHUB_OUTPUT
7373
7474
head:
@@ -80,7 +80,7 @@ jobs:
8080
benchmarks: ${{ inputs.benchmarks }}
8181
pgo: true
8282
perf: false
83-
force: true
83+
force: ${{ ! startsWith(inputs.machine, 'group ') }}
8484
secrets: inherit
8585

8686
base:

0 commit comments

Comments
 (0)