Make weekly runs configurable

Author: mdboom

CHANGELOG.md

@@ -63,3 +63,26 @@ PYTHON_CONFIGURE_FLAGS = '`"/p:PlatformToolset=clangcl`" `"/p:LLVMInstallDir=C:\
 Runners have a new configuration `use_cores` to control the number of CPU cores
 used to build CPython. By default, this will use all available cores, but some
 Cloud VMs require using fewer.
+
+### Configurable weekly runs
+
+The set of weekly runs are now configured from the `bench_runner.toml`.
+If you don't add any configuration, the only weekly runs performed will be default builds on every runner.
+
+The `weekly` section in the configuration is made up of sections, each of which
+has a `flags` parameter specifying the flags to use and a `runner` parameter
+specifying the runners to run on.
+
+For example:
+
+```toml
+[[weekly]]
+
+[weekly.default]
+flags = []
+runners = ["linux", "darwin", "windows"]
+
+[weekly.tailcall]
+flags = ["TAILCALL"]
+runners = ["linux_clang", "darwin", "windows_clang"]
+```

bench_runner/flags.py

@@ -37,12 +37,16 @@ def parse_flags(flag_str: str | None) -> list[str]:
 
 
 def flags_to_gha_variables(flags: list[str]) -> dict[str, str]:
+    return {
+        k: "true" if v else "false"
+        for k, v in flags_to_gha_variables_yml(flags).items()
+    }
+
+
+def flags_to_gha_variables_yml(flags: list[str]) -> dict[str, bool]:
     output = {}
     for flag_descr in FLAGS:
-        if flag_descr.name in flags:
-            output[flag_descr.gha_variable] = "true"
-        else:
-            output[flag_descr.gha_variable] = "false"
+        output[flag_descr.gha_variable] = flag_descr.name in flags
     return output
 
 

bench_runner/scripts/install.py

@@ -226,6 +226,45 @@ def generate__notify(dst: Any) -> Any:
     return dst
 
 
+def generate__weekly(dst: Any) -> Any:
+    cfg = config.get_bench_runner_config()
+
+    weekly = cfg.get("weekly", {})
+    if not weekly:
+        weekly = [{"default": {"flags": [], "runners": cfg.get("runners", {}).keys()}}]
+    weekly = weekly[0]
+
+    all_jobs = []
+
+    for name, weekly_cfg in weekly.items():
+        for runner_nickname in weekly_cfg.get("runners", []):
+            runner = runners.get_runner_by_nickname(runner_nickname)
+            if runner.nickname == "unknown":
+                raise ValueError(
+                    f"Runner {runner_nickname} not found in bench_runner.toml"
+                )
+            weekly_flags = weekly_cfg.get("flags", [])
+            job = {
+                "uses": "./.github/workflows/_benchmark.yml",
+                "needs": "determine_head",
+                "with": {
+                    "fork": "python",
+                    "ref": "${{ needs.determine_head.outputs.commit }}",
+                    "machine": runner.name,
+                    "benchmarks": "all_and_excluded",
+                    **flags.flags_to_gha_variables_yml(weekly_flags),
+                },
+                "secrets": "inherit",
+            }
+            job_name = f"weekly-{name}-{runner.nickname}"
+            dst["jobs"][job_name] = job
+            all_jobs.append(job_name)
+
+    dst["jobs"]["generate"]["needs"].extend(all_jobs)
+
+    return dst
+
+
 def generate_generic(dst: Any) -> Any:
     return dst
 
@@ -235,6 +274,7 @@ def generate_generic(dst: Any) -> Any:
     "_benchmark.src.yml": generate__benchmark,
     "_pystats.src.yml": generate__pystats,
     "_notify.src.yml": generate__notify,
+    "_weekly.src.yml": generate__weekly,
 }
 
 

bench_runner/templates/_weekly.src.yml

@@ -28,60 +28,6 @@ jobs:
           cd cpython
           git log -n 1 --format="commit=%H" >> $GITHUB_OUTPUT
 
-  weekly:
-    uses: ./.github/workflows/_benchmark.yml
-    needs: determine_head
-    with:
-      fork: python
-      ref: ${{ needs.determine_head.outputs.commit }}
-      machine: __really_all
-      benchmarks: all_and_excluded
-      pgo: true
-      tier2: false
-    secrets: inherit
-
-  weekly-jit:
-    uses: ./.github/workflows/_benchmark.yml
-    needs: determine_head
-    with:
-      fork: python
-      ref: ${{ needs.determine_head.outputs.commit }}
-      machine: __really_all
-      benchmarks: all_and_excluded
-      pgo: true
-      tier2: false
-      jit: true
-    secrets: inherit
-
-  weekly-nogil:
-    uses: ./.github/workflows/_benchmark.yml
-    needs: determine_head
-    with:
-      fork: python
-      ref: ${{ needs.determine_head.outputs.commit }}
-      machine: __really_all
-      benchmarks: all_and_excluded
-      pgo: true
-      tier2: false
-      jit: false
-      nogil: true
-    secrets: inherit
-
-  weekly-tailcall:
-    uses: ./.github/workflows/_benchmark.yml
-    needs: determine_head
-    with:
-      fork: python
-      ref: ${{ needs.determine_head.outputs.commit }}
-      machine: __really_all
-      benchmarks: all_and_excluded
-      pgo: true
-      tier2: false
-      jit: false
-      nogil: false
-      tailcall: true
-    secrets: inherit
-
   pystats:
     uses: ./.github/workflows/_pystats.yml
     needs: determine_head
@@ -109,7 +55,7 @@ jobs:
     if: ${{ always() }}
     with:
       force: false
-    needs: [weekly, weekly-jit, weekly-nogil, pystats, pystats-tier2]
+    needs: [pystats, pystats-tier2]
     secrets: inherit
 
   publish: