|
| 1 | +# /// script |
| 2 | +# requires-python = ">=3.9" |
| 3 | +# dependencies = [ |
| 4 | +# "shrub.py>=3.2.0", |
| 5 | +# "pyyaml>=6.0.2" |
| 6 | +# ] |
| 7 | +# /// |
| 8 | + |
| 9 | +# Note: Run this file with `hatch run`, `pipx run`, or `uv run`. |
| 10 | +from __future__ import annotations |
| 11 | + |
| 12 | +from dataclasses import dataclass |
| 13 | +from itertools import cycle, product, zip_longest |
| 14 | +from typing import Any |
| 15 | + |
| 16 | +from shrub.v3.evg_build_variant import BuildVariant |
| 17 | +from shrub.v3.evg_project import EvgProject |
| 18 | +from shrub.v3.evg_task import EvgTaskRef |
| 19 | +from shrub.v3.shrub_service import ShrubService |
| 20 | + |
| 21 | +############## |
| 22 | +# Globals |
| 23 | +############## |
| 24 | + |
| 25 | +ALL_VERSIONS = ["4.0", "4.4", "5.0", "6.0", "7.0", "8.0", "rapid", "latest"] |
| 26 | +CPYTHONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] |
| 27 | +PYPYS = ["pypy3.9", "pypy3.10"] |
| 28 | +ALL_PYTHONS = CPYTHONS + PYPYS |
| 29 | +MIN_MAX_PYTHON = [CPYTHONS[0], CPYTHONS[-1]] |
| 30 | +BATCHTIME_WEEK = 10080 |
| 31 | +AUTH_SSLS = [("auth", "ssl"), ("noauth", "ssl"), ("noauth", "nossl")] |
| 32 | +TOPOLOGIES = ["standalone", "replica_set", "sharded_cluster"] |
| 33 | +SYNCS = ["sync", "async"] |
| 34 | +DISPLAY_LOOKUP = dict( |
| 35 | + ssl=dict(ssl="SSL", nossl="NoSSL"), |
| 36 | + auth=dict(auth="Auth", noauth="NoAuth"), |
| 37 | + test_suites=dict(default="Sync", default_async="Async"), |
| 38 | + coverage=dict(coverage="cov"), |
| 39 | +) |
| 40 | +HOSTS = dict() |
| 41 | + |
| 42 | + |
| 43 | +@dataclass |
| 44 | +class Host: |
| 45 | + name: str |
| 46 | + run_on: str |
| 47 | + display_name: str |
| 48 | + expansions: dict[str, str] |
| 49 | + |
| 50 | + |
| 51 | +_macos_expansions = dict( # CSOT tests are unreliable on slow hosts. |
| 52 | + SKIP_CSOT_TESTS="true" |
| 53 | +) |
| 54 | + |
| 55 | +HOSTS["rhel8"] = Host("rhel8", "rhel87-small", "RHEL8", dict()) |
| 56 | +HOSTS["win64"] = Host("win64", "windows-64-vsMulti-small", "Win64", _macos_expansions) |
| 57 | +HOSTS["win32"] = Host("win32", "windows-64-vsMulti-small", "Win32", _macos_expansions) |
| 58 | +HOSTS["macos"] = Host("macos", "macos-14", "macOS", _macos_expansions) |
| 59 | +HOSTS["macos-arm64"] = Host("macos-arm64", "macos-14-arm64", "macOS Arm64", _macos_expansions) |
| 60 | + |
| 61 | + |
| 62 | +############## |
| 63 | +# Helpers |
| 64 | +############## |
| 65 | + |
| 66 | + |
| 67 | +def create_variant( |
| 68 | + task_names: list[str], |
| 69 | + display_name: str, |
| 70 | + *, |
| 71 | + python: str | None = None, |
| 72 | + version: str | None = None, |
| 73 | + host: str | None = None, |
| 74 | + **kwargs: Any, |
| 75 | +) -> BuildVariant: |
| 76 | + """Create a build variant for the given inputs.""" |
| 77 | + task_refs = [EvgTaskRef(name=n) for n in task_names] |
| 78 | + kwargs.setdefault("expansions", dict()) |
| 79 | + expansions = kwargs.pop("expansions", dict()).copy() |
| 80 | + host = host or "rhel8" |
| 81 | + run_on = [HOSTS[host].run_on] |
| 82 | + name = display_name.replace(" ", "-").lower() |
| 83 | + if python: |
| 84 | + expansions["PYTHON_BINARY"] = get_python_binary(python, host) |
| 85 | + if version: |
| 86 | + expansions["VERSION"] = version |
| 87 | + expansions.update(HOSTS[host].expansions) |
| 88 | + expansions = expansions or None |
| 89 | + return BuildVariant( |
| 90 | + name=name, |
| 91 | + display_name=display_name, |
| 92 | + tasks=task_refs, |
| 93 | + expansions=expansions, |
| 94 | + run_on=run_on, |
| 95 | + **kwargs, |
| 96 | + ) |
| 97 | + |
| 98 | + |
| 99 | +def get_python_binary(python: str, host: str) -> str: |
| 100 | + """Get the appropriate python binary given a python version and host.""" |
| 101 | + if host in ["win64", "win32"]: |
| 102 | + if host == "win32": |
| 103 | + base = "C:/python/32" |
| 104 | + else: |
| 105 | + base = "C:/python" |
| 106 | + python = python.replace(".", "") |
| 107 | + return f"{base}/Python{python}/python.exe" |
| 108 | + |
| 109 | + if host == "rhel8": |
| 110 | + return f"/opt/python/{python}/bin/python3" |
| 111 | + |
| 112 | + if host in ["macos", "macos-arm64"]: |
| 113 | + return f"/Library/Frameworks/Python.Framework/Versions/{python}/bin/python3" |
| 114 | + |
| 115 | + raise ValueError(f"no match found for python {python} on {host}") |
| 116 | + |
| 117 | + |
| 118 | +def get_display_name(base: str, host: str, **kwargs) -> str: |
| 119 | + """Get the display name of a variant.""" |
| 120 | + display_name = f"{base} {HOSTS[host].display_name}" |
| 121 | + for key, value in kwargs.items(): |
| 122 | + name = value |
| 123 | + if key == "version": |
| 124 | + if value not in ["rapid", "latest"]: |
| 125 | + name = f"v{value}" |
| 126 | + elif key == "python": |
| 127 | + if not value.startswith("pypy"): |
| 128 | + name = f"py{value}" |
| 129 | + elif key.lower() in DISPLAY_LOOKUP: |
| 130 | + name = DISPLAY_LOOKUP[key.lower()][value] |
| 131 | + else: |
| 132 | + raise ValueError(f"Missing display handling for {key}") |
| 133 | + display_name = f"{display_name} {name}" |
| 134 | + return display_name |
| 135 | + |
| 136 | + |
| 137 | +def zip_cycle(*iterables, empty_default=None): |
| 138 | + """Get all combinations of the inputs, cycling over the shorter list(s).""" |
| 139 | + cycles = [cycle(i) for i in iterables] |
| 140 | + for _ in zip_longest(*iterables): |
| 141 | + yield tuple(next(i, empty_default) for i in cycles) |
| 142 | + |
| 143 | + |
| 144 | +def generate_yaml(tasks=None, variants=None): |
| 145 | + """Generate the yaml for a given set of tasks and variants.""" |
| 146 | + project = EvgProject(tasks=tasks, buildvariants=variants) |
| 147 | + out = ShrubService.generate_yaml(project) |
| 148 | + # Dedent by two spaces to match what we use in config.yml |
| 149 | + lines = [line[2:] for line in out.splitlines()] |
| 150 | + print("\n".join(lines)) # noqa: T201 |
| 151 | + |
| 152 | + |
| 153 | +############## |
| 154 | +# Variants |
| 155 | +############## |
| 156 | + |
| 157 | + |
| 158 | +def create_ocsp_variants() -> list[BuildVariant]: |
| 159 | + variants = [] |
| 160 | + batchtime = BATCHTIME_WEEK * 2 |
| 161 | + expansions = dict(AUTH="noauth", SSL="ssl", TOPOLOGY="server") |
| 162 | + base_display = "OCSP test" |
| 163 | + |
| 164 | + # OCSP tests on rhel8 with all servers v4.4+ and all python versions. |
| 165 | + versions = [v for v in ALL_VERSIONS if v != "4.0"] |
| 166 | + for version, python in zip_cycle(versions, ALL_PYTHONS): |
| 167 | + host = "rhel8" |
| 168 | + variant = create_variant( |
| 169 | + [".ocsp"], |
| 170 | + get_display_name(base_display, host, version, python), |
| 171 | + python=python, |
| 172 | + version=version, |
| 173 | + host=host, |
| 174 | + expansions=expansions, |
| 175 | + batchtime=batchtime, |
| 176 | + ) |
| 177 | + variants.append(variant) |
| 178 | + |
| 179 | + # OCSP tests on Windows and MacOS. |
| 180 | + # MongoDB servers on these hosts do not staple OCSP responses and only support RSA. |
| 181 | + for host, version in product(["win64", "macos"], ["4.4", "8.0"]): |
| 182 | + python = CPYTHONS[0] if version == "4.4" else CPYTHONS[-1] |
| 183 | + variant = create_variant( |
| 184 | + [".ocsp-rsa !.ocsp-staple"], |
| 185 | + get_display_name(base_display, host, version, python), |
| 186 | + python=python, |
| 187 | + version=version, |
| 188 | + host=host, |
| 189 | + expansions=expansions, |
| 190 | + batchtime=batchtime, |
| 191 | + ) |
| 192 | + variants.append(variant) |
| 193 | + |
| 194 | + return variants |
| 195 | + |
| 196 | + |
| 197 | +def create_server_variants() -> list[BuildVariant]: |
| 198 | + variants = [] |
| 199 | + |
| 200 | + # Run the full matrix on linux with min and max CPython, and latest pypy. |
| 201 | + host = "rhel8" |
| 202 | + for python, (auth, ssl) in product([*MIN_MAX_PYTHON, PYPYS[-1]], AUTH_SSLS): |
| 203 | + display_name = f"Test {host}" |
| 204 | + expansions = dict(AUTH=auth, SSL=ssl, COVERAGE="coverage") |
| 205 | + display_name = get_display_name("Test", host, python=python, **expansions) |
| 206 | + variant = create_variant( |
| 207 | + [f".{t}" for t in TOPOLOGIES], |
| 208 | + display_name, |
| 209 | + python=python, |
| 210 | + host=host, |
| 211 | + tags=["coverage_tag"], |
| 212 | + expansions=expansions, |
| 213 | + ) |
| 214 | + variants.append(variant) |
| 215 | + |
| 216 | + # Test the rest of the pythons on linux. |
| 217 | + for python, (auth, ssl), topology in zip_cycle( |
| 218 | + CPYTHONS[1:-1] + PYPYS[:-1], AUTH_SSLS, TOPOLOGIES |
| 219 | + ): |
| 220 | + display_name = f"Test {host}" |
| 221 | + expansions = dict(AUTH=auth, SSL=ssl) |
| 222 | + display_name = get_display_name("Test", host, python=python, **expansions) |
| 223 | + variant = create_variant( |
| 224 | + [f".{topology}"], |
| 225 | + display_name, |
| 226 | + python=python, |
| 227 | + host=host, |
| 228 | + expansions=expansions, |
| 229 | + ) |
| 230 | + variants.append(variant) |
| 231 | + |
| 232 | + # Test a subset on each of the other platforms. |
| 233 | + for host in ("macos", "macos-arm64", "win64", "win32"): |
| 234 | + for (python, (auth, ssl), topology), sync in product( |
| 235 | + zip_cycle(MIN_MAX_PYTHON, AUTH_SSLS, TOPOLOGIES), SYNCS |
| 236 | + ): |
| 237 | + test_suite = "default" if sync == "sync" else "default_async" |
| 238 | + expansions = dict(AUTH=auth, SSL=ssl, TEST_SUITES=test_suite) |
| 239 | + display_name = get_display_name("Test", host, python=python, **expansions) |
| 240 | + variant = create_variant( |
| 241 | + [f".{topology}"], |
| 242 | + display_name, |
| 243 | + python=python, |
| 244 | + host=host, |
| 245 | + expansions=expansions, |
| 246 | + ) |
| 247 | + variants.append(variant) |
| 248 | + |
| 249 | + return variants |
| 250 | + |
| 251 | + |
| 252 | +################## |
| 253 | +# Generate Config |
| 254 | +################## |
| 255 | + |
| 256 | +generate_yaml(variants=create_server_variants()) |
0 commit comments