Skip to content

Commit 3cc722e

Browse files
blink1073blink1073
authored
PYTHON-4838 Generate OCSP build variants using shrub.py (mongodb#1910)
1 parent 9ba780c commit 3cc722e

File tree

2 files changed

+308
-33
lines changed

2 files changed

+308
-33
lines changed

‎.evergreen/config.yml

Lines changed: 141 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -2826,42 +2826,150 @@ buildvariants:
28262826
- "test-6.0-standalone"
28272827
- "test-5.0-standalone"
28282828

2829-
- matrix_name: "ocsp-test"
2830-
matrix_spec:
2831-
platform: rhel8
2832-
python-version: ["3.9", "3.10", "pypy3.9", "pypy3.10"]
2833-
mongodb-version: ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"]
2834-
auth: "noauth"
2835-
ssl: "ssl"
2836-
display_name: "OCSP test ${platform} ${python-version} ${mongodb-version}"
2837-
batchtime: 20160 # 14 days
2829+
# OCSP test matrix.
2830+
- name: ocsp-test-rhel8-v4.4-py3.9
28382831
tasks:
2839-
- name: ".ocsp"
2840-
2841-
- matrix_name: "ocsp-test-windows"
2842-
matrix_spec:
2843-
platform: windows
2844-
python-version-windows: ["3.9", "3.10"]
2845-
mongodb-version: ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"]
2846-
auth: "noauth"
2847-
ssl: "ssl"
2848-
display_name: "OCSP test ${platform} ${python-version-windows} ${mongodb-version}"
2849-
batchtime: 20160 # 14 days
2832+
- name: .ocsp
2833+
display_name: OCSP test RHEL8 v4.4 py3.9
2834+
run_on:
2835+
- rhel87-small
2836+
batchtime: 20160
2837+
expansions:
2838+
AUTH: noauth
2839+
SSL: ssl
2840+
TOPOLOGY: server
2841+
VERSION: "4.4"
2842+
PYTHON_BINARY: /opt/python/3.9/bin/python3
2843+
- name: ocsp-test-rhel8-v5.0-py3.10
28502844
tasks:
2851-
# Windows MongoDB servers do not staple OCSP responses and only support RSA.
2852-
- name: ".ocsp-rsa !.ocsp-staple"
2853-
2854-
- matrix_name: "ocsp-test-macos"
2855-
matrix_spec:
2856-
platform: macos
2857-
mongodb-version: ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"]
2858-
auth: "noauth"
2859-
ssl: "ssl"
2860-
display_name: "OCSP test ${platform} ${mongodb-version}"
2861-
batchtime: 20160 # 14 days
2845+
- name: .ocsp
2846+
display_name: OCSP test RHEL8 v5.0 py3.10
2847+
run_on:
2848+
- rhel87-small
2849+
batchtime: 20160
2850+
expansions:
2851+
AUTH: noauth
2852+
SSL: ssl
2853+
TOPOLOGY: server
2854+
VERSION: "5.0"
2855+
PYTHON_BINARY: /opt/python/3.10/bin/python3
2856+
- name: ocsp-test-rhel8-v6.0-py3.11
28622857
tasks:
2863-
# macOS MongoDB servers do not staple OCSP responses and only support RSA.
2864-
- name: ".ocsp-rsa !.ocsp-staple"
2858+
- name: .ocsp
2859+
display_name: OCSP test RHEL8 v6.0 py3.11
2860+
run_on:
2861+
- rhel87-small
2862+
batchtime: 20160
2863+
expansions:
2864+
AUTH: noauth
2865+
SSL: ssl
2866+
TOPOLOGY: server
2867+
VERSION: "6.0"
2868+
PYTHON_BINARY: /opt/python/3.11/bin/python3
2869+
- name: ocsp-test-rhel8-v7.0-py3.12
2870+
tasks:
2871+
- name: .ocsp
2872+
display_name: OCSP test RHEL8 v7.0 py3.12
2873+
run_on:
2874+
- rhel87-small
2875+
batchtime: 20160
2876+
expansions:
2877+
AUTH: noauth
2878+
SSL: ssl
2879+
TOPOLOGY: server
2880+
VERSION: "7.0"
2881+
PYTHON_BINARY: /opt/python/3.12/bin/python3
2882+
- name: ocsp-test-rhel8-v8.0-py3.13
2883+
tasks:
2884+
- name: .ocsp
2885+
display_name: OCSP test RHEL8 v8.0 py3.13
2886+
run_on:
2887+
- rhel87-small
2888+
batchtime: 20160
2889+
expansions:
2890+
AUTH: noauth
2891+
SSL: ssl
2892+
TOPOLOGY: server
2893+
VERSION: "8.0"
2894+
PYTHON_BINARY: /opt/python/3.13/bin/python3
2895+
- name: ocsp-test-rhel8-rapid-pypy3.9
2896+
tasks:
2897+
- name: .ocsp
2898+
display_name: OCSP test RHEL8 rapid pypy3.9
2899+
run_on:
2900+
- rhel87-small
2901+
batchtime: 20160
2902+
expansions:
2903+
AUTH: noauth
2904+
SSL: ssl
2905+
TOPOLOGY: server
2906+
VERSION: rapid
2907+
PYTHON_BINARY: /opt/python/pypy3.9/bin/python3
2908+
- name: ocsp-test-rhel8-latest-pypy3.10
2909+
tasks:
2910+
- name: .ocsp
2911+
display_name: OCSP test RHEL8 latest pypy3.10
2912+
run_on:
2913+
- rhel87-small
2914+
batchtime: 20160
2915+
expansions:
2916+
AUTH: noauth
2917+
SSL: ssl
2918+
TOPOLOGY: server
2919+
VERSION: latest
2920+
PYTHON_BINARY: /opt/python/pypy3.10/bin/python3
2921+
- name: ocsp-test-win64-v4.4-py3.9
2922+
tasks:
2923+
- name: .ocsp-rsa !.ocsp-staple
2924+
display_name: OCSP test Win64 v4.4 py3.9
2925+
run_on:
2926+
- windows-64-vsMulti-small
2927+
batchtime: 20160
2928+
expansions:
2929+
AUTH: noauth
2930+
SSL: ssl
2931+
TOPOLOGY: server
2932+
VERSION: "4.4"
2933+
PYTHON_BINARY: C:/python/Python39/python.exe
2934+
- name: ocsp-test-win64-v8.0-py3.13
2935+
tasks:
2936+
- name: .ocsp-rsa !.ocsp-staple
2937+
display_name: OCSP test Win64 v8.0 py3.13
2938+
run_on:
2939+
- windows-64-vsMulti-small
2940+
batchtime: 20160
2941+
expansions:
2942+
AUTH: noauth
2943+
SSL: ssl
2944+
TOPOLOGY: server
2945+
VERSION: "8.0"
2946+
PYTHON_BINARY: C:/python/Python313/python.exe
2947+
- name: ocsp-test-macos-v4.4-py3.9
2948+
tasks:
2949+
- name: .ocsp-rsa !.ocsp-staple
2950+
display_name: OCSP test macOS v4.4 py3.9
2951+
run_on:
2952+
- macos-14
2953+
batchtime: 20160
2954+
expansions:
2955+
AUTH: noauth
2956+
SSL: ssl
2957+
TOPOLOGY: server
2958+
VERSION: "4.4"
2959+
PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.9/bin/python3
2960+
- name: ocsp-test-macos-v8.0-py3.13
2961+
tasks:
2962+
- name: .ocsp-rsa !.ocsp-staple
2963+
display_name: OCSP test macOS v8.0 py3.13
2964+
run_on:
2965+
- macos-14
2966+
batchtime: 20160
2967+
expansions:
2968+
AUTH: noauth
2969+
SSL: ssl
2970+
TOPOLOGY: server
2971+
VERSION: "8.0"
2972+
PYTHON_BINARY: /Library/Frameworks/Python.Framework/Versions/3.13/bin/python3
28652973

28662974
- matrix_name: "oidc-auth-test"
28672975
matrix_spec:

‎.evergreen/scripts/generate_config.py

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
# /// script
2+
# requires-python = ">=3.9"
3+
# dependencies = [
4+
# "shrub.py>=3.2.0",
5+
# "pyyaml>=6.0.2"
6+
# ]
7+
# ///
8+
9+
# Note: Run this file with `hatch run`, `pipx run`, or `uv run`.
10+
from __future__ import annotations
11+
12+
from dataclasses import dataclass
13+
from itertools import cycle, product, zip_longest
14+
from typing import Any
15+
16+
from shrub.v3.evg_build_variant import BuildVariant
17+
from shrub.v3.evg_project import EvgProject
18+
from shrub.v3.evg_task import EvgTaskRef
19+
from shrub.v3.shrub_service import ShrubService
20+
21+
##############
22+
# Globals
23+
##############
24+
25+
ALL_VERSIONS = ["4.0", "4.4", "5.0", "6.0", "7.0", "8.0", "rapid", "latest"]
26+
CPYTHONS = ["3.9", "3.10", "3.11", "3.12", "3.13"]
27+
PYPYS = ["pypy3.9", "pypy3.10"]
28+
ALL_PYTHONS = CPYTHONS + PYPYS
29+
BATCHTIME_WEEK = 10080
30+
HOSTS = dict()
31+
32+
33+
@dataclass
34+
class Host:
35+
name: str
36+
run_on: str
37+
display_name: str
38+
39+
40+
HOSTS["rhel8"] = Host("rhel8", "rhel87-small", "RHEL8")
41+
HOSTS["win64"] = Host("win64", "windows-64-vsMulti-small", "Win64")
42+
HOSTS["macos"] = Host("macos", "macos-14", "macOS")
43+
44+
45+
##############
46+
# Helpers
47+
##############
48+
49+
50+
def create_variant(
51+
task_names: list[str],
52+
display_name: str,
53+
*,
54+
python: str | None = None,
55+
version: str | None = None,
56+
host: str | None = None,
57+
**kwargs: Any,
58+
) -> BuildVariant:
59+
"""Create a build variant for the given inputs."""
60+
task_refs = [EvgTaskRef(name=n) for n in task_names]
61+
kwargs.setdefault("expansions", dict())
62+
expansions = kwargs.pop("expansions", dict()).copy()
63+
host = host or "rhel8"
64+
run_on = [HOSTS[host].run_on]
65+
name = display_name.replace(" ", "-").lower()
66+
if python:
67+
expansions["PYTHON_BINARY"] = get_python_binary(python, host)
68+
if version:
69+
expansions["VERSION"] = version
70+
expansions = expansions or None
71+
return BuildVariant(
72+
name=name,
73+
display_name=display_name,
74+
tasks=task_refs,
75+
expansions=expansions,
76+
run_on=run_on,
77+
**kwargs,
78+
)
79+
80+
81+
def get_python_binary(python: str, host: str) -> str:
82+
"""Get the appropriate python binary given a python version and host."""
83+
if host == "win64":
84+
is_32 = python.startswith("32-bit")
85+
if is_32:
86+
_, python = python.split()
87+
base = "C:/python/32"
88+
else:
89+
base = "C:/python"
90+
python = python.replace(".", "")
91+
return f"{base}/Python{python}/python.exe"
92+
93+
if host == "rhel8":
94+
return f"/opt/python/{python}/bin/python3"
95+
96+
if host == "macos":
97+
return f"/Library/Frameworks/Python.Framework/Versions/{python}/bin/python3"
98+
99+
raise ValueError(f"no match found for python {python} on {host}")
100+
101+
102+
def get_display_name(base: str, host: str, version: str, python: str) -> str:
103+
"""Get the display name of a variant."""
104+
if version not in ["rapid", "latest"]:
105+
version = f"v{version}"
106+
if not python.startswith("pypy"):
107+
python = f"py{python}"
108+
return f"{base} {HOSTS[host].display_name} {version} {python}"
109+
110+
111+
def zip_cycle(*iterables, empty_default=None):
112+
"""Get all combinations of the inputs, cycling over the shorter list(s)."""
113+
cycles = [cycle(i) for i in iterables]
114+
for _ in zip_longest(*iterables):
115+
yield tuple(next(i, empty_default) for i in cycles)
116+
117+
118+
##############
119+
# Variants
120+
##############
121+
122+
123+
def create_ocsp_variants() -> list[BuildVariant]:
124+
variants = []
125+
batchtime = BATCHTIME_WEEK * 2
126+
expansions = dict(AUTH="noauth", SSL="ssl", TOPOLOGY="server")
127+
base_display = "OCSP test"
128+
129+
# OCSP tests on rhel8 with all servers v4.4+ and all python versions.
130+
versions = [v for v in ALL_VERSIONS if v != "4.0"]
131+
for version, python in zip_cycle(versions, ALL_PYTHONS):
132+
host = "rhel8"
133+
variant = create_variant(
134+
[".ocsp"],
135+
get_display_name(base_display, host, version, python),
136+
python=python,
137+
version=version,
138+
host=host,
139+
expansions=expansions,
140+
batchtime=batchtime,
141+
)
142+
variants.append(variant)
143+
144+
# OCSP tests on Windows and MacOS.
145+
# MongoDB servers on these hosts do not staple OCSP responses and only support RSA.
146+
for host, version in product(["win64", "macos"], ["4.4", "8.0"]):
147+
python = CPYTHONS[0] if version == "4.4" else CPYTHONS[-1]
148+
variant = create_variant(
149+
[".ocsp-rsa !.ocsp-staple"],
150+
get_display_name(base_display, host, version, python),
151+
python=python,
152+
version=version,
153+
host=host,
154+
expansions=expansions,
155+
batchtime=batchtime,
156+
)
157+
variants.append(variant)
158+
159+
return variants
160+
161+
162+
##################
163+
# Generate Config
164+
##################
165+
166+
project = EvgProject(tasks=None, buildvariants=create_ocsp_variants())
167+
print(ShrubService.generate_yaml(project)) # noqa: T201

0 commit comments

Comments
 (0)