Skip to content

Commit f91adf9

Browse files
zaniebzanieb
authored
Generate CI matrix at runtime (#428)
Generates the CI matrix dynamically so we can run subsets on pull requests. - Moves the matrix definition out of GitHub workflows into a `ci-targets.yaml` file - Adds a `ci-matrix.py` script which parses the `ci-targets.yaml` file and outputs a JSON matrix - Updates the GitHub Actions workflows to use the script to generate a matrix dynamically - Uses the labels on the pull request to allow subsetting of the matrix For example, you can run the matrix generator locally to see the label subsetting in action: ```console ❯ uv run ci-matrix.py --label arch:x86_64,platform:linux,libc:gnu,build:freethreaded,build:lto | jq Reading inline script metadata from `ci-matrix.py` { "include": [ { "arch": "x86_64", "target_triple": "x86_64-unknown-linux-gnu", "platform": "linux", "libc": "gnu", "run": "true", "python": "3.13", "build_options": "freethreaded+pgo+lto" }, { "arch": "x86_64", "target_triple": "x86_64_v2-unknown-linux-gnu", "platform": "linux", "arch_variant": "v2", "libc": "gnu", "run": "true", "python": "3.13", "build_options": "freethreaded+pgo+lto" }, { "arch": "x86_64", "target_triple": "x86_64_v3-unknown-linux-gnu", "platform": "linux", "arch_variant": "v3", "libc": "gnu", "run": "true", "python": "3.13", "build_options": "freethreaded+pgo+lto" }, { "arch": "x86_64", "target_triple": "x86_64_v4-unknown-linux-gnu", "platform": "linux", "arch_variant": "v4", "libc": "gnu", "python": "3.13", "build_options": "freethreaded+lto" } ] } ``` I'll add labels for - `platform:darwin` - `platform:linux` - `platform:windows` - `python:3.9` - `python:3.10` - `python:3.11` - `python:3.12` - `python:3.13` - `build:debug` - `build:pgo` - `build:lto` - `build:noopt` - `build:freethreaded` - `arch:x86_64` - `arch:aarch64` - `arch:armv7` - `arch:s390x` - `arch:ppc64le` - `arch:x86` - `libc:gnu` - `libc:musl` In a follow-up, I'll update this to use different runners in forks as noted in #426
1 parent 4446f7d commit f91adf9

File tree

5 files changed

+608
-1125
lines changed

5 files changed

+608
-1125
lines changed

‎.github/workflows/apple.yml‎

Lines changed: 35 additions & 131 deletions
Original file line numberDiff line numberDiff line change
@@ -40,135 +40,39 @@ jobs:
4040
name: pythonbuild
4141
path: target/release/pythonbuild
4242

43+
generate-matrix:
44+
runs-on: ubuntu-latest
45+
outputs:
46+
matrix: ${{ steps.set-matrix.outputs.matrix }}
47+
steps:
48+
- uses: actions/checkout@v4
49+
50+
- name: Set up Python
51+
uses: astral-sh/setup-uv@v4
52+
53+
- name: Get pull request labels
54+
id: get-labels
55+
run: |
56+
# Convert GitHub labels array to comma-separated string
57+
LABELS=$(echo '${{ toJson(github.event.pull_request.labels.*.name) }}' | jq -r 'join(",")')
58+
echo "labels=$LABELS" >> $GITHUB_OUTPUT
59+
60+
- name: Generate build matrix
61+
id: set-matrix
62+
run: |
63+
uv run ci-matrix.py --platform darwin --labels "${{ steps.get-labels.outputs.labels }}" > matrix.json && echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
64+
# Display the matrix for debugging too
65+
cat matrix.json | jq
66+
4367
build:
44-
strategy:
45-
fail-fast: false
46-
matrix:
47-
build:
48-
- target_triple: 'aarch64-apple-darwin'
49-
py: 'cpython-3.9'
50-
options: 'debug'
51-
- target_triple: 'aarch64-apple-darwin'
52-
py: 'cpython-3.9'
53-
options: 'pgo'
54-
- target_triple: 'aarch64-apple-darwin'
55-
py: 'cpython-3.9'
56-
options: 'pgo+lto'
57-
58-
- target_triple: 'aarch64-apple-darwin'
59-
py: 'cpython-3.10'
60-
options: 'debug'
61-
- target_triple: 'aarch64-apple-darwin'
62-
py: 'cpython-3.10'
63-
options: 'pgo'
64-
- target_triple: 'aarch64-apple-darwin'
65-
py: 'cpython-3.10'
66-
options: 'pgo+lto'
67-
68-
- target_triple: 'aarch64-apple-darwin'
69-
py: 'cpython-3.11'
70-
options: 'debug'
71-
- target_triple: 'aarch64-apple-darwin'
72-
py: 'cpython-3.11'
73-
options: 'pgo'
74-
- target_triple: 'aarch64-apple-darwin'
75-
py: 'cpython-3.11'
76-
options: 'pgo+lto'
77-
78-
- target_triple: 'aarch64-apple-darwin'
79-
py: 'cpython-3.12'
80-
options: 'debug'
81-
- target_triple: 'aarch64-apple-darwin'
82-
py: 'cpython-3.12'
83-
options: 'pgo'
84-
- target_triple: 'aarch64-apple-darwin'
85-
py: 'cpython-3.12'
86-
options: 'pgo+lto'
87-
88-
- target_triple: 'aarch64-apple-darwin'
89-
py: 'cpython-3.13'
90-
options: 'debug'
91-
- target_triple: 'aarch64-apple-darwin'
92-
py: 'cpython-3.13'
93-
options: 'pgo'
94-
- target_triple: 'aarch64-apple-darwin'
95-
py: 'cpython-3.13'
96-
options: 'pgo+lto'
97-
98-
- target_triple: 'aarch64-apple-darwin'
99-
py: 'cpython-3.13'
100-
options: 'freethreaded+debug'
101-
- target_triple: 'aarch64-apple-darwin'
102-
py: 'cpython-3.13'
103-
options: 'freethreaded+pgo'
104-
- target_triple: 'aarch64-apple-darwin'
105-
py: 'cpython-3.13'
106-
options: 'freethreaded+pgo+lto'
107-
108-
# macOS on Intel hardware. This is pretty straightforward. We exclude
109-
# noopt because it doesn't provide any compelling advantages over PGO
110-
# or LTO builds.
111-
- target_triple: 'x86_64-apple-darwin'
112-
py: 'cpython-3.9'
113-
options: 'debug'
114-
- target_triple: 'x86_64-apple-darwin'
115-
py: 'cpython-3.9'
116-
options: 'pgo'
117-
- target_triple: 'x86_64-apple-darwin'
118-
py: 'cpython-3.9'
119-
options: 'pgo+lto'
120-
121-
- target_triple: 'x86_64-apple-darwin'
122-
py: 'cpython-3.10'
123-
options: 'debug'
124-
- target_triple: 'x86_64-apple-darwin'
125-
py: 'cpython-3.10'
126-
options: 'pgo'
127-
- target_triple: 'x86_64-apple-darwin'
128-
py: 'cpython-3.10'
129-
options: 'pgo+lto'
130-
131-
- target_triple: 'x86_64-apple-darwin'
132-
py: 'cpython-3.11'
133-
options: 'debug'
134-
- target_triple: 'x86_64-apple-darwin'
135-
py: 'cpython-3.11'
136-
options: 'pgo'
137-
- target_triple: 'x86_64-apple-darwin'
138-
py: 'cpython-3.11'
139-
options: 'pgo+lto'
140-
141-
- target_triple: 'x86_64-apple-darwin'
142-
py: 'cpython-3.12'
143-
options: 'debug'
144-
- target_triple: 'x86_64-apple-darwin'
145-
py: 'cpython-3.12'
146-
options: 'pgo'
147-
- target_triple: 'x86_64-apple-darwin'
148-
py: 'cpython-3.12'
149-
options: 'pgo+lto'
150-
151-
- target_triple: 'x86_64-apple-darwin'
152-
py: 'cpython-3.13'
153-
options: 'debug'
154-
- target_triple: 'x86_64-apple-darwin'
155-
py: 'cpython-3.13'
156-
options: 'pgo'
157-
- target_triple: 'x86_64-apple-darwin'
158-
py: 'cpython-3.13'
159-
options: 'pgo+lto'
160-
- target_triple: 'x86_64-apple-darwin'
161-
py: 'cpython-3.13'
162-
options: 'freethreaded+debug'
163-
- target_triple: 'x86_64-apple-darwin'
164-
py: 'cpython-3.13'
165-
options: 'freethreaded+pgo'
166-
- target_triple: 'x86_64-apple-darwin'
167-
py: 'cpython-3.13'
168-
options: 'freethreaded+pgo+lto'
16968
needs:
69+
- generate-matrix
17070
- pythonbuild
171-
runs-on: depot-macos-14
71+
strategy:
72+
matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
73+
fail-fast: false
74+
runs-on: macos-14
75+
name: ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }}
17276
steps:
17377
- uses: actions/checkout@v4
17478
with:
@@ -187,21 +91,21 @@ jobs:
18791

18892
- name: Build
18993
run: |
190-
if [ "${{ matrix.build.target_triple }}" = "aarch64-apple-darwin" ]; then
94+
if [ "${{ matrix.target_triple }}" = "aarch64-apple-darwin" ]; then
19195
export APPLE_SDK_PATH=/Applications/Xcode_15.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.2.sdk
192-
elif [ "${{ matrix.build.target_triple }}" = "x86_64-apple-darwin" ]; then
96+
elif [ "${{ matrix.target_triple }}" = "x86_64-apple-darwin" ]; then
19397
export APPLE_SDK_PATH=/Applications/Xcode_15.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.2.sdk
19498
else
195-
echo "unhandled target triple: ${{ matrix.build.target_triple }}"
99+
echo "unhandled target triple: ${{ matrix.target_triple }}"
196100
exit 1
197101
fi
198102
199-
./build-macos.py --target-triple ${{ matrix.build.target_triple }} --python ${{ matrix.build.py }} --options ${{ matrix.build.options }}
103+
./build-macos.py --target-triple ${{ matrix.target_triple }} --python cpython-${{ matrix.python }} --options ${{ matrix.build_options }}
200104
201105
- name: Upload Distributions
202106
uses: actions/upload-artifact@v4
203107
with:
204-
name: ${{ matrix.build.py }}-${{ matrix.build.target_triple }}-${{ matrix.build.options }}
108+
name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}
205109
path: dist/*
206110

207111
- uses: actions/checkout@v4

0 commit comments

Comments
 (0)