Skip to content

Commit 3b3ed59

Browse files
zaniebzanieb
committed
Generate the matrix for building the pythonbuild crate
1 parent ce2f884 commit 3b3ed59

File tree

4 files changed

+138
-64
lines changed

4 files changed

+138
-64
lines changed

‎.github/workflows/linux.yml‎

Lines changed: 24 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,14 @@ env:
1313
FORCE_COLOR: 1
1414

1515
jobs:
16-
pythonbuild:
17-
if: ${{ needs.generate-matrix.outputs.pythonbuild_changed == 'true' || needs.generate-matrix.outputs.any_builds == 'true' || github.ref == 'refs/heads/main' }}
16+
crate-build:
1817
needs:
1918
- generate-matrix
20-
runs-on: depot-ubuntu-22.04
19+
runs-on: ${{ matrix.runner }}
20+
strategy:
21+
matrix: ${{ fromJson(needs.generate-matrix.outputs.crate-build-matrix) }}
22+
fail-fast: false
23+
name: crate-build | ${{ matrix.arch }}
2124
steps:
2225
- name: Install System Dependencies
2326
run: |
@@ -122,8 +125,9 @@ jobs:
122125
python-build-matrix-0: ${{ steps.set-matrix.outputs.python-build-matrix-0 }}
123126
python-build-matrix-1: ${{ steps.set-matrix.outputs.python-build-matrix-1 }}
124127
docker-build-matrix: ${{ steps.set-matrix.outputs.docker-build-matrix }}
128+
crate-build-matrix: ${{ steps.set-matrix.outputs.crate-build-matrix }}
125129
any_builds: ${{ steps.set-matrix.outputs.any_builds }}
126-
pythonbuild_changed: ${{ steps.check-pythonbuild.outputs.changed }}
130+
crate_changed: ${{ steps.check-pythonbuild.outputs.changed }}
127131
steps:
128132
- uses: actions/checkout@v4
129133
with:
@@ -139,18 +143,32 @@ jobs:
139143
LABELS=$(echo '${{ toJson(github.event.pull_request.labels.*.name) }}' | jq -r 'join(",")')
140144
echo "labels=$LABELS" >> $GITHUB_OUTPUT
141145
146+
- name: Check if the `pythonbuild` crate changed
147+
id: check-pythonbuild
148+
env:
149+
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
150+
run: |
151+
merge_base=$(git merge-base HEAD "origin/${BASE_REF}")
152+
if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then
153+
echo "changed=false" >> "$GITHUB_OUTPUT"
154+
else
155+
echo "changed=true" >> "$GITHUB_OUTPUT"
156+
fi
157+
142158
- name: Generate build matrix
143159
id: set-matrix
144160
run: |
145161
uv run ci-matrix.py \
146162
--platform linux \
147163
--labels '${{ steps.get-labels.outputs.labels }}' \
148164
--max-shards 2 \
165+
${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} \
149166
> matrix.json
150167
151168
echo "python-build-matrix-0=$(jq -c '."python-build"["0"]' matrix.json)" >> $GITHUB_OUTPUT
152169
echo "python-build-matrix-1=$(jq -c '."python-build"["1"]' matrix.json)" >> $GITHUB_OUTPUT
153170
echo "docker-build-matrix=$(jq -c '."docker-build"' matrix.json)" >> $GITHUB_OUTPUT
171+
echo "crate-build-matrix=$(jq -c '."crate-build"' matrix.json)" >> $GITHUB_OUTPUT
154172
155173
# Display the matrix for debugging too
156174
cat matrix.json | jq
@@ -163,22 +181,10 @@ jobs:
163181
echo "any_builds=false" >> $GITHUB_OUTPUT
164182
fi
165183
166-
- name: Check if the `pythonbuild` crate changed
167-
id: check-pythonbuild
168-
env:
169-
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
170-
run: |
171-
merge_base=$(git merge-base HEAD "origin/${BASE_REF}")
172-
if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then
173-
echo "changed=false" >> "$GITHUB_OUTPUT"
174-
else
175-
echo "changed=true" >> "$GITHUB_OUTPUT"
176-
fi
177-
178184
build-0:
179185
needs:
180186
- generate-matrix
181-
- pythonbuild
187+
- crate-build
182188
- image
183189
# Permissions used for actions/attest-build-provenance
184190
permissions:
@@ -278,7 +284,7 @@ jobs:
278284
build-1:
279285
needs:
280286
- generate-matrix
281-
- pythonbuild
287+
- crate-build
282288
- image
283289
# Permissions used for actions/attest-build-provenance
284290
permissions:

‎.github/workflows/macos.yml‎

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,14 @@ env:
1313
FORCE_COLOR: 1
1414

1515
jobs:
16-
pythonbuild:
17-
if: ${{ needs.generate-matrix.outputs.pythonbuild_changed == 'true' || needs.generate-matrix.outputs.any_builds == 'true' || github.ref == 'refs/heads/main' }}
16+
crate-build:
1817
needs:
1918
- generate-matrix
20-
runs-on: depot-macos-latest
19+
runs-on: ${{ matrix.runner }}
20+
strategy:
21+
matrix: ${{ fromJson(needs.generate-matrix.outputs.crate-build-matrix) }}
22+
fail-fast: false
23+
name: crate-build | ${{ matrix.arch }}
2124
steps:
2225
- uses: actions/checkout@v4
2326

@@ -47,8 +50,9 @@ jobs:
4750
runs-on: ubuntu-latest
4851
outputs:
4952
matrix: ${{ steps.set-matrix.outputs.matrix }}
53+
crate-build-matrix: ${{ steps.set-matrix.outputs.crate-build-matrix }}
5054
any_builds: ${{ steps.set-matrix.outputs.any_builds }}
51-
pythonbuild_changed: ${{ steps.check-pythonbuild.outputs.changed }}
55+
crate_changed: ${{ steps.check-pythonbuild.outputs.changed }}
5256
steps:
5357
- uses: actions/checkout@v4
5458
with:
@@ -64,13 +68,26 @@ jobs:
6468
LABELS=$(echo '${{ toJson(github.event.pull_request.labels.*.name) }}' | jq -r 'join(",")')
6569
echo "labels=$LABELS" >> $GITHUB_OUTPUT
6670
71+
- name: Check if the `pythonbuild` crate changed
72+
id: check-pythonbuild
73+
env:
74+
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
75+
run: |
76+
merge_base=$(git merge-base HEAD "origin/${BASE_REF}")
77+
if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then
78+
echo "changed=false" >> "$GITHUB_OUTPUT"
79+
else
80+
echo "changed=true" >> "$GITHUB_OUTPUT"
81+
fi
82+
6783
- name: Generate build matrix
6884
id: set-matrix
6985
run: |
70-
uv run ci-matrix.py --platform darwin --labels '${{ steps.get-labels.outputs.labels }}' > matrix.json
86+
uv run ci-matrix.py --platform darwin --labels '${{ steps.get-labels.outputs.labels }}' ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json
7187
7288
# Extract python-build matrix
7389
echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT
90+
echo "crate-build-matrix=$(jq -c '."crate-build"' matrix.json)" >> $GITHUB_OUTPUT
7491
7592
# Display the matrix for debugging too
7693
cat matrix.json | jq
@@ -83,22 +100,10 @@ jobs:
83100
echo "any_builds=false" >> $GITHUB_OUTPUT
84101
fi
85102
86-
- name: Check if the `pythonbuild` crate changed
87-
id: check-pythonbuild
88-
env:
89-
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
90-
run: |
91-
merge_base=$(git merge-base HEAD "origin/${BASE_REF}")
92-
if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then
93-
echo "changed=false" >> "$GITHUB_OUTPUT"
94-
else
95-
echo "changed=true" >> "$GITHUB_OUTPUT"
96-
fi
97-
98103
build:
99104
needs:
100105
- generate-matrix
101-
- pythonbuild
106+
- crate-build
102107
# Permissions used for actions/attest-build-provenance
103108
permissions:
104109
id-token: write

‎.github/workflows/windows.yml‎

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,14 @@ env:
1313
FORCE_COLOR: 1
1414

1515
jobs:
16-
pythonbuild:
17-
if: ${{ needs.generate-matrix.outputs.pythonbuild_changed == 'true' || needs.generate-matrix.outputs.any_builds == 'true' || github.ref == 'refs/heads/main' }}
16+
crate-build:
1817
needs:
1918
- generate-matrix
20-
runs-on: 'windows-2022'
19+
runs-on: ${{ matrix.runner }}
20+
strategy:
21+
matrix: ${{ fromJson(needs.generate-matrix.outputs.crate-build-matrix) }}
22+
fail-fast: false
23+
name: crate-build | ${{ matrix.arch }}
2124
steps:
2225
- uses: actions/checkout@v4
2326

@@ -47,8 +50,9 @@ jobs:
4750
runs-on: ubuntu-latest
4851
outputs:
4952
matrix: ${{ steps.set-matrix.outputs.matrix }}
53+
crate-build-matrix: ${{ steps.set-matrix.outputs.crate-build-matrix }}
5054
any_builds: ${{ steps.set-matrix.outputs.any_builds }}
51-
pythonbuild_changed: ${{ steps.check-pythonbuild.outputs.changed }}
55+
crate_changed: ${{ steps.check-pythonbuild.outputs.changed }}
5256
steps:
5357
- uses: actions/checkout@v4
5458
with:
@@ -64,13 +68,26 @@ jobs:
6468
LABELS=$(echo '${{ toJson(github.event.pull_request.labels.*.name) }}' | jq -r 'join(",")')
6569
echo "labels=$LABELS" >> $GITHUB_OUTPUT
6670
71+
- name: Check if the `pythonbuild` crate changed
72+
id: check-pythonbuild
73+
env:
74+
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
75+
run: |
76+
merge_base=$(git merge-base HEAD "origin/${BASE_REF}")
77+
if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then
78+
echo "changed=false" >> "$GITHUB_OUTPUT"
79+
else
80+
echo "changed=true" >> "$GITHUB_OUTPUT"
81+
fi
82+
6783
- name: Generate build matrix
6884
id: set-matrix
6985
run: |
70-
uv run ci-matrix.py --platform windows --labels '${{ steps.get-labels.outputs.labels }}' > matrix.json
86+
uv run ci-matrix.py --platform windows --labels '${{ steps.get-labels.outputs.labels }}' ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json
7187
7288
# Extract python-build matrix
7389
echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT
90+
echo "crate-build-matrix=$(jq -c '."crate-build"' matrix.json)" >> $GITHUB_OUTPUT
7491
7592
# Display the matrix for debugging too
7693
cat matrix.json | jq
@@ -83,23 +100,11 @@ jobs:
83100
echo "any_builds=false" >> $GITHUB_OUTPUT
84101
fi
85102
86-
- name: Check if the `pythonbuild` crate changed
87-
id: check-pythonbuild
88-
env:
89-
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
90-
run: |
91-
merge_base=$(git merge-base HEAD "origin/${BASE_REF}")
92-
if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then
93-
echo "changed=false" >> "$GITHUB_OUTPUT"
94-
else
95-
echo "changed=true" >> "$GITHUB_OUTPUT"
96-
fi
97-
98103
build:
99104
timeout-minutes: 60
100105
needs:
101106
- generate-matrix
102-
- pythonbuild
107+
- crate-build
103108
# Permissions used for actions/attest-build-provenance
104109
permissions:
105110
id-token: write

‎ci-matrix.py‎

Lines changed: 68 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,48 @@ def generate_docker_matrix_entries(
120120
return matrix_entries
121121

122122

123+
def generate_crate_build_matrix_entries(
124+
python_entries: list[dict[str, str]],
125+
runners: dict[str, Any],
126+
config: dict[str, Any],
127+
force_crate_build: bool = False,
128+
platform_filter: Optional[str] = None,
129+
) -> list[dict[str, str]]:
130+
"""Generate matrix entries for crate builds based on python build matrix."""
131+
# Include entries for the Python builds we're doing
132+
targets = {
133+
(entry["platform"], entry["arch"])
134+
for entry in python_entries
135+
# Only include if native
136+
if entry.get("run", "false") == "true"
137+
}
138+
139+
# If forcing crate build, include all possible targets
140+
if force_crate_build:
141+
# Add targets for all platforms, but only native/host architectures
142+
for platform, platform_config in config.items():
143+
# Filter by platform if specified
144+
if platform_filter and platform != platform_filter:
145+
continue
146+
147+
for target_config in platform_config.values():
148+
# Only include if native
149+
if target_config.get("run"):
150+
continue
151+
152+
arch = target_config["arch"]
153+
targets.add((platform, arch))
154+
155+
return [
156+
{
157+
"platform": platform,
158+
"arch": arch,
159+
"runner": find_runner(runners, platform, arch),
160+
}
161+
for platform, arch in targets
162+
]
163+
164+
123165
def generate_python_build_matrix_entries(
124166
config: dict[str, Any],
125167
runners: dict[str, Any],
@@ -266,9 +308,14 @@ def parse_args() -> argparse.Namespace:
266308
action="store_true",
267309
help="If only free runners should be used.",
268310
)
311+
parser.add_argument(
312+
"--force-crate-build",
313+
action="store_true",
314+
help="Force crate builds to be included even without python builds.",
315+
)
269316
parser.add_argument(
270317
"--matrix-type",
271-
choices=["python-build", "docker-build", "all"],
318+
choices=["python-build", "docker-build", "crate-build", "all"],
272319
default="all",
273320
help="Which matrix types to generate (default: all)",
274321
)
@@ -295,16 +342,16 @@ def main() -> None:
295342

296343
result = {}
297344

298-
# Generate python-build matrix if requested
299-
python_entries = []
300-
if args.matrix_type in ["python-build", "all"]:
301-
python_entries = generate_python_build_matrix_entries(
302-
config,
303-
runners,
304-
args.platform,
305-
labels,
306-
)
345+
# Generate python build entries
346+
python_entries = generate_python_build_matrix_entries(
347+
config,
348+
runners,
349+
args.platform,
350+
labels,
351+
)
307352

353+
# Output python-build matrix if requested
354+
if args.matrix_type in ["python-build", "all"]:
308355
if args.max_shards:
309356
python_build_matrix = {}
310357
shards = (len(python_entries) // CI_MATRIX_SIZE_LIMIT) + 1
@@ -345,6 +392,17 @@ def main() -> None:
345392
)
346393
result["docker-build"] = {"include": docker_entries}
347394

395+
# Generate crate-build matrix if requested
396+
if args.matrix_type in ["crate-build", "all"]:
397+
crate_entries = generate_crate_build_matrix_entries(
398+
python_entries,
399+
runners,
400+
config,
401+
args.force_crate_build,
402+
args.platform,
403+
)
404+
result["crate-build"] = {"include": crate_entries}
405+
348406
print(json.dumps(result))
349407

350408

0 commit comments

Comments
 (0)