Skip to content

Commit c3478db

Browse files
zaniebzanieb
committed
Generate the matrix for building the pythonbuild crate
1 parent ce2f884 commit c3478db

File tree

4 files changed

+129
-64
lines changed

4 files changed

+129
-64
lines changed

‎.github/workflows/linux.yml‎

Lines changed: 24 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,14 @@ env:
1313
FORCE_COLOR: 1
1414

1515
jobs:
16-
pythonbuild:
17-
if: ${{ needs.generate-matrix.outputs.pythonbuild_changed == 'true' || needs.generate-matrix.outputs.any_builds == 'true' || github.ref == 'refs/heads/main' }}
16+
crate-build:
1817
needs:
1918
- generate-matrix
20-
runs-on: depot-ubuntu-22.04
19+
runs-on: ${{ matrix.runner }}
20+
strategy:
21+
matrix: ${{ fromJson(needs.generate-matrix.outputs.crate-build-matrix) }}
22+
fail-fast: false
23+
name: ${{ matrix.arch }}
2124
steps:
2225
- name: Install System Dependencies
2326
run: |
@@ -122,8 +125,9 @@ jobs:
122125
python-build-matrix-0: ${{ steps.set-matrix.outputs.python-build-matrix-0 }}
123126
python-build-matrix-1: ${{ steps.set-matrix.outputs.python-build-matrix-1 }}
124127
docker-build-matrix: ${{ steps.set-matrix.outputs.docker-build-matrix }}
128+
crate-build-matrix: ${{ steps.set-matrix.outputs.crate-build-matrix }}
125129
any_builds: ${{ steps.set-matrix.outputs.any_builds }}
126-
pythonbuild_changed: ${{ steps.check-pythonbuild.outputs.changed }}
130+
crate_changed: ${{ steps.check-pythonbuild.outputs.changed }}
127131
steps:
128132
- uses: actions/checkout@v4
129133
with:
@@ -139,18 +143,32 @@ jobs:
139143
LABELS=$(echo '${{ toJson(github.event.pull_request.labels.*.name) }}' | jq -r 'join(",")')
140144
echo "labels=$LABELS" >> $GITHUB_OUTPUT
141145
146+
- name: Check if the `pythonbuild` crate changed
147+
id: check-pythonbuild
148+
env:
149+
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
150+
run: |
151+
merge_base=$(git merge-base HEAD "origin/${BASE_REF}")
152+
if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then
153+
echo "changed=false" >> "$GITHUB_OUTPUT"
154+
else
155+
echo "changed=true" >> "$GITHUB_OUTPUT"
156+
fi
157+
142158
- name: Generate build matrix
143159
id: set-matrix
144160
run: |
145161
uv run ci-matrix.py \
146162
--platform linux \
147163
--labels '${{ steps.get-labels.outputs.labels }}' \
148164
--max-shards 2 \
165+
${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} \
149166
> matrix.json
150167
151168
echo "python-build-matrix-0=$(jq -c '."python-build"["0"]' matrix.json)" >> $GITHUB_OUTPUT
152169
echo "python-build-matrix-1=$(jq -c '."python-build"["1"]' matrix.json)" >> $GITHUB_OUTPUT
153170
echo "docker-build-matrix=$(jq -c '."docker-build"' matrix.json)" >> $GITHUB_OUTPUT
171+
echo "crate-build-matrix=$(jq -c '."crate-build"' matrix.json)" >> $GITHUB_OUTPUT
154172
155173
# Display the matrix for debugging too
156174
cat matrix.json | jq
@@ -163,22 +181,10 @@ jobs:
163181
echo "any_builds=false" >> $GITHUB_OUTPUT
164182
fi
165183
166-
- name: Check if the `pythonbuild` crate changed
167-
id: check-pythonbuild
168-
env:
169-
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
170-
run: |
171-
merge_base=$(git merge-base HEAD "origin/${BASE_REF}")
172-
if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then
173-
echo "changed=false" >> "$GITHUB_OUTPUT"
174-
else
175-
echo "changed=true" >> "$GITHUB_OUTPUT"
176-
fi
177-
178184
build-0:
179185
needs:
180186
- generate-matrix
181-
- pythonbuild
187+
- crate-build
182188
- image
183189
# Permissions used for actions/attest-build-provenance
184190
permissions:
@@ -278,7 +284,7 @@ jobs:
278284
build-1:
279285
needs:
280286
- generate-matrix
281-
- pythonbuild
287+
- crate-build
282288
- image
283289
# Permissions used for actions/attest-build-provenance
284290
permissions:

‎.github/workflows/macos.yml‎

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,14 @@ env:
1313
FORCE_COLOR: 1
1414

1515
jobs:
16-
pythonbuild:
17-
if: ${{ needs.generate-matrix.outputs.pythonbuild_changed == 'true' || needs.generate-matrix.outputs.any_builds == 'true' || github.ref == 'refs/heads/main' }}
16+
crate-build:
1817
needs:
1918
- generate-matrix
20-
runs-on: depot-macos-latest
19+
runs-on: ${{ matrix.runner }}
20+
strategy:
21+
matrix: ${{ fromJson(needs.generate-matrix.outputs.crate-build-matrix) }}
22+
fail-fast: false
23+
name: ${{ matrix.arch }}
2124
steps:
2225
- uses: actions/checkout@v4
2326

@@ -47,8 +50,9 @@ jobs:
4750
runs-on: ubuntu-latest
4851
outputs:
4952
matrix: ${{ steps.set-matrix.outputs.matrix }}
53+
crate-build-matrix: ${{ steps.set-matrix.outputs.crate-build-matrix }}
5054
any_builds: ${{ steps.set-matrix.outputs.any_builds }}
51-
pythonbuild_changed: ${{ steps.check-pythonbuild.outputs.changed }}
55+
crate_changed: ${{ steps.check-pythonbuild.outputs.changed }}
5256
steps:
5357
- uses: actions/checkout@v4
5458
with:
@@ -64,13 +68,26 @@ jobs:
6468
LABELS=$(echo '${{ toJson(github.event.pull_request.labels.*.name) }}' | jq -r 'join(",")')
6569
echo "labels=$LABELS" >> $GITHUB_OUTPUT
6670
71+
- name: Check if the `pythonbuild` crate changed
72+
id: check-pythonbuild
73+
env:
74+
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
75+
run: |
76+
merge_base=$(git merge-base HEAD "origin/${BASE_REF}")
77+
if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then
78+
echo "changed=false" >> "$GITHUB_OUTPUT"
79+
else
80+
echo "changed=true" >> "$GITHUB_OUTPUT"
81+
fi
82+
6783
- name: Generate build matrix
6884
id: set-matrix
6985
run: |
70-
uv run ci-matrix.py --platform darwin --labels '${{ steps.get-labels.outputs.labels }}' > matrix.json
86+
uv run ci-matrix.py --platform darwin --labels '${{ steps.get-labels.outputs.labels }}' ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json
7187
7288
# Extract python-build matrix
7389
echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT
90+
echo "crate-build-matrix=$(jq -c '."crate-build"' matrix.json)" >> $GITHUB_OUTPUT
7491
7592
# Display the matrix for debugging too
7693
cat matrix.json | jq
@@ -83,22 +100,10 @@ jobs:
83100
echo "any_builds=false" >> $GITHUB_OUTPUT
84101
fi
85102
86-
- name: Check if the `pythonbuild` crate changed
87-
id: check-pythonbuild
88-
env:
89-
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
90-
run: |
91-
merge_base=$(git merge-base HEAD "origin/${BASE_REF}")
92-
if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then
93-
echo "changed=false" >> "$GITHUB_OUTPUT"
94-
else
95-
echo "changed=true" >> "$GITHUB_OUTPUT"
96-
fi
97-
98103
build:
99104
needs:
100105
- generate-matrix
101-
- pythonbuild
106+
- crate-build
102107
# Permissions used for actions/attest-build-provenance
103108
permissions:
104109
id-token: write

‎.github/workflows/windows.yml‎

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,14 @@ env:
1313
FORCE_COLOR: 1
1414

1515
jobs:
16-
pythonbuild:
17-
if: ${{ needs.generate-matrix.outputs.pythonbuild_changed == 'true' || needs.generate-matrix.outputs.any_builds == 'true' || github.ref == 'refs/heads/main' }}
16+
crate-build:
1817
needs:
1918
- generate-matrix
20-
runs-on: 'windows-2022'
19+
runs-on: ${{ matrix.runner }}
20+
strategy:
21+
matrix: ${{ fromJson(needs.generate-matrix.outputs.crate-build-matrix) }}
22+
fail-fast: false
23+
name: ${{ matrix.arch }}
2124
steps:
2225
- uses: actions/checkout@v4
2326

@@ -47,8 +50,9 @@ jobs:
4750
runs-on: ubuntu-latest
4851
outputs:
4952
matrix: ${{ steps.set-matrix.outputs.matrix }}
53+
crate-build-matrix: ${{ steps.set-matrix.outputs.crate-build-matrix }}
5054
any_builds: ${{ steps.set-matrix.outputs.any_builds }}
51-
pythonbuild_changed: ${{ steps.check-pythonbuild.outputs.changed }}
55+
crate_changed: ${{ steps.check-pythonbuild.outputs.changed }}
5256
steps:
5357
- uses: actions/checkout@v4
5458
with:
@@ -64,13 +68,26 @@ jobs:
6468
LABELS=$(echo '${{ toJson(github.event.pull_request.labels.*.name) }}' | jq -r 'join(",")')
6569
echo "labels=$LABELS" >> $GITHUB_OUTPUT
6670
71+
- name: Check if the `pythonbuild` crate changed
72+
id: check-pythonbuild
73+
env:
74+
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
75+
run: |
76+
merge_base=$(git merge-base HEAD "origin/${BASE_REF}")
77+
if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then
78+
echo "changed=false" >> "$GITHUB_OUTPUT"
79+
else
80+
echo "changed=true" >> "$GITHUB_OUTPUT"
81+
fi
82+
6783
- name: Generate build matrix
6884
id: set-matrix
6985
run: |
70-
uv run ci-matrix.py --platform windows --labels '${{ steps.get-labels.outputs.labels }}' > matrix.json
86+
uv run ci-matrix.py --platform windows --labels '${{ steps.get-labels.outputs.labels }}' ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json
7187
7288
# Extract python-build matrix
7389
echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT
90+
echo "crate-build-matrix=$(jq -c '."crate-build"' matrix.json)" >> $GITHUB_OUTPUT
7491
7592
# Display the matrix for debugging too
7693
cat matrix.json | jq
@@ -83,23 +100,11 @@ jobs:
83100
echo "any_builds=false" >> $GITHUB_OUTPUT
84101
fi
85102
86-
- name: Check if the `pythonbuild` crate changed
87-
id: check-pythonbuild
88-
env:
89-
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
90-
run: |
91-
merge_base=$(git merge-base HEAD "origin/${BASE_REF}")
92-
if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then
93-
echo "changed=false" >> "$GITHUB_OUTPUT"
94-
else
95-
echo "changed=true" >> "$GITHUB_OUTPUT"
96-
fi
97-
98103
build:
99104
timeout-minutes: 60
100105
needs:
101106
- generate-matrix
102-
- pythonbuild
107+
- crate-build
103108
# Permissions used for actions/attest-build-provenance
104109
permissions:
105110
id-token: write

‎ci-matrix.py‎

Lines changed: 59 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,39 @@ def generate_docker_matrix_entries(
120120
return matrix_entries
121121

122122

123+
def generate_crate_build_matrix_entries(
124+
python_entries: list[dict[str, str]],
125+
runners: dict[str, Any],
126+
force_crate_build: bool = False,
127+
platform_filter: Optional[str] = None,
128+
config: Optional[dict[str, Any]] = None,
129+
) -> list[dict[str, str]]:
130+
"""Generate matrix entries for crate builds based on python build matrix."""
131+
unique_combinations = {
132+
(entry["platform"], entry["arch"]) for entry in python_entries
133+
}
134+
135+
# If forcing crate build but no python builds, include all targets
136+
if force_crate_build and not unique_combinations and config:
137+
# Add all targets for all platforms
138+
for platform, platform_config in config.items():
139+
# Filter by platform if specified
140+
if platform_filter and platform != platform_filter:
141+
continue
142+
for target_triple, target_config in platform_config.items():
143+
arch = target_config["arch"]
144+
unique_combinations.add((platform, arch))
145+
146+
return [
147+
{
148+
"platform": platform,
149+
"arch": arch,
150+
"runner": find_runner(runners, platform, arch),
151+
}
152+
for platform, arch in unique_combinations
153+
]
154+
155+
123156
def generate_python_build_matrix_entries(
124157
config: dict[str, Any],
125158
runners: dict[str, Any],
@@ -266,9 +299,14 @@ def parse_args() -> argparse.Namespace:
266299
action="store_true",
267300
help="If only free runners should be used.",
268301
)
302+
parser.add_argument(
303+
"--force-crate-build",
304+
action="store_true",
305+
help="Force crate builds to be included even without python builds.",
306+
)
269307
parser.add_argument(
270308
"--matrix-type",
271-
choices=["python-build", "docker-build", "all"],
309+
choices=["python-build", "docker-build", "crate-build", "all"],
272310
default="all",
273311
help="Which matrix types to generate (default: all)",
274312
)
@@ -295,16 +333,16 @@ def main() -> None:
295333

296334
result = {}
297335

298-
# Generate python-build matrix if requested
299-
python_entries = []
300-
if args.matrix_type in ["python-build", "all"]:
301-
python_entries = generate_python_build_matrix_entries(
302-
config,
303-
runners,
304-
args.platform,
305-
labels,
306-
)
336+
# Generate python build entries
337+
python_entries = generate_python_build_matrix_entries(
338+
config,
339+
runners,
340+
args.platform,
341+
labels,
342+
)
307343

344+
# Output python-build matrix if requested
345+
if args.matrix_type in ["python-build", "all"]:
308346
if args.max_shards:
309347
python_build_matrix = {}
310348
shards = (len(python_entries) // CI_MATRIX_SIZE_LIMIT) + 1
@@ -345,6 +383,17 @@ def main() -> None:
345383
)
346384
result["docker-build"] = {"include": docker_entries}
347385

386+
# Generate crate-build matrix if requested
387+
if args.matrix_type in ["crate-build", "all"]:
388+
crate_entries = generate_crate_build_matrix_entries(
389+
python_entries,
390+
runners,
391+
args.force_crate_build,
392+
args.platform,
393+
config,
394+
)
395+
result["crate-build"] = {"include": crate_entries}
396+
348397
print(json.dumps(result))
349398

350399

0 commit comments

Comments
 (0)