Skip to content

Commit e03e383

Browse files
AleksPopov4AleksPopov4
committed
refactor(flake-checks-ci-matrix): Merge matrix generation jobs into a single one
1 parent e15b065 commit e03e383

File tree

1 file changed

+51
-76
lines changed

1 file changed

+51
-76
lines changed

‎.github/workflows/reusable-flake-checks-ci-matrix.yml‎

Lines changed: 51 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,12 @@ jobs:
3838
3939
This comment will be updated automatically with the status of each package.
4040
41-
generate-matrix-matrix:
42-
name: Generate Matrix of Matrices
41+
generate-matrices:
42+
name: Generate Matrices
4343
runs-on: ${{ fromJSON(inputs.runner) }}
44+
outputs:
45+
matrix: ${{ steps.merge-matrix.outputs.matrix }}
46+
fullMatrix: ${{ steps.merge-matrix.outputs.fullMatrix }}
4447
steps:
4548
- name: Install Nix
4649
uses: metacraft-labs/nixos-modules/.github/install-nix@main
@@ -54,122 +57,99 @@ jobs:
5457
- uses: actions/checkout@v4
5558

5659
- name: Generate Matrix for Matrix
57-
id: generate-matrix
60+
id: generate-matrix-of-matrices
5861
env:
5962
CACHIX_CACHE: ${{ vars.CACHIX_CACHE }}
6063
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
6164
MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }}
62-
run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl shard_matrix
63-
outputs:
64-
gen_matrix: ${{ steps.generate-matrix.outputs.gen_matrix }}
65-
66-
generate-matrix:
67-
needs: generate-matrix-matrix
68-
runs-on: ${{ fromJSON(inputs.runner) }}
69-
strategy:
70-
matrix: ${{fromJSON(needs.generate-matrix-matrix.outputs.gen_matrix)}}
71-
name: Generate Matrix ${{ matrix.digit != -1 && matrix.digit || '' }}
72-
steps:
73-
- name: Install Nix
74-
uses: metacraft-labs/nixos-modules/.github/install-nix@main
75-
with:
76-
cachix-cache: ${{ vars.CACHIX_CACHE }}
77-
cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
78-
trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
79-
substituters: ${{ vars.SUBSTITUTERS }}
80-
nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
81-
82-
- uses: actions/checkout@v4
65+
run: |
66+
nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH}}#mcl shard_matrix
8367
8468
- name: Generate CI Matrix
8569
id: generate-matrix
86-
shell: bash
8770
env:
8871
IS_INITIAL: 'true'
8972
CACHIX_CACHE: ${{ vars.CACHIX_CACHE }}
9073
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
91-
FLAKE_PRE: ${{ matrix.prefix }}
92-
FLAKE_POST: ${{ matrix.postfix }}
9374
MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }}
94-
run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl ci_matrix
75+
run: |
76+
matrix_json='${{ steps.generate-matrix-of-matrices.outputs.gen_matrix }}'
9577
96-
- uses: actions/upload-artifact@v4
97-
with:
98-
name: matrix-pre${{ matrix.digit != -1 && format('-{0}', matrix.digit) || '' }}
99-
path: matrix-pre.json
100-
outputs:
101-
matrix: ${{ steps.generate-matrix.outputs.matrix }}
78+
echo "GEN_MATRIX_JSON is:"
79+
echo "$matrix_json"
80+
81+
final_matrix='[]'
82+
83+
while read shard; do
84+
prefix=$(jq -r '.prefix' <<<"$shard")
85+
postfix=$(jq -r '.postfix' <<<"$shard")
86+
87+
echo "Processing shard -> prefix: $prefix, postfix: $postfix"
88+
89+
shard_output=$(PREFIX="$prefix" POSTFIX="$postfix" \
90+
nix run --accept-flake-config \
91+
github:metacraft-labs/nixos-modules/${MCL_BRANCH}#mcl ci_matrix)
92+
93+
echo "Shard output: $shard_output"
94+
95+
final_matrix=$(jq --argjson shardData "$shard_output" \
96+
'. + [$shardData]' <<<"$final_matrix")
97+
done < <(jq -c '.include[]' <<< "$matrix_json")
98+
99+
echo "Final aggregated matrix: $final_matrix"
102100
103-
slurp-matrix:
104-
runs-on: ${{ fromJSON(inputs.runner) }}
105-
needs: generate-matrix
106-
name: Merge matrices
107-
outputs:
108-
matrix: ${{ steps.matrix.outputs.matrix }}
109-
fullMatrix: ${{ steps.matrix.outputs.fullMatrix }}
110-
steps:
111-
- uses: actions/download-artifact@v4
112101
- name: Merge matrices
102+
id: merge-matrix
113103
run: |
114104
# Check if jq is installed
115105
if ! command -v jq &> /dev/null; then
116106
PATH="$(nix build --print-out-paths 'nixpkgs#jq^bin')/bin:$PATH"
117107
export PATH
118108
fi
119109
120-
ls */matrix-pre.json
121-
matrix="$(cat */matrix-pre.json | jq -cr '.include[]' | jq '[ select (.isCached == false) ]' | jq -s 'add' | jq -c '. | {include: .}')"
122-
123-
if [[ "$matrix" == '' ]] || [[ "$matrix" == '{}' ]] || [[ "$matrix" == '{"include":null}' ]] || [[ "$matrix" == '{"include":[]}' ]]; then
110+
if [ -f matrix-pre.json ]; then
111+
echo "Found matrix-pre.json. Merging..."
112+
matrix=$(jq -cr '.include' matrix-pre.json | jq -c '{include: .}')
113+
else
114+
# Fallback if the file doesn't exist
124115
matrix='{"include":[]}'
125116
fi
126117
127118
echo "---"
128119
echo "Matrix:"
129120
echo "$matrix" | jq
130121
echo "---"
131-
echo
132-
echo
133-
134-
fullMatrix="$(cat */matrix-pre.json | jq -cr '.include' | jq -s 'add' | jq -c '. | {include: .}')"
135122
136-
echo "---"
137-
echo "Full Matrix:"
138-
echo "$fullMatrix" | jq
139-
echo "---"
123+
# fullMatrix could be a superset of 'matrix'
124+
# or exactly the same, depending on your logic
125+
fullMatrix="$matrix"
140126
141-
echo "matrix=$matrix" >> $GITHUB_OUTPUT
142-
echo "fullMatrix=$fullMatrix" >> $GITHUB_OUTPUT
127+
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
128+
echo "fullMatrix=$fullMatrix" >> "$GITHUB_OUTPUT"
143129
144-
id: matrix
145130
- name: Post Comment
146-
id: print-matrix
147131
uses: metacraft-labs/nixos-modules/.github/print-matrix@main
148132
with:
149133
is-initial: 'true'
150134
cachix-cache: ${{ vars.CACHIX_CACHE }}
151135
cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
152136
trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
153137
substituters: ${{ vars.SUBSTITUTERS }}
154-
precalc_matrix: ${{ steps.matrix.outputs.fullMatrix }}
138+
precalc_matrix: ${{ steps.merge-matrix.outputs.fullMatrix }}
155139
pr-comment-github-token: ${{ secrets.GITHUB_TOKEN }}
156140
nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
157141

158142
build:
159-
needs: slurp-matrix
160-
if: needs.slurp-matrix.outputs.matrix != '{"include":[]}'
161-
143+
needs: generate-matrices
144+
if: needs.generate-matrices.outputs.matrix != '{"include":[]}'
162145
strategy:
163146
fail-fast: false
164-
matrix: ${{fromJSON( needs.slurp-matrix.outputs.matrix )}}
165-
147+
matrix: ${{ fromJSON(needs.generate-matrices.outputs.matrix) }}
166148
name: ${{ matrix.name }} | ${{ matrix.system }}
167149
runs-on: ${{ matrix.os }}
168150
continue-on-error: ${{ matrix.allowedToFail }}
169-
170151
steps:
171152
- uses: actions/checkout@v4
172-
173153
- name: Install Nix
174154
uses: metacraft-labs/nixos-modules/.github/install-nix@main
175155
with:
@@ -178,20 +158,18 @@ jobs:
178158
trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
179159
substituters: ${{ vars.SUBSTITUTERS }}
180160
nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
181-
182161
- name: Build ${{ matrix.name }}
183162
run: |
184163
nix build -L --no-link --keep-going --show-trace \
185164
'.#${{ matrix.attrPath }}'
186-
187165
- name: Push to Cachix ${{ matrix.name }}
188166
run: |
189167
cachix push ${{ vars.CACHIX_CACHE }} ${{ matrix.output }}
190168
191169
results:
170+
needs: [build, generate-matrices]
192171
runs-on: ${{ fromJSON(inputs.runner) }}
193172
name: Final Results
194-
needs: [build, slurp-matrix]
195173
if: always()
196174
steps:
197175
- name: Post Comment
@@ -202,24 +180,21 @@ jobs:
202180
cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
203181
trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
204182
substituters: ${{ vars.SUBSTITUTERS }}
205-
precalc_matrix: ${{ needs.slurp-matrix.outputs.fullMatrix }}
183+
precalc_matrix: ${{ needs.generate-matrices.outputs.fullMatrix }}
206184
pr-comment-github-token: ${{ secrets.GITHUB_TOKEN }}
207185
nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
208-
209186
- run: exit 1
210187
if: >-
211-
needs.slurp-matrix.outputs.matrix != '{"include":[]}'
188+
needs.generate-matrices.outputs.matrix != '{"include":[]}'
212189
&& contains(needs.*.result, 'failure')
213190
|| contains(needs.*.result, 'cancelled')
214-
215191
- uses: actions/checkout@v4
216192
if: inputs.run-cachix-deploy
217-
218193
- name: Deploy
219194
if: inputs.run-cachix-deploy
220195
env:
221196
CACHIX_CACHE: ${{ vars.CACHIX_CACHE }}
222197
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
223198
CACHIX_ACTIVATE_TOKEN: '${{ secrets.CACHIX_ACTIVATE_TOKEN }}'
224199
MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }}
225-
run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl deploy_spec
200+
run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${MCL_BRANCH}#mcl deploy_spec

0 commit comments

Comments
 (0)