Skip to content

Commit 8216b25

Browse files
AleksPopov4AleksPopov4
committed
refactor(flake-checks-ci-matrix): Merge matrix generation jobs into a single one
1 parent e15b065 commit 8216b25

File tree

2 files changed

+53
-76
lines changed

2 files changed

+53
-76
lines changed

‎.github/workflows/reusable-flake-checks-ci-matrix.yml‎

Lines changed: 52 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,12 @@ jobs:
3838
3939
This comment will be updated automatically with the status of each package.
4040
41-
generate-matrix-matrix:
42-
name: Generate Matrix of Matrices
41+
generate-matrices:
42+
name: Generate Matrices
4343
runs-on: ${{ fromJSON(inputs.runner) }}
44+
outputs:
45+
matrix: ${{ steps.merge-matrix.outputs.matrix }}
46+
fullMatrix: ${{ steps.merge-matrix.outputs.fullMatrix }}
4447
steps:
4548
- name: Install Nix
4649
uses: metacraft-labs/nixos-modules/.github/install-nix@main
@@ -54,122 +57,101 @@ jobs:
5457
- uses: actions/checkout@v4
5558

5659
- name: Generate Matrix for Matrix
57-
id: generate-matrix
60+
id: generate-matrix-of-matrices
5861
env:
5962
CACHIX_CACHE: ${{ vars.CACHIX_CACHE }}
6063
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
6164
MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }}
62-
run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl shard_matrix
63-
outputs:
64-
gen_matrix: ${{ steps.generate-matrix.outputs.gen_matrix }}
65-
66-
generate-matrix:
67-
needs: generate-matrix-matrix
68-
runs-on: ${{ fromJSON(inputs.runner) }}
69-
strategy:
70-
matrix: ${{fromJSON(needs.generate-matrix-matrix.outputs.gen_matrix)}}
71-
name: Generate Matrix ${{ matrix.digit != -1 && matrix.digit || '' }}
72-
steps:
73-
- name: Install Nix
74-
uses: metacraft-labs/nixos-modules/.github/install-nix@main
75-
with:
76-
cachix-cache: ${{ vars.CACHIX_CACHE }}
77-
cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
78-
trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
79-
substituters: ${{ vars.SUBSTITUTERS }}
80-
nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
65+
run: |
66+
gen_matrix_output=$(nix run --accept-flake-config \
67+
github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH}}#mcl shard_matrix)
8168
82-
- uses: actions/checkout@v4
69+
echo $gen_matrix_output
70+
echo "gen_matrix=$gen_matrix_output" >> $GITHUB_OUTPUT
8371
8472
- name: Generate CI Matrix
8573
id: generate-matrix
86-
shell: bash
8774
env:
8875
IS_INITIAL: 'true'
8976
CACHIX_CACHE: ${{ vars.CACHIX_CACHE }}
9077
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
91-
FLAKE_PRE: ${{ matrix.prefix }}
92-
FLAKE_POST: ${{ matrix.postfix }}
9378
MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }}
94-
run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl ci_matrix
79+
run: |
80+
echo "GEN_MATRIX_JSON is:"
81+
echo ${{ steps.generate-matrix-of-matrices.outputs.gen_matrix }}
9582
96-
- uses: actions/upload-artifact@v4
97-
with:
98-
name: matrix-pre${{ matrix.digit != -1 && format('-{0}', matrix.digit) || '' }}
99-
path: matrix-pre.json
100-
outputs:
101-
matrix: ${{ steps.generate-matrix.outputs.matrix }}
83+
final_matrix='[]'
84+
85+
while read shard; do
86+
prefix=$(jq -r '.prefix' <<<"$shard")
87+
postfix=$(jq -r '.postfix' <<<"$shard")
88+
89+
echo "Processing shard -> prefix: $prefix, postfix: $postfix"
90+
91+
shard_output=$(PREFIX="$prefix" POSTFIX="$postfix" \
92+
nix run --accept-flake-config \
93+
github:metacraft-labs/nixos-modules/${MCL_BRANCH}#mcl ci_matrix)
94+
95+
echo "Shard output: $shard_output"
96+
97+
final_matrix=$(jq --argjson shardData "$shard_output" \
98+
'. + [$shardData]' <<<"$final_matrix")
99+
done < <(jq -c '.include[]' <<<${{ steps.generate-matrix-of-matrices.outputs.gen_matrix }})
100+
101+
echo "Final aggregated matrix: $final_matrix"
102102
103-
slurp-matrix:
104-
runs-on: ${{ fromJSON(inputs.runner) }}
105-
needs: generate-matrix
106-
name: Merge matrices
107-
outputs:
108-
matrix: ${{ steps.matrix.outputs.matrix }}
109-
fullMatrix: ${{ steps.matrix.outputs.fullMatrix }}
110-
steps:
111-
- uses: actions/download-artifact@v4
112103
- name: Merge matrices
104+
id: merge-matrix
113105
run: |
114106
# Check if jq is installed
115107
if ! command -v jq &> /dev/null; then
116108
PATH="$(nix build --print-out-paths 'nixpkgs#jq^bin')/bin:$PATH"
117109
export PATH
118110
fi
119111
120-
ls */matrix-pre.json
121-
matrix="$(cat */matrix-pre.json | jq -cr '.include[]' | jq '[ select (.isCached == false) ]' | jq -s 'add' | jq -c '. | {include: .}')"
122-
123-
if [[ "$matrix" == '' ]] || [[ "$matrix" == '{}' ]] || [[ "$matrix" == '{"include":null}' ]] || [[ "$matrix" == '{"include":[]}' ]]; then
112+
if [ -f matrix-pre.json ]; then
113+
echo "Found matrix-pre.json. Merging..."
114+
matrix=$(jq -cr '.include' matrix-pre.json | jq -c '{include: .}')
115+
else
116+
# Fallback if the file doesn't exist
124117
matrix='{"include":[]}'
125118
fi
126119
127120
echo "---"
128121
echo "Matrix:"
129122
echo "$matrix" | jq
130123
echo "---"
131-
echo
132-
echo
133124
134-
fullMatrix="$(cat */matrix-pre.json | jq -cr '.include' | jq -s 'add' | jq -c '. | {include: .}')"
125+
# fullMatrix could be a superset of 'matrix'
126+
# or exactly the same, depending on your logic
127+
fullMatrix="$matrix"
135128
136-
echo "---"
137-
echo "Full Matrix:"
138-
echo "$fullMatrix" | jq
139-
echo "---"
129+
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
130+
echo "fullMatrix=$fullMatrix" >> "$GITHUB_OUTPUT"
140131
141-
echo "matrix=$matrix" >> $GITHUB_OUTPUT
142-
echo "fullMatrix=$fullMatrix" >> $GITHUB_OUTPUT
143-
144-
id: matrix
145132
- name: Post Comment
146-
id: print-matrix
147133
uses: metacraft-labs/nixos-modules/.github/print-matrix@main
148134
with:
149135
is-initial: 'true'
150136
cachix-cache: ${{ vars.CACHIX_CACHE }}
151137
cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
152138
trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
153139
substituters: ${{ vars.SUBSTITUTERS }}
154-
precalc_matrix: ${{ steps.matrix.outputs.fullMatrix }}
140+
precalc_matrix: ${{ steps.merge-matrix.outputs.fullMatrix }}
155141
pr-comment-github-token: ${{ secrets.GITHUB_TOKEN }}
156142
nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
157143

158144
build:
159-
needs: slurp-matrix
160-
if: needs.slurp-matrix.outputs.matrix != '{"include":[]}'
161-
145+
needs: generate-matrices
146+
if: needs.generate-matrices.outputs.matrix != '{"include":[]}'
162147
strategy:
163148
fail-fast: false
164-
matrix: ${{fromJSON( needs.slurp-matrix.outputs.matrix )}}
165-
149+
matrix: ${{ fromJSON(needs.generate-matrices.outputs.matrix) }}
166150
name: ${{ matrix.name }} | ${{ matrix.system }}
167151
runs-on: ${{ matrix.os }}
168152
continue-on-error: ${{ matrix.allowedToFail }}
169-
170153
steps:
171154
- uses: actions/checkout@v4
172-
173155
- name: Install Nix
174156
uses: metacraft-labs/nixos-modules/.github/install-nix@main
175157
with:
@@ -178,20 +160,18 @@ jobs:
178160
trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
179161
substituters: ${{ vars.SUBSTITUTERS }}
180162
nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
181-
182163
- name: Build ${{ matrix.name }}
183164
run: |
184165
nix build -L --no-link --keep-going --show-trace \
185166
'.#${{ matrix.attrPath }}'
186-
187167
- name: Push to Cachix ${{ matrix.name }}
188168
run: |
189169
cachix push ${{ vars.CACHIX_CACHE }} ${{ matrix.output }}
190170
191171
results:
172+
needs: [build, generate-matrices]
192173
runs-on: ${{ fromJSON(inputs.runner) }}
193174
name: Final Results
194-
needs: [build, slurp-matrix]
195175
if: always()
196176
steps:
197177
- name: Post Comment
@@ -202,24 +182,21 @@ jobs:
202182
cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
203183
trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
204184
substituters: ${{ vars.SUBSTITUTERS }}
205-
precalc_matrix: ${{ needs.slurp-matrix.outputs.fullMatrix }}
185+
precalc_matrix: ${{ needs.generate-matrices.outputs.fullMatrix }}
206186
pr-comment-github-token: ${{ secrets.GITHUB_TOKEN }}
207187
nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
208-
209188
- run: exit 1
210189
if: >-
211-
needs.slurp-matrix.outputs.matrix != '{"include":[]}'
190+
needs.generate-matrices.outputs.matrix != '{"include":[]}'
212191
&& contains(needs.*.result, 'failure')
213192
|| contains(needs.*.result, 'cancelled')
214-
215193
- uses: actions/checkout@v4
216194
if: inputs.run-cachix-deploy
217-
218195
- name: Deploy
219196
if: inputs.run-cachix-deploy
220197
env:
221198
CACHIX_CACHE: ${{ vars.CACHIX_CACHE }}
222199
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
223200
CACHIX_ACTIVATE_TOKEN: '${{ secrets.CACHIX_ACTIVATE_TOKEN }}'
224201
MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }}
225-
run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl deploy_spec
202+
run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${MCL_BRANCH}#mcl deploy_spec

‎packages/mcl/src/src/mcl/commands/shard_matrix.d‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ void saveShardMatrix(ShardMatrix matrix, Params params)
143143
const matrixJson = matrix.toJSON();
144144
const matrixString = matrixJson.toString();
145145
infof("Shard matrix: %s", matrixJson.toPrettyString);
146-
const envLine = "gen_matrix=" ~ matrixString;
146+
const envLine = matrixString;
147147
if (params.githubOutput != "")
148148
{
149149
params.githubOutput.append(envLine);

0 commit comments

Comments
 (0)