Skip to content

Commit 60ae9c0

Browse files
AleksPopov4AleksPopov4
committed
refactor(flake-checks-ci-matrix): Merge matrix generation jobs into a single one
1 parent e15b065 commit 60ae9c0

File tree

2 files changed

+56
-77
lines changed

2 files changed

+56
-77
lines changed

‎.github/workflows/reusable-flake-checks-ci-matrix.yml‎

Lines changed: 52 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,12 @@ jobs:
3838
3939
This comment will be updated automatically with the status of each package.
4040
41-
generate-matrix-matrix:
42-
name: Generate Matrix of Matrices
41+
generate-matrices:
42+
name: Generate Matrices
4343
runs-on: ${{ fromJSON(inputs.runner) }}
44+
outputs:
45+
matrix: ${{ steps.merge-matrix.outputs.matrix }}
46+
fullMatrix: ${{ steps.merge-matrix.outputs.fullMatrix }}
4447
steps:
4548
- name: Install Nix
4649
uses: metacraft-labs/nixos-modules/.github/install-nix@main
@@ -54,122 +57,100 @@ jobs:
5457
- uses: actions/checkout@v4
5558

5659
- name: Generate Matrix for Matrix
57-
id: generate-matrix
60+
id: generate-matrix-of-matrices
5861
env:
5962
CACHIX_CACHE: ${{ vars.CACHIX_CACHE }}
6063
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
6164
MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }}
62-
run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl shard_matrix
63-
outputs:
64-
gen_matrix: ${{ steps.generate-matrix.outputs.gen_matrix }}
65-
66-
generate-matrix:
67-
needs: generate-matrix-matrix
68-
runs-on: ${{ fromJSON(inputs.runner) }}
69-
strategy:
70-
matrix: ${{fromJSON(needs.generate-matrix-matrix.outputs.gen_matrix)}}
71-
name: Generate Matrix ${{ matrix.digit != -1 && matrix.digit || '' }}
72-
steps:
73-
- name: Install Nix
74-
uses: metacraft-labs/nixos-modules/.github/install-nix@main
75-
with:
76-
cachix-cache: ${{ vars.CACHIX_CACHE }}
77-
cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
78-
trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
79-
substituters: ${{ vars.SUBSTITUTERS }}
80-
nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
81-
82-
- uses: actions/checkout@v4
65+
run: |
66+
nix run --accept-flake-config \
67+
github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH}}#mcl shard_matrix
8368
8469
- name: Generate CI Matrix
8570
id: generate-matrix
86-
shell: bash
8771
env:
8872
IS_INITIAL: 'true'
8973
CACHIX_CACHE: ${{ vars.CACHIX_CACHE }}
9074
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
91-
FLAKE_PRE: ${{ matrix.prefix }}
92-
FLAKE_POST: ${{ matrix.postfix }}
9375
MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }}
94-
run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl ci_matrix
76+
run: |
77+
matrix_json= ${{ steps.generate-matrix-of-matrices.outputs.gen_matrix }}
9578
96-
- uses: actions/upload-artifact@v4
97-
with:
98-
name: matrix-pre${{ matrix.digit != -1 && format('-{0}', matrix.digit) || '' }}
99-
path: matrix-pre.json
100-
outputs:
101-
matrix: ${{ steps.generate-matrix.outputs.matrix }}
79+
echo "GEN_MATRIX_JSON is:"
80+
echo ${{ steps.generate-matrix-of-matrices.outputs.gen_matrix }}
81+
82+
final_matrix='[]'
83+
84+
while read shard; do
85+
prefix=$(jq -r '.prefix' <<<"$shard")
86+
postfix=$(jq -r '.postfix' <<<"$shard")
87+
88+
echo "Processing shard -> prefix: $prefix, postfix: $postfix"
89+
90+
shard_output=$(PREFIX="$prefix" POSTFIX="$postfix" \
91+
nix run --accept-flake-config \
92+
github:metacraft-labs/nixos-modules/${MCL_BRANCH}#mcl ci_matrix)
93+
94+
echo "Shard output: $shard_output"
95+
96+
final_matrix=$(jq --argjson shardData "$shard_output" \
97+
'. + [$shardData]' <<<"$final_matrix")
98+
done < <(jq -c '.include[]' <<< "${matrix_json:-[]}")
99+
100+
echo "Final aggregated matrix: $final_matrix"
102101
103-
slurp-matrix:
104-
runs-on: ${{ fromJSON(inputs.runner) }}
105-
needs: generate-matrix
106-
name: Merge matrices
107-
outputs:
108-
matrix: ${{ steps.matrix.outputs.matrix }}
109-
fullMatrix: ${{ steps.matrix.outputs.fullMatrix }}
110-
steps:
111-
- uses: actions/download-artifact@v4
112102
- name: Merge matrices
103+
id: merge-matrix
113104
run: |
114105
# Check if jq is installed
115106
if ! command -v jq &> /dev/null; then
116107
PATH="$(nix build --print-out-paths 'nixpkgs#jq^bin')/bin:$PATH"
117108
export PATH
118109
fi
119110
120-
ls */matrix-pre.json
121-
matrix="$(cat */matrix-pre.json | jq -cr '.include[]' | jq '[ select (.isCached == false) ]' | jq -s 'add' | jq -c '. | {include: .}')"
122-
123-
if [[ "$matrix" == '' ]] || [[ "$matrix" == '{}' ]] || [[ "$matrix" == '{"include":null}' ]] || [[ "$matrix" == '{"include":[]}' ]]; then
111+
if [ -f matrix-pre.json ]; then
112+
echo "Found matrix-pre.json. Merging..."
113+
matrix=$(jq -cr '.include' matrix-pre.json | jq -c '{include: .}')
114+
else
115+
# Fallback if the file doesn't exist
124116
matrix='{"include":[]}'
125117
fi
126118
127119
echo "---"
128120
echo "Matrix:"
129121
echo "$matrix" | jq
130122
echo "---"
131-
echo
132-
echo
133-
134-
fullMatrix="$(cat */matrix-pre.json | jq -cr '.include' | jq -s 'add' | jq -c '. | {include: .}')"
135123
136-
echo "---"
137-
echo "Full Matrix:"
138-
echo "$fullMatrix" | jq
139-
echo "---"
124+
# fullMatrix could be a superset of 'matrix'
125+
# or exactly the same, depending on your logic
126+
fullMatrix="$matrix"
140127
141-
echo "matrix=$matrix" >> $GITHUB_OUTPUT
142-
echo "fullMatrix=$fullMatrix" >> $GITHUB_OUTPUT
128+
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
129+
echo "fullMatrix=$fullMatrix" >> "$GITHUB_OUTPUT"
143130
144-
id: matrix
145131
- name: Post Comment
146-
id: print-matrix
147132
uses: metacraft-labs/nixos-modules/.github/print-matrix@main
148133
with:
149134
is-initial: 'true'
150135
cachix-cache: ${{ vars.CACHIX_CACHE }}
151136
cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
152137
trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
153138
substituters: ${{ vars.SUBSTITUTERS }}
154-
precalc_matrix: ${{ steps.matrix.outputs.fullMatrix }}
139+
precalc_matrix: ${{ steps.merge-matrix.outputs.fullMatrix }}
155140
pr-comment-github-token: ${{ secrets.GITHUB_TOKEN }}
156141
nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
157142

158143
build:
159-
needs: slurp-matrix
160-
if: needs.slurp-matrix.outputs.matrix != '{"include":[]}'
161-
144+
needs: generate-matrices
145+
if: needs.generate-matrices.outputs.matrix != '{"include":[]}'
162146
strategy:
163147
fail-fast: false
164-
matrix: ${{fromJSON( needs.slurp-matrix.outputs.matrix )}}
165-
148+
matrix: ${{ fromJSON(needs.generate-matrices.outputs.matrix) }}
166149
name: ${{ matrix.name }} | ${{ matrix.system }}
167150
runs-on: ${{ matrix.os }}
168151
continue-on-error: ${{ matrix.allowedToFail }}
169-
170152
steps:
171153
- uses: actions/checkout@v4
172-
173154
- name: Install Nix
174155
uses: metacraft-labs/nixos-modules/.github/install-nix@main
175156
with:
@@ -178,20 +159,18 @@ jobs:
178159
trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
179160
substituters: ${{ vars.SUBSTITUTERS }}
180161
nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
181-
182162
- name: Build ${{ matrix.name }}
183163
run: |
184164
nix build -L --no-link --keep-going --show-trace \
185165
'.#${{ matrix.attrPath }}'
186-
187166
- name: Push to Cachix ${{ matrix.name }}
188167
run: |
189168
cachix push ${{ vars.CACHIX_CACHE }} ${{ matrix.output }}
190169
191170
results:
171+
needs: [build, generate-matrices]
192172
runs-on: ${{ fromJSON(inputs.runner) }}
193173
name: Final Results
194-
needs: [build, slurp-matrix]
195174
if: always()
196175
steps:
197176
- name: Post Comment
@@ -202,24 +181,21 @@ jobs:
202181
cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
203182
trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
204183
substituters: ${{ vars.SUBSTITUTERS }}
205-
precalc_matrix: ${{ needs.slurp-matrix.outputs.fullMatrix }}
184+
precalc_matrix: ${{ needs.generate-matrices.outputs.fullMatrix }}
206185
pr-comment-github-token: ${{ secrets.GITHUB_TOKEN }}
207186
nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
208-
209187
- run: exit 1
210188
if: >-
211-
needs.slurp-matrix.outputs.matrix != '{"include":[]}'
189+
needs.generate-matrices.outputs.matrix != '{"include":[]}'
212190
&& contains(needs.*.result, 'failure')
213191
|| contains(needs.*.result, 'cancelled')
214-
215192
- uses: actions/checkout@v4
216193
if: inputs.run-cachix-deploy
217-
218194
- name: Deploy
219195
if: inputs.run-cachix-deploy
220196
env:
221197
CACHIX_CACHE: ${{ vars.CACHIX_CACHE }}
222198
CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
223199
CACHIX_ACTIVATE_TOKEN: '${{ secrets.CACHIX_ACTIVATE_TOKEN }}'
224200
MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }}
225-
run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl deploy_spec
201+
run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${MCL_BRANCH}#mcl deploy_spec

‎packages/mcl/src/src/mcl/commands/shard_matrix.d‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,10 +143,13 @@ void saveShardMatrix(ShardMatrix matrix, Params params)
143143
const matrixJson = matrix.toJSON();
144144
const matrixString = matrixJson.toString();
145145
infof("Shard matrix: %s", matrixJson.toPrettyString);
146-
const envLine = "gen_matrix=" ~ matrixString;
146+
const envLine = matrixString;
147+
infof("githubOutput: '%s'", params.githubOutput);
148+
infof("envLine: '%s'", envLine);
147149
if (params.githubOutput != "")
148150
{
149151
params.githubOutput.append(envLine);
152+
infof("githubOutput: '%s'", params.githubOutput);
150153
}
151154
else
152155
{

0 commit comments

Comments
 (0)