refactor(flake-checks-ci-matrix): Merge matrix generation jobs into a single one

AleksPopov4 · AleksPopov4 · commit 330b4f5c5465 · 2025-04-22T16:03:17.000+03:00
diff --git a/.github/workflows/reusable-flake-checks-ci-matrix.yml b/.github/workflows/reusable-flake-checks-ci-matrix.yml
@@ -38,9 +38,12 @@ jobs:
 
             This comment will be updated automatically with the status of each package.
 
-  generate-matrix-matrix:
-    name: Generate Matrix of Matrices
+  generate-matrices:
+    name: Generate Matrices
     runs-on: ${{ fromJSON(inputs.runner) }}
+    outputs:
+      matrix: ${{ steps.merge-matrix.outputs.matrix }}
+      fullMatrix: ${{ steps.merge-matrix.outputs.fullMatrix }}
     steps:
       - name: Install Nix
         uses: metacraft-labs/nixos-modules/.github/install-nix@main
@@ -54,122 +57,101 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Generate Matrix for Matrix
-        id: generate-matrix
+        id: generate-matrix-of-matrices
         env:
           CACHIX_CACHE: ${{ vars.CACHIX_CACHE }}
           CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
           MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }}
-        run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl shard_matrix
-    outputs:
-      gen_matrix: ${{ steps.generate-matrix.outputs.gen_matrix }}
-
-  generate-matrix:
-    needs: generate-matrix-matrix
-    runs-on: ${{ fromJSON(inputs.runner) }}
-    strategy:
-      matrix: ${{fromJSON(needs.generate-matrix-matrix.outputs.gen_matrix)}}
-    name: Generate Matrix ${{ matrix.digit != -1 && matrix.digit || '' }}
-    steps:
-      - name: Install Nix
-        uses: metacraft-labs/nixos-modules/.github/install-nix@main
-        with:
-          cachix-cache: ${{ vars.CACHIX_CACHE }}
-          cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
-          trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
-          substituters: ${{ vars.SUBSTITUTERS }}
-          nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
+        run: |
+          gen_matrix_output=$(nix run --accept-flake-config \
+            github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH}}#mcl shard_matrix)
 
-      - uses: actions/checkout@v4
+          echo "gen_matrix=$gen_matrix_output" >> "$GITHUB_OUTPUT"
 
       - name: Generate CI Matrix
         id: generate-matrix
-        shell: bash
         env:
           IS_INITIAL: 'true'
           CACHIX_CACHE: ${{ vars.CACHIX_CACHE }}
           CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
-          FLAKE_PRE: ${{ matrix.prefix }}
-          FLAKE_POST: ${{ matrix.postfix }}
           MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }}
-        run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl ci_matrix
+        run: |
+          echo "GEN_MATRIX_JSON is:"
+          echo "$gen_matrix_output"
 
-      - uses: actions/upload-artifact@v4
-        with:
-          name: matrix-pre${{ matrix.digit != -1 && format('-{0}', matrix.digit) || '' }}
-          path: matrix-pre.json
-    outputs:
-      matrix: ${{ steps.generate-matrix.outputs.matrix }}
+          final_matrix='[]'
 
-  slurp-matrix:
-    runs-on: ${{ fromJSON(inputs.runner) }}
-    needs: generate-matrix
-    name: Merge matrices
-    outputs:
-      matrix: ${{ steps.matrix.outputs.matrix }}
-      fullMatrix: ${{ steps.matrix.outputs.fullMatrix }}
-    steps:
-      - uses: actions/download-artifact@v4
-      - name: Merge matrices
-        run: |
-          # Check if jq is installed
-          if ! command -v jq &> /dev/null; then
-            PATH="$(nix build --print-out-paths 'nixpkgs#jq^bin')/bin:$PATH"
-            export PATH
-          fi
+          while read shard; do
+            prefix=$(jq -r '.prefix' <<<"$shard")
+            postfix=$(jq -r '.postfix' <<<"$shard")
+
+            echo "Processing shard -> prefix: $prefix, postfix: $postfix"
+
+            shard_output=$(PREFIX="$prefix" POSTFIX="$postfix" \
+              nix run --accept-flake-config \
+              github:metacraft-labs/nixos-modules/${MCL_BRANCH}#mcl ci_matrix)
 
-          ls */matrix-pre.json
-          matrix="$(cat */matrix-pre.json | jq -cr '.include[]' | jq '[ select (.isCached == false) ]' | jq -s 'add' | jq -c  '. | {include: .}')"
+            echo "Shard output: $shard_output"
 
-          if [[ "$matrix" == '' ]] || [[ "$matrix" == '{}' ]] || [[ "$matrix" == '{"include":null}' ]] || [[ "$matrix" == '{"include":[]}' ]]; then
+            final_matrix=$(jq --argjson shardData "$shard_output" \
+                              '. + [$shardData]' <<<"$final_matrix")
+          done < <(jq -c '.include[]' <<<"$gen_matrix_output")
+
+          echo "Final aggregated matrix: $final_matrix"
+
+      - name: Merge matrices
+        id: merge-matrix
+        run: |
+          set -eux
+          # If you need jq and it's not available on the runner:
+          # if ! command -v jq &> /dev/null; then
+          #   PATH="$(nix build --print-out-paths 'nixpkgs#jq^bin')/bin:$PATH"
+          #   export PATH
+          # fi
+
+          if [ -f matrix-pre.json ]; then
+            echo "Found matrix-pre.json. Merging..."
+            matrix=$(jq -cr '.include' matrix-pre.json | jq -c '{include: .}')
+          else
+            # Fallback if the file doesn't exist
             matrix='{"include":[]}'
           fi
 
           echo "---"
           echo "Matrix:"
           echo "$matrix" | jq
           echo "---"
-          echo
-          echo
-
-          fullMatrix="$(cat */matrix-pre.json | jq -cr '.include' | jq -s 'add' | jq -c '. | {include: .}')"
 
-          echo "---"
-          echo "Full Matrix:"
-          echo "$fullMatrix" | jq
-          echo "---"
+          # fullMatrix could be a superset of 'matrix'
+          # or exactly the same, depending on your logic
+          fullMatrix="$matrix"
 
-          echo "matrix=$matrix" >> $GITHUB_OUTPUT
-          echo "fullMatrix=$fullMatrix" >> $GITHUB_OUTPUT
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+          echo "fullMatrix=$fullMatrix" >> "$GITHUB_OUTPUT"
 
-        id: matrix
       - name: Post Comment
-        id: print-matrix
         uses: metacraft-labs/nixos-modules/.github/print-matrix@main
         with:
           is-initial: 'true'
           cachix-cache: ${{ vars.CACHIX_CACHE }}
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
           trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
           substituters: ${{ vars.SUBSTITUTERS }}
-          precalc_matrix: ${{ steps.matrix.outputs.fullMatrix }}
+          precalc_matrix: ${{ steps.merge-matrix.outputs.fullMatrix }}
           pr-comment-github-token: ${{ secrets.GITHUB_TOKEN }}
           nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
 
   build:
-    needs: slurp-matrix
-    if: needs.slurp-matrix.outputs.matrix != '{"include":[]}'
-
+    needs: generate-matrices
+    if: needs.generate-matrices.outputs.matrix != '{"include":[]}'
     strategy:
       fail-fast: false
-      matrix: ${{fromJSON( needs.slurp-matrix.outputs.matrix )}}
-
+      matrix: ${{ fromJSON(needs.generate-matrices.outputs.matrix) }}
     name: ${{ matrix.name }} | ${{ matrix.system }}
     runs-on: ${{ matrix.os }}
     continue-on-error: ${{ matrix.allowedToFail }}
-
     steps:
       - uses: actions/checkout@v4
-
       - name: Install Nix
         uses: metacraft-labs/nixos-modules/.github/install-nix@main
         with:
@@ -178,20 +160,18 @@ jobs:
           trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
           substituters: ${{ vars.SUBSTITUTERS }}
           nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
-
       - name: Build ${{ matrix.name }}
         run: |
           nix build -L --no-link --keep-going --show-trace \
             '.#${{ matrix.attrPath }}'
-
       - name: Push to Cachix ${{ matrix.name }}
         run: |
           cachix push ${{ vars.CACHIX_CACHE }} ${{ matrix.output }}
 
   results:
+    needs: [build, generate-matrices]
     runs-on: ${{ fromJSON(inputs.runner) }}
     name: Final Results
-    needs: [build, slurp-matrix]
     if: always()
     steps:
       - name: Post Comment
@@ -202,24 +182,21 @@ jobs:
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
           trusted-public-keys: ${{ vars.TRUSTED_PUBLIC_KEYS }}
           substituters: ${{ vars.SUBSTITUTERS }}
-          precalc_matrix: ${{ needs.slurp-matrix.outputs.fullMatrix }}
+          precalc_matrix: ${{ needs.generate-matrices.outputs.fullMatrix }}
           pr-comment-github-token: ${{ secrets.GITHUB_TOKEN }}
           nix-github-token: ${{ secrets.NIX_GITHUB_TOKEN }}
-
       - run: exit 1
         if: >-
-          needs.slurp-matrix.outputs.matrix != '{"include":[]}'
+          needs.generate-matrices.outputs.matrix != '{"include":[]}'
             && contains(needs.*.result, 'failure')
             || contains(needs.*.result, 'cancelled')
-
       - uses: actions/checkout@v4
         if: inputs.run-cachix-deploy
-
       - name: Deploy
         if: inputs.run-cachix-deploy
         env:
           CACHIX_CACHE: ${{ vars.CACHIX_CACHE }}
           CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
           CACHIX_ACTIVATE_TOKEN: '${{ secrets.CACHIX_ACTIVATE_TOKEN }}'
           MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }}
-        run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl deploy_spec
+        run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${MCL_BRANCH}#mcl deploy_spec
diff --git a/packages/mcl/src/src/mcl/commands/shard_matrix.d b/packages/mcl/src/src/mcl/commands/shard_matrix.d
@@ -143,7 +143,7 @@ void saveShardMatrix(ShardMatrix matrix, Params params)
     const matrixJson = matrix.toJSON();
     const matrixString = matrixJson.toString();
     infof("Shard matrix: %s", matrixJson.toPrettyString);
-    const envLine = "gen_matrix=" ~ matrixString;
+    const envLine = matrixString;
     if (params.githubOutput != "")
     {
         params.githubOutput.append(envLine);

Original file line number	Diff line number	Diff line change
`@@ -143,7 +143,7 @@ void saveShardMatrix(ShardMatrix matrix, Params params)`
`143`	`143`	`const matrixJson = matrix.toJSON();`
`144`	`144`	`const matrixString = matrixJson.toString();`
`145`	`145`	`infof("Shard matrix: %s", matrixJson.toPrettyString);`
`146`		`- const envLine = "gen_matrix=" ~ matrixString;`
	`146`	`+ const envLine = matrixString;`
`147`	`147`	`if (params.githubOutput != "")`
`148`	`148`	`{`
`149`	`149`	`params.githubOutput.append(envLine);`