Speedup CI by splitting binary builds for each architecture

Author: sandhose

.github/workflows/build.yaml

@@ -52,6 +52,55 @@ jobs:
           echo "describe=$(git describe --tags --match 'v*.*.*' --always)" >> $GITHUB_OUTPUT
           echo "timestamp=$(git log -1 --format=%ct)" >> $GITHUB_OUTPUT
 
+  build-assets:
+    name: Build assets
+    runs-on: ubuntu-22.04
+
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout the code
+        uses: actions/[email protected]
+
+      - name: Setup OPA
+        uses: open-policy-agent/[email protected]
+        with:
+          version: 0.64.1
+
+      - name: Install frontend Node
+        uses: actions/[email protected]
+        with:
+          node-version: 20
+
+      - name: Install frontend Node dependencies
+        working-directory: ./frontend
+        run: npm ci
+
+      - name: Build frontend
+        working-directory: ./frontend
+        run: npm run build
+
+      - name: Build policies
+        working-directory: ./policies
+        run: make
+
+      - name: Prepare assets artifact
+        run: |
+          mkdir -p assets-dist/share
+          cp policies/policy.wasm assets-dist/share/policy.wasm
+          cp frontend/dist/manifest.json assets-dist/share/manifest.json
+          cp -r frontend/dist/ assets-dist/share/assets
+          cp -r templates/ assets-dist/share/templates
+          cp -r translations/ assets-dist/share/translations
+          cp LICENSE assets-dist/LICENSE
+          chmod -R u=rwX,go=rX assets-dist/
+
+      - name: Upload assets
+        uses: actions/[email protected]
+        with:
+          name: assets
+          path: assets-dist
 
   build-binaries:
     name: Build binaries
@@ -60,6 +109,12 @@ jobs:
     needs:
       - compute-version
 
+    strategy:
+      matrix:
+        include:
+          - target: x86_64-unknown-linux-gnu
+          - target: aarch64-unknown-linux-gnu
+
     env:
       VERGEN_GIT_DESCRIBE: ${{ needs.compute-version.outputs.describe }}
       SOURCE_DATE_EPOCH: ${{ needs.compute-version.outputs.timestamp }}
@@ -71,17 +126,11 @@ jobs:
       - name: Checkout the code
         uses: actions/[email protected]
 
-      - name: Setup OPA
-        uses: open-policy-agent/[email protected]
-        with:
-          version: 0.64.1
-
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
         with:
           targets: |
-            x86_64-unknown-linux-gnu
-            aarch64-unknown-linux-gnu
+            ${{ matrix.target }}
 
       - name: Setup sccache
         uses: mozilla-actions/[email protected]
@@ -96,52 +145,60 @@ jobs:
         with:
           tool: cargo-zigbuild
 
-      - name: Install frontend Node
-        uses: actions/[email protected]
-        with:
-          node-version: 20
-
-      - name: Install frontend Node dependencies
-        working-directory: ./frontend
-        run: npm ci
-
-      - name: Build frontend
-        working-directory: ./frontend
-        run: npm run build
-
-      - name: Build policies
-        working-directory: ./policies
-        run: make
-
       - name: Build the binary
         run: |
           cargo zigbuild \
             --release \
-            --target x86_64-unknown-linux-gnu.2.17 \
-            --target aarch64-unknown-linux-gnu.2.17 \
+            --target ${{ matrix.target }}.2.17 \
             --no-default-features \
             --features dist \
             -p mas-cli
 
-      - name: Create one archive per architecture
+      - name: Upload binary artifact
+        uses: actions/[email protected]
+        with:
+          name: binary-${{ matrix.target }}
+          path: target/${{ matrix.target }}/release/mas-cli
+
+  assemble-archives:
+    name: Assemble release archives
+    runs-on: ubuntu-22.04
+
+    needs:
+      - build-assets
+      - build-binaries
+
+    permissions:
+      contents: read
+
+    steps:
+      - name: Download assets
+        uses: actions/[email protected]
+        with:
+          name: assets
+          path: assets-dist
+
+      - name: Download binary x86_64
+        uses: actions/[email protected]
+        with:
+          name: binary-x86_64-unknown-linux-gnu
+          path: binary-x86_64
+
+      - name: Download binary aarch64
+        uses: actions/[email protected]
+        with:
+          name: binary-aarch64-unknown-linux-gnu
+          path: binary-aarch64
+
+      - name: Create final archives
         run: |
           for arch in x86_64 aarch64; do
-            # Create one directory per architecture
-            mkdir -p dist/${arch}/share/
-            # Copy the artifacts to the right place
-            cp policies/policy.wasm dist/${arch}/share/policy.wasm
-            cp frontend/dist/manifest.json dist/${arch}/share/manifest.json
-            cp -r frontend/dist/ dist/${arch}/share/assets
-            cp -r templates/ dist/${arch}/share/templates
-            cp -r translations/ dist/${arch}/share/translations
-            cp LICENSE dist/${arch}/LICENSE
+            mkdir -p dist/${arch}/share
+            cp -r assets-dist/share/* dist/${arch}/share/
+            cp assets-dist/LICENSE dist/${arch}/LICENSE
+            cp binary-$arch/mas-cli dist/${arch}/mas-cli
             chmod -R u=rwX,go=rX dist/${arch}/
-
-            # Copy the binary to the right place
-            cp target/${arch}-unknown-linux-gnu/release/mas-cli dist/${arch}/
             chmod u=rwx,go=rx dist/${arch}/mas-cli
-
-            # Create the archive
             tar -czvf mas-cli-${arch}-linux.tar.gz --owner=0 --group=0 -C dist/${arch}/ .
           done
 
@@ -257,209 +314,3 @@ jobs:
         if: github.event_name != 'pull_request'
         with:
           files: |
-            ./docker-bake.hcl
-            cwd://${{ steps.meta.outputs.bake-file }}
-            cwd://${{ steps.meta-debug.outputs.bake-file }}
-            cwd://${{ steps.meta-syn2mas.outputs.bake-file }}
-          set: |
-            base.output=type=image,push=true
-            base.cache-from=type=registry,ref=${{ env.BUILDCACHE }}:buildcache
-            base.cache-to=type=registry,ref=${{ env.BUILDCACHE }}:buildcache,mode=max
-
-      - name: Transform bake output
-        # This transforms the ouput to an object which looks like this:
-        # { reguar: { digest: "…", tags: ["…", "…"] }, debug: { digest: "…", tags: ["…"] }, … }
-        id: output
-        if: github.event_name != 'pull_request'
-        run: |
-          echo 'metadata<<EOF' >> $GITHUB_OUTPUT
-          echo '${{ steps.bake.outputs.metadata }}' | jq -c 'with_entries(select(.value | (type == "object" and has("containerimage.digest")))) | map_values({ digest: .["containerimage.digest"], tags: (.["image.name"] | split(",")) })' >> $GITHUB_OUTPUT
-          echo 'EOF' >> $GITHUB_OUTPUT
-
-      - name: Sign the images with GitHub Actions provided token
-        # Only sign on tags and on commits on main branch
-        if: |
-          github.event_name != 'pull_request'
-          && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
-
-        env:
-          REGULAR_DIGEST: ${{ steps.output.outputs.metadata && fromJSON(steps.output.outputs.metadata).regular.digest }}
-          DEBUG_DIGEST: ${{ steps.output.outputs.metadata && fromJSON(steps.output.outputs.metadata).debug.digest }}
-          SYN2MAS_DIGEST: ${{ steps.output.outputs.metadata && fromJSON(steps.output.outputs.metadata).syn2mas.digest }}
-
-        run: |-
-          cosign sign --yes \
-            "$IMAGE@$REGULAR_DIGEST" \
-            "$IMAGE@$DEBUG_DIGEST" \
-            "$IMAGE_SYN2MAS@$SYN2MAS_DIGEST"
-
-  syn2mas:
-    name: Release syn2mas on NPM
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-      id-token: write
-
-    steps:
-      - name: Checkout the code
-        uses: actions/[email protected]
-
-      - name: Install Node
-        uses: actions/[email protected]
-        with:
-          node-version-file: ./tools/syn2mas/.nvmrc
-
-      - name: Install Node dependencies
-        working-directory: ./tools/syn2mas
-        run: npm ci
-
-      - name: Publish
-        uses: JS-DevTools/npm-publish@v3
-        with:
-          package: ./tools/syn2mas
-          token: ${{ secrets.NPM_TOKEN }}
-          provenance: true
-          dry-run: ${{ !startsWith(github.ref, 'refs/tags/') }}
-
-  release:
-    name: Release
-    if: startsWith(github.ref, 'refs/tags/')
-    runs-on: ubuntu-latest
-    needs:
-      - build-binaries
-      - build-image
-      - syn2mas
-    steps:
-      - name: Download the artifacts from the previous job
-        uses: actions/download-artifact@v4
-        with:
-          name: binaries
-          path: artifacts
-
-      - name: Prepare a release
-        uses: softprops/action-gh-release@v2
-        with:
-          generate_release_notes: true
-          body: |
-            ### Docker image
-
-            Regular image:
-
-              - Digest:
-                ```
-                ${{ env.IMAGE }}@${{ fromJSON(needs.build-image.outputs.metadata).regular.digest }}
-                ```
-              - Tags:
-                ```
-                ${{ join(fromJSON(needs.build-image.outputs.metadata).regular.tags, '
-                ') }}
-                ```
-
-            Debug variant:
-
-              - Digest:
-                ```
-                ${{ env.IMAGE }}@${{ fromJSON(needs.build-image.outputs.metadata).debug.digest }}
-                ```
-              - Tags:
-                ```
-                ${{ join(fromJSON(needs.build-image.outputs.metadata).debug.tags, '
-                ') }}
-                ```
-
-            `syn2mas` migration tool:
-
-              - Digest:
-                ```
-                ${{ env.IMAGE_SYN2MAS }}@${{ fromJSON(needs.build-image.outputs.metadata).syn2mas.digest }}
-                ```
-              - Tags:
-                ```
-                ${{ join(fromJSON(needs.build-image.outputs.metadata).syn2mas.tags, '
-                ') }}
-                ```
-
-          files: |
-            artifacts/mas-cli-aarch64-linux.tar.gz
-            artifacts/mas-cli-x86_64-linux.tar.gz
-          draft: true
-
-  unstable:
-    name: Update the unstable release
-    runs-on: ubuntu-24.04
-    needs:
-      - build-binaries
-      - build-image
-    if: github.ref == 'refs/heads/main'
-
-    permissions:
-      contents: write
-
-    steps:
-      - name: Download the artifacts from the previous job
-        uses: actions/download-artifact@v4
-        with:
-          name: binaries
-          path: artifacts
-
-      - name: Update unstable git tag
-        uses: actions/[email protected]
-        with:
-          script: |
-            const [owner, repo] = process.env.GITHUB_REPOSITORY.split("/");
-            const sha = process.env.GITHUB_SHA;
-
-            const tag = await github.rest.git.updateRef({
-              owner,
-              repo,
-              force: true,
-              ref: 'tags/unstable',
-              sha,
-            });
-            console.log("Updated tag ref:", tag.data.url);
-
-      - name: Update unstable release
-        uses: softprops/action-gh-release@v2
-        with:
-          name: 'Unstable build'
-          tag_name: unstable
-          body: |
-            This is an automatically updated unstable release containing the latest builds from the main branch.
-
-            **⚠️ Warning: These are development builds and may be unstable.**
-
-            Last updated: ${{ github.event.head_commit.timestamp }}
-            Commit: ${{ github.sha }}
-
-            ### Docker image
-
-            Regular image:
-
-              - Digest:
-                ```
-                ${{ env.IMAGE }}@${{ fromJSON(needs.build-image.outputs.metadata).regular.digest }}
-                ```
-              - Tags:
-                ```
-                ${{ join(fromJSON(needs.build-image.outputs.metadata).regular.tags, '
-                ') }}
-                ```
-
-            Debug variant:
-
-              - Digest:
-                ```
-                ${{ env.IMAGE }}@${{ fromJSON(needs.build-image.outputs.metadata).debug.digest }}
-                ```
-              - Tags:
-                ```
-                ${{ join(fromJSON(needs.build-image.outputs.metadata).debug.tags, '
-                ') }}
-                ```
-
-          files: |
-            artifacts/mas-cli-aarch64-linux.tar.gz
-            artifacts/mas-cli-x86_64-linux.tar.gz
-          prerelease: true
-          make_latest: false