Separate builds with native runners

victorlin · victorlin · commit 9b4087eb1add · 2026-01-06T12:56:10.000-08:00
Instead of using one long-running job to build 6 images (3 stages x 2
platforms) and 3 manifest lists, use 2 platform-specific jobs to build 3
images (one per stage) and a third job to build the manifest lists.

This change comes with two functional improvements:

1. Most significantly, per-runner disk space usage and build times go
   down. This is due to both parallelizing the build per platform and
   removing emulation for the linux/arm64 build. The new time bottleneck
   for the workflow is now the linux/arm64 test job, which allows for an
   easy follow-up improvement (see added comment).

2. As a byproduct of the platform-specific images being tagged, they can
   now be retrieved from GitHub REST API results, removing the need to
   fetch from GHCR's undocumented Docker Registry API.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -37,11 +37,22 @@ jobs:
       cache-date: ${{ env.CACHE_DATE }}
       tag: ${{ env.TAG }}
 
-  # Build multi-platform builder and final images with caching from Docker Hub
-  # and GitHub Container Registry; push to GitHub Container Registry.
+  # Build platform-specific builder and final images with caching from Docker
+  # Hub and GitHub Container Registry; push to GitHub Container Registry.
   build:
+    name: build (${{ matrix.platform }})
     needs: vars
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+            arch: amd64
+            runner: ubuntu-latest
+
+          - platform: linux/arm64
+            arch: arm64
+            runner: ubuntu-24.04-arm
     steps:
 
     - uses: actions/checkout@v6
@@ -50,8 +61,6 @@ jobs:
       with:
         python-version: '>=3.8'
 
-    - uses: docker/setup-qemu-action@v3
-
     # GITHUB_TOKEN is unreliable¹ so use a token from nextstrain-bot.
     # ¹ https://github.com/docker/build-push-action/issues/463#issuecomment-939394233
     - uses: docker/login-action@v3
@@ -60,7 +69,12 @@ jobs:
         username: nextstrain-bot
         password: ${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_MANAGE_PACKAGES }}
 
-    - run: ./devel/build -p linux/amd64,linux/arm64 -r ghcr.io -t "$TAG" -l logs/
+    - run: |
+        ./devel/build \
+          -p ${{ matrix.platform }} \
+          -r ghcr.io \
+          -t "$TAG-${{ matrix.arch }}" \
+          -l logs-${{ matrix.arch }}/
       env:
         TAG: ${{ needs.vars.outputs.tag }}
         CACHE_DATE: ${{ needs.vars.outputs.cache-date }}
@@ -69,15 +83,15 @@ jobs:
       name: Upload build logs as artifacts
       uses: actions/upload-artifact@v6
       with:
-        name: build-logs
-        path: logs/
+        name: build-logs-${{ matrix.arch }}
+        path: logs-${{ matrix.arch }}/
 
     - if: always()
       name: Summarize build logs for the GitHub Actions run summary page
       run: |
-        for log in logs/*; do
+        for log in logs-${{ matrix.arch }}/*; do
         {
-          echo "## $(basename "$log")"
+          echo "## [${{ matrix.platform }}] $(basename "$log")"
           echo ''
           echo '```'
           ./devel/summarize-buildkit-output "$log" 2>&1
@@ -86,16 +100,55 @@ jobs:
         } >> "$GITHUB_STEP_SUMMARY"
         done
 
+  # Merge platform-specific images into multi-platform images
+  merge-builds:
+    needs: [vars, build]
+    runs-on: ubuntu-latest
+    env:
+      TAG: ${{ needs.vars.outputs.tag }}
+    steps:
+    - uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: nextstrain-bot
+        password: ${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_MANAGE_PACKAGES }}
+
+    - name: Create base-builder-build-platform image
+      run: |
+        docker buildx imagetools create \
+          -t "ghcr.io/nextstrain/base-builder-build-platform:$TAG" \
+             "ghcr.io/nextstrain/base-builder-build-platform:$TAG-amd64" \
+             "ghcr.io/nextstrain/base-builder-build-platform:$TAG-arm64"
+
+    - name: Create base-builder-target-platform image
+      run: |
+        docker buildx imagetools create \
+          -t "ghcr.io/nextstrain/base-builder-target-platform:$TAG" \
+             "ghcr.io/nextstrain/base-builder-target-platform:$TAG-amd64" \
+             "ghcr.io/nextstrain/base-builder-target-platform:$TAG-arm64"
+
+    - name: Create base image
+      run: |
+        docker buildx imagetools create \
+          -t "ghcr.io/nextstrain/base:$TAG" \
+             "ghcr.io/nextstrain/base:$TAG-amd64" \
+             "ghcr.io/nextstrain/base:$TAG-arm64"
+
   # Run tests with the final image from GitHub Container Registry.
   test:
     name: test (${{ matrix.platform }})
-    needs: [vars, build]
-    runs-on: ubuntu-latest
+    needs: [vars, merge-builds]
+    runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        platform:
-          - linux/amd64
-          - linux/arm64
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            # TODO: use ubuntu-24.04-arm and remove emulation when Nextstrain
+            # CLI can be installed natively on aarch64.¹
+            # ¹ <https://github.com/nextstrain/cli/pull/490>
+            runner: ubuntu-latest
     permissions:
       contents: read
       id-token: write
@@ -116,9 +169,7 @@ jobs:
       with:
         python-version: ~3
 
-    # The ubuntu-latest runner is linux/amd64 so anything else will
-    # run with emulation, which is still better than nothing.
-    - if: matrix.platform != 'linux/amd64'
+    - if: matrix.platform == 'linux/arm64'
       uses: docker/setup-qemu-action@v3
 
     - uses: actions/checkout@v6
@@ -141,7 +192,7 @@ jobs:
 
   validate-platforms:
     name: Validate platforms
-    needs: [vars, build]
+    needs: [vars, merge-builds]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v6
@@ -161,7 +212,7 @@ jobs:
   # Docker Hub, where they will persist. Do this regardless of test results.
   push-branch:
     if: startsWith(needs.vars.outputs.tag, 'branch-') && github.event_name != 'pull_request'
-    needs: [vars, build]
+    needs: [vars, merge-builds]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v6
@@ -185,7 +236,7 @@ jobs:
   # Docker Hub, where they will persist. Only do this if tests pass.
   push-build:
     if: startsWith(needs.vars.outputs.tag, 'build-')
-    needs: [vars, build, test, validate-platforms]
+    needs: [vars, merge-builds, test, validate-platforms]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v6
@@ -283,7 +334,7 @@ jobs:
   # Delete the builder and final images from GitHub Container Registry.
   cleanup-registry:
     if: always()
-    needs: [vars, build, test, validate-platforms, push-branch, push-build]
+    needs: [vars, merge-builds, test, validate-platforms, push-branch, push-build]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v6
@@ -294,5 +345,4 @@ jobs:
         script: |
           const script = require('./devel/delete-from-ghcr.js');
           const tag = "${{ needs.vars.outputs.tag }}";
-          const token = "${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_MANAGE_PACKAGES }}";
-          script({fetch, octokit: github, tag, token});
+          script({octokit: github, tag});
diff --git a/devel/delete-from-ghcr.js b/devel/delete-from-ghcr.js
@@ -7,7 +7,7 @@
 // ¹ https://github.com/octokit/core.js#authentication
 // ² https://github.com/settings/tokens/new?scopes=delete:packages
 
-module.exports = async ({fetch, octokit, tag, token}) => {
+module.exports = async ({octokit, tag}) => {
   org = 'nextstrain';
   packages = [
     'base',
@@ -35,44 +35,39 @@ module.exports = async ({fetch, octokit, tag, token}) => {
       continue;
     }
 
-    // The manifest list + one manifest per platform are pushed from the build.
-    // Only the manifest list is tagged for direct removal via GitHub's REST API.
-
-    // The GitHub REST API does not provide a way to retrieve manifest digests
-    // from the manifest list, so use GHCR's (undocumented) Docker Registry API
-    // to do that.
-    // This works when a GitHub token with the right permissions is passed in
-    // base64 form as a Bearer token¹.
-    // ¹ https://github.com/orgs/community/discussions/26279#discussioncomment-3251172
-    const res = await fetch(`https://ghcr.io/v2/${org}/${packageName}/manifests/${tag}`,
-      {
-        headers: {
-          Accept: "application/vnd.docker.distribution.manifest.list.v2+json",
-          Authorization: `Bearer ${btoa(token)}`
-        }
-      });
-    const resData = await res.json();
-    const manifestDigests = resData.manifests.map(manifest => manifest.digest);
+    // Delete platform-specific images first. These are created during the split
+    // build process and should be cleaned up even if the main manifest merge
+    // fails.
+    const platformTags = [`${tag}-amd64`, `${tag}-arm64`];
+    for (const platformTag of platformTags) {
+      const platformVersionsWithTag = packageVersions.filter(version => version.metadata.container.tags.includes(platformTag));
+
+      if (platformVersionsWithTag.length == 0) {
+        console.log(`${org}/${packageName}:${platformTag} was not found (may have already been deleted).`);
+        continue;
+      }
+
+      const platformVersionId = platformVersionsWithTag[0].id;
+      console.log(`Version for ${org}/${packageName}:${platformTag} is ${platformVersionId}.`);
 
-    const versions = packageVersions.filter(version => manifestDigests.includes(version.name));
-    for (const version of versions) {
-      console.log(`Deleting the package version for ${org}/${packageName}:${version.name} ...`);
+      console.log(`Deleting the package version for ${org}/${packageName}:${platformTag} ...`);
       try {
         await octokit.request('DELETE /orgs/{org}/packages/{package_type}/{package_name}/versions/{package_version_id}', {
           org: org,
           package_type: 'container',
           package_name: packageName,
-          package_version_id: version.id,
+          package_version_id: platformVersionId,
         });
         console.log("Done.");
       } catch (deleteVersionError) {
         console.log(deleteVersionError);
+        console.error(`Could not delete ${org}/${packageName}:${platformTag}.`);
         errorEncountered = true;
         continue;
       }
     }
 
-    // Delete the manifest list after deleting individual manifests.
+    // Delete the merged image after deleting platform-specific images.
 
     const versionsWithTag = packageVersions.filter(version => version.metadata.container.tags.includes(tag));
 
