ci: Improve docker cache and test that the image works (#349)

Author: sdn4z

.github/actions/docker-build/action.yml

@@ -1,16 +1,21 @@
 # Action that wraps together all the necessary actions to build an image and publish it.
 # It does not include login actions.
 # This is a convenience, so that we know that everything related to the build lies in this file,
-# and if it gets updated we can automatically run our `test-docker-build` from `build-test.yml.`
+# and if it gets updated we can automatically run our `test-docker-build` from `build-test.yml.`,
+# so there is no confusion about which is the job that was updated, if there are many in the same workflow file.
 
 name: "Docker Build"
 description: "Builds Docker images with optional caching and multi-platform support"
 
 inputs:
-  push-image:
+  push:
     description: "Whether to push the built image"
     required: false
     default: "false"
+  load:
+    description: "Whether to load the built image"
+    required: false
+    default: "false"
   platforms:
     description: "Target platforms for the build"
     required: false
@@ -55,9 +60,10 @@ runs:
       with:
         context: ${{ inputs.context }}
         file: ${{ inputs.dockerfile }}
-        push: ${{ inputs['push-image'] }}
+        push: ${{ inputs.push }}
         platforms: ${{ inputs.platforms }}
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         cache-from: ${{ inputs['cache-from'] }}
         cache-to: ${{ inputs['cache-to'] }}
+        load: ${{ inputs.load }}

.github/workflows/build-test.yml

@@ -7,7 +7,7 @@ on:
     branches: ["main"]
   pull_request:
     branches: ["main"]
-  
+
 jobs:
   test-build-package:
     name: Test package build
@@ -34,7 +34,7 @@ jobs:
           WHEEL_FILE=$(ls dist/*.whl | head -1)
           echo "Installing wheel: $WHEEL_FILE"
           uv pip install "$WHEEL_FILE"
-        
+
       - name: Test twyn as a library
         run: uv run python -c "import twyn; twyn.check_dependencies"
 
@@ -73,12 +73,22 @@ jobs:
 
   test-docker-build:
     needs: [should-test-docker-build]
-    name: Test Docker build
+    name: Test Docker build ${{ matrix.arch }}
     runs-on: ubuntu-latest
     if: (needs.should-test-docker-build.outputs.workflow == 'true' || needs.should-test-docker-build.outputs.docker == 'true') && !startsWith(github.event.head_commit.message, 'bump:')
     permissions:
       contents: read
       packages: read
+    strategy:
+      matrix:
+        include:
+          - arch: amd64
+            platform: linux/amd64
+            image-name: build-amd64
+          - arch: arm64
+            platform: linux/arm64
+            image-name: build-arm64
+            needs-qemu: true
     steps:
       - name: Check out the repo
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -90,12 +100,23 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set up QEMU
+        if: matrix.needs-qemu
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
+        with:
+          platforms: ${{ matrix.platform }}
+
       - name: Build image
         uses: ./.github/actions/docker-build
         with:
           context: .
           file: ./Dockerfile
           push: false
-          platforms: linux/amd64,linux/arm64
-          cache-from: type=registry,ref=ghcr.io/elementsinteractive/twyn:buildcache
-       
+          load: true
+          platforms: ${{ matrix.platform }}
+          cache-from: type=registry,ref=ghcr.io/elementsinteractive/twyn:buildcache-${{ matrix.arch }}
+          image-name: ${{ matrix.image-name }}
+
+      - name: Test
+        run: |
+          docker run --platform ${{ matrix.platform }} --rm ${{ matrix.image-name }}:pr-${{ github.event.pull_request.number }} --version

.github/workflows/publish.yml

@@ -27,9 +27,59 @@ jobs:
           uv build
           uv publish
 
-  push_to_docker_hub:
-    name: Push Docker image to Docker Hub
+  build_and_test_docker:
+    name: Build and test Docker image (${{ matrix.arch }})
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      matrix:
+        include:
+          - arch: amd64
+            platform: linux/amd64
+            cache-ref: buildcache-amd64
+          - arch: arm64
+            platform: linux/arm64
+            cache-ref: buildcache-arm64
+            needs-qemu: true
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up QEMU
+        if: matrix.needs-qemu
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
+        with:
+          platforms: ${{ matrix.arch }}
+
+      - name: Build Docker image (${{ matrix.arch }})
+        uses: ./.github/actions/docker-build
+        with:
+          push: false
+          load: true
+          platforms: ${{ matrix.platform }}
+          dockerfile: ./Dockerfile
+          context: .
+          image-name: elementsinteractive/twyn
+          cache-from: type=registry,ref=ghcr.io/elementsinteractive/twyn:${{ matrix.cache-ref }}
+          cache-to: type=registry,ref=ghcr.io/elementsinteractive/twyn:${{ matrix.cache-ref }},mode=max,compression=zstd,force-compression=true,oci-mediatypes=true
+
+      - name: Test Docker image (${{ matrix.arch }})
+        run: |
+          docker run --platform ${{ matrix.platform }} --rm elementsinteractive/twyn --version
+
+  publish_docker_images:
+    name: Push Docker images to registries
+    runs-on: ubuntu-latest
+    needs: [build_and_test_docker]
     permissions:
       contents: read
       packages: write
@@ -50,16 +100,15 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push Docker image
+      - name: Push Docker images
         uses: ./.github/actions/docker-build
         with:
-          push-image: "true"
+          push: "true"
           platforms: linux/amd64,linux/arm64
           dockerfile: ./Dockerfile
           context: .
           image-name: elementsinteractive/twyn
-          cache-from: type=registry,ref=ghcr.io/elementsinteractive/twyn:buildcache
-          cache-to: type=registry,ref=ghcr.io/elementsinteractive/twyn:buildcache,mode=max,compression=zstd,force-compression=true,oci-mediatypes=true
+          cache-from: type=registry,ref=ghcr.io/elementsinteractive/twyn:buildcache-amd64,ref=ghcr.io/elementsinteractive/twyn:buildcache-arm64
 
       - name: Delete old cache entries
         env:
@@ -68,17 +117,18 @@ jobs:
           # Get all versions of the container package
           versions=$(gh api "orgs/elementsinteractive/packages/container/twyn/versions" --paginate)
 
-          # Extract version IDs that do NOT have the 'buildcache' tag
-          ids_to_delete=$(echo "$versions" | jq -r '.[] | select(.metadata.container.tags | index("buildcache") | not) | .id')
+          # Extract version IDs that do NOT have any buildcache-* tags (buildcache-amd64, buildcache-arm64, etc.)
+          ids_to_delete=$(echo "$versions" | jq -r '.[] | select(.metadata.container.tags | map(test("^buildcache-")) | any | not) | .id')
 
           # Delete them
           for id in $ids_to_delete; do
             echo "Deleting old cache version ID: $id"
             gh api -X DELETE "orgs/elementsinteractive/packages/container/twyn/versions/$id"
           done
+
   release_notes:
     runs-on: ubuntu-latest
-    needs: [push_to_pypi, push_to_docker_hub]
+    needs: [push_to_pypi, publish_docker_images]
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       - name: Release

Dockerfile

@@ -44,7 +44,7 @@ RUN ${BIN_PATH}/python -m ensurepip
 COPY uv.lock pyproject.toml ./
 
 # Install dependencies using uv (only dependencies, not the project itself)
-RUN uv sync --inexact --frozen --all-extras --no-install-project --compile-bytecode
+RUN uv sync --inexact --frozen --all-extras --no-install-project --compile-bytecode 
 
 # --------------- `final` stage --------------- 
 FROM base AS final

pyproject.toml

@@ -62,6 +62,8 @@ download = [
 ]
 local = ["ipdb<1.0.0,>=0.13.9", "commitizen<5.0,>=2.38", "pdbpp<1.0.0,>=0.11.6"]
 
+[tool.uv]
+default-groups = []
 
 [build-system]
 requires = ["hatchling"]