diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 4f5b840..d911017 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -24,10 +24,30 @@ env: jobs: test: runs-on: ubuntu-latest + permissions: + contents: read + packages: write steps: + # GitHub gives only repository complete in / format. + # Need some manual shenanigans + # Set IMAGE_NAME so we can push to // + # Transform os/arch to os-arch for suffix target + - name: Set ENV variables + run: | + echo "IMAGE_NAME=${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/}" >> $GITHUB_ENV + - name: Checkout repository uses: actions/checkout@v4 + # Login against a Docker registry + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v3.3.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # This might be unnecessary as tests are not # multiplatform - name: Setup Docker buildx @@ -43,8 +63,8 @@ jobs: load: true target: dev tags: ${{ env.TEST_TAG }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-cache:tests + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-cache:tests,mode=max # This is a barrier check to make sure we push a functional # docker image, we can avoid linting @@ -52,8 +72,9 @@ jobs: run: | docker run --rm ${{ env.TEST_TAG }} make ci-test - build: - runs-on: ubuntu-latest + # Inspired to https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners + build-arch: + runs-on: ${{ matrix.arch.runner }} needs: test permissions: contents: read @@ -63,6 +84,13 @@ jobs: id-token: write strategy: matrix: + arch: + - platform: linux/amd64 + runner: ubuntu-latest + - platform: linux/arm64 + # There is no latest for ARM yet + # https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources + runner: ubuntu-24.04-arm docker_target: - migrations - http @@ -70,17 +98,106 @@ jobs: - dramatiq steps: # GitHub gives only repository complete in / format. - # Need some manual sheanigans + # Need some manual shenanigans # Set IMAGE_NAME so we can push to // + # Transform os/arch to os-arch for suffix target - name: Set ENV variables run: | echo "IMAGE_NAME=${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/}" >> $GITHUB_ENV + platform=${{ matrix.arch.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - name: Checkout repository uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + # Install the cosign tool + # https://github.com/sigstore/cosign-installer + - name: Install cosign + uses: sigstore/cosign-installer@v3.8.1 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v3.10.0 + + # Login against a Docker registry + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v3.3.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # We extract metadata without tags for single image + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5.7.0 + with: + # list of Docker images to use as base name for tags + # //- + images: | + ${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-${{ matrix.docker_target }} + + + # This build an image WITHOUT tags and outputs the digests, so that we can aggragate them later + - name: Build and push production image + id: build-and-push + uses: docker/build-push-action@v6.15.0 + with: + context: . + target: ${{ matrix.docker_target }} + platforms: ${{ matrix.arch.platform }} + outputs: type=image,push-by-digest=true,name-canonical=true,push=true + tags: ${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-${{ matrix.docker_target }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-cache:buildcache-${{ matrix.docker_target }}-${{ env.PLATFORM_PAIR }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-cache:buildcache-${{ matrix.docker_target }}-${{ env.PLATFORM_PAIR }},mode=max + + - name: Export digest + run: | + mkdir -p ${{ runner.temp }}/digests/${{ matrix.docker_target }} + digest="${{ steps.build-and-push.outputs.digest }}" + touch "${{ runner.temp }}/digests/${{ matrix.docker_target }}/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: ${{ runner.temp }}/digests/${{ matrix.docker_target }}/* + if-no-files-found: error + retention-days: 1 + + + aggregate-manifests: + runs-on: ubuntu-latest + needs: build-arch + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + strategy: + matrix: + docker_target: + - migrations + - http + - socketio + - dramatiq + + steps: + # GitHub gives only repository complete in / format. + # Need some manual sheanigans + # Set IMAGE_NAME so we can push to // + - name: Set ENV variables + run: | + echo "IMAGE_NAME=${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/}" >> $GITHUB_ENV + + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: ${{ runner.temp }}/digests/${{ matrix.docker_target }} + pattern: digests-* + merge-multiple: true # Install the cosign tool # https://github.com/sigstore/cosign-installer @@ -115,20 +232,15 @@ jobs: type=raw,value={{branch}}-latest type=raw,value={{branch}}-{{date 'YYYYMMDDHHmmss'}} - # Build and push Docker image with Buildx - # https://github.com/docker/build-push-action - - name: Build and push production image - id: build-and-push - uses: docker/build-push-action@v6.15.0 - with: - context: . - target: ${{ matrix.docker_target }} - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + - name: Create manifest list and push + working-directory: ${{ runner.temp }}/digests/${{ matrix.docker_target }} + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-${{ matrix.docker_target }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-${{ matrix.docker_target }}:${{ steps.meta.outputs.version }} #TODO: Implement signature using generated key: https://docs.sigstore.dev/signing/quickstart/#signing-with-a-generated-key @@ -141,12 +253,11 @@ jobs: env: # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable TAGS: ${{ steps.meta.outputs.tags }} - DIGEST: ${{ steps.build-and-push.outputs.digest }} # This step uses the identity token to provision an ephemeral certificate # against the sigstore community Fulcio instance. run: | images="" for tag in ${TAGS}; do - images+="${tag}@${DIGEST} " + images+="${tag}@$(docker buildx imagetools inspect --format '{{json .Manifest.Digest}}' ${tag} | xargs) " done cosign sign --yes ${images} diff --git a/Dockerfile b/Dockerfile index 5af24f8..129c64f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,22 +14,25 @@ RUN mkdir /venv && chown nonroot:nonroot /venv ENV PATH="/venv/bin:$PATH" # Install necessary runtime libraries (e.g. libmysql) -RUN apt-get update \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update \ && apt-get install -y --no-install-recommends \ - make \ - && rm -rf /var/lib/apt/lists/* + make FROM base AS base_builder ENV UV_PROJECT_ENVIRONMENT=/venv # Enable bytecode compilation ENV UV_COMPILE_BYTECODE=1 +ENV UV_LINK_MODE=copy # Install build system requirements (gcc, library headers, etc.) # for compiled Python requirements like psycopg2 -RUN apt-get update \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential gcc git \ - && rm -rf /var/lib/apt/lists/* + build-essential gcc git COPY --from=ghcr.io/astral-sh/uv:0.6.3 /uv /uvx /bin/ @@ -44,7 +47,7 @@ COPY --chown=nonroot:nonroot Makefile . # Dev image, contains all files and dependencies FROM base_builder AS dev COPY --chown=nonroot:nonroot . . -RUN --mount=type=cache,target=~/.cache/uv \ +RUN --mount=type=cache,target=/home/nonroot/.cache/uv,sharing=locked,uid=$UID,gid=$GID \ make dev-dependencies # Note that opentelemetry doesn't play well together with uvicorn reloader @@ -53,22 +56,22 @@ CMD ["uvicorn", "http_app:create_app", "--host", "0.0.0.0", "--port", "8000", "- # Installs requirements to run production dramatiq application FROM base_builder AS dramatiq_builder -RUN --mount=type=cache,target=~/.cache/uv \ +RUN --mount=type=cache,target=/home/nonroot/.cache/uv,sharing=locked,uid=$UID,gid=$GID \ uv sync --no-dev --no-install-project --frozen --no-editable # Installs requirements to run production http application FROM base_builder AS http_builder -RUN --mount=type=cache,target=~/.cache/uv \ +RUN --mount=type=cache,target=/home/nonroot/.cache/uv,sharing=locked,uid=$UID,gid=$GID \ uv sync --no-dev --group http --no-install-project --frozen --no-editable # Installs requirements to run production socketio application FROM base_builder AS socketio_builder -RUN --mount=type=cache,target=~/.cache/uv \ +RUN --mount=type=cache,target=/home/nonroot/.cache/uv,sharing=locked,uid=$UID,gid=$GID \ uv sync --no-dev --group socketio --no-install-project --frozen --no-editable # Installs requirements to run production migrations application FROM base_builder AS migrations_builder -RUN --mount=type=cache,target=~/.cache/uv \ +RUN --mount=type=cache,target=/home/nonroot/.cache/uv,sharing=locked,uid=$UID,gid=$GID \ uv sync --no-dev --group migrations --no-install-project --frozen --no-editable # Create the base app with the common python packages