From d16cd3b13975d40711c5b5cf640df4e1c8101267 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Sat, 8 Feb 2025 10:38:49 +0000 Subject: [PATCH 01/11] Split test and build steps (preparation for matrix build) --- .github/workflows/ci-pipeline.yml | 52 ++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 1235831..b4c4c2f 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -17,8 +17,39 @@ env: TEST_TAG: user/app:test jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # This might be unnecessary as tests are not + # multiplatform + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v3.9.0 + + # Build but not push Docker image with Buildx + # https://github.com/docker/build-push-action + - name: Build test image + id: build-test + uses: docker/build-push-action@v6.13.0 + with: + context: . + load: true + target: dev + tags: ${{ env.TEST_TAG }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # This is a barrier check to make sure we push a functional + # docker image, we can avoid linting + - name: Run tests in the test image + run: | + docker run --rm ${{ env.TEST_TAG }} make ci-test + build: runs-on: ubuntu-latest + needs: test permissions: contents: read packages: write @@ -33,7 +64,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v3 - # Install the cosign tool except on PR + # Install the cosign tool # https://github.com/sigstore/cosign-installer - name: Install cosign uses: sigstore/cosign-installer@v3.8.0 @@ -41,7 +72,7 @@ jobs: - name: Setup Docker buildx uses: docker/setup-buildx-action@v3.9.0 - # Login against a Docker registry except on PR + # Login against a Docker registry # https://github.com/docker/login-action - name: Log into registry ${{ env.REGISTRY }} uses: docker/login-action@v3.3.0 @@ -50,23 +81,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # Build and push Docker image with Buildx - # https://github.com/docker/build-push-action - - name: Build test image - id: build-test - uses: docker/build-push-action@v6.13.0 - with: - context: . - load: true - target: dev - tags: ${{ env.TEST_TAG }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Test - run: | - docker run --rm ${{ env.TEST_TAG }} make ci-test - # Extract metadata (tags, labels) for Docker # https://github.com/docker/metadata-action - name: Extract Docker metadata From 1f2b0ce9372e27a0a8c2eca181a172022bdc3c29 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Sat, 8 Feb 2025 10:39:34 +0000 Subject: [PATCH 02/11] Temporary run ci pipeline in PR --- .github/workflows/ci-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b4c4c2f..a02d4f4 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -8,6 +8,8 @@ name: CI Pipeline on: push: branches: [ "main" ] + pull_request: + branches: [ "main" ] env: # Use docker.io for Docker Hub if empty From a7c70408860196ed754e8d0ef9b0a748210db472 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Sat, 8 Feb 2025 10:50:00 +0000 Subject: [PATCH 03/11] Temporary disable branch in tags --- .github/workflows/ci-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index a02d4f4..cca5398 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -95,8 +95,8 @@ jobs: # generate Docker tags based on the following events/attributes tags: | type=sha - type=raw,value={{branch}}-latest - type=raw,value={{branch}}-{{date 'YYYYMMDDHHmmss'}} +# type=raw,value={{branch}}-latest +# type=raw,value={{branch}}-{{date 'YYYYMMDDHHmmss'}} # Build and push Docker image with Buildx # https://github.com/docker/build-push-action From b290e486f20408b5e392afa3409c19a5f15c04a9 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Sat, 8 Feb 2025 10:50:17 +0000 Subject: [PATCH 04/11] Matrix build all targets --- .github/workflows/ci-pipeline.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index cca5398..9d88587 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -58,7 +58,12 @@ jobs: # This is used to complete the identity challenge # with sigstore/fulcio when running outside of PRs. id-token: write - + strategy: + matrix: + docker_target: + - http_app + - socketio_app + - dramatiq_app steps: - name: Checkout repository uses: actions/checkout@v4 @@ -91,7 +96,7 @@ jobs: with: # list of Docker images to use as base name for tags images: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ env.IMAGE_NAME }}-${{ matrix.docker_target }} # generate Docker tags based on the following events/attributes tags: | type=sha @@ -105,7 +110,7 @@ jobs: uses: docker/build-push-action@v6.13.0 with: context: . - target: http_app + target: ${{ matrix.docker_target }} platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} From ae8993f4999591e93f724cf7e13a3445b8ec67ea Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Sat, 8 Feb 2025 10:52:40 +0000 Subject: [PATCH 05/11] Fix typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 162cd29..60148cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -84,7 +84,7 @@ CMD ["opentelemetry-instrument", "python", "-m", "http_app"] # Copy the socketio python package and requirements from relevant builder FROM base_app AS socketio_app -COPY --from=socketio_builder_builder /venv /venv +COPY --from=socketio_builder /venv /venv COPY --chown=nonroot:nonroot src/socketio_app ./socketio_app # Run CMD using array syntax, so it's uses `exec` and runs as PID1 CMD ["opentelemetry-instrument", "python", "-m", "socketio_app"] From 6ed15b852fdb17cae302df56c31bc4af94185533 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Sat, 8 Feb 2025 10:53:55 +0000 Subject: [PATCH 06/11] Comments --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 60148cb..f8bb811 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,20 +79,20 @@ COPY --chown=nonroot:nonroot src/alembic.ini . FROM base_app AS http_app COPY --from=http_builder /venv /venv COPY --chown=nonroot:nonroot src/http_app ./http_app -# Run CMD using array syntax, so it's uses `exec` and runs as PID1 +# Run CMD using array syntax, so it uses `exec` and runs as PID1 CMD ["opentelemetry-instrument", "python", "-m", "http_app"] # Copy the socketio python package and requirements from relevant builder FROM base_app AS socketio_app COPY --from=socketio_builder /venv /venv COPY --chown=nonroot:nonroot src/socketio_app ./socketio_app -# Run CMD using array syntax, so it's uses `exec` and runs as PID1 +# Run CMD using array syntax, so it uses `exec` and runs as PID1 CMD ["opentelemetry-instrument", "python", "-m", "socketio_app"] # Copy the dramatiq python package and requirements from relevant builder FROM base_app AS dramatiq_app COPY --from=dramatiq_builder /venv /venv COPY --chown=nonroot:nonroot src/dramatiq_worker ./dramatiq_worker -# Run CMD using array syntax, so it's uses `exec` and runs as PID1 +# Run CMD using array syntax, so it uses `exec` and runs as PID1 # TODO: Review processes/threads CMD ["opentelemetry-instrument", "dramatiq", "-p", "1", "-t", "1", "dramatiq_worker"] From 9547a81919c5dbc2d4e8f984f30e0c6a3484615d Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Sat, 8 Feb 2025 10:54:45 +0000 Subject: [PATCH 07/11] Remove pytest marks --- tests/socketio_app/namespaces/test_chat.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/socketio_app/namespaces/test_chat.py b/tests/socketio_app/namespaces/test_chat.py index b50c0c4..cd327b4 100644 --- a/tests/socketio_app/namespaces/test_chat.py +++ b/tests/socketio_app/namespaces/test_chat.py @@ -10,7 +10,6 @@ def chat_namespace(): return ChatNamespace("/chat") -@pytest.mark.asyncio async def test_on_connect(chat_namespace): sid = "test_session_id" environ = {} @@ -19,7 +18,6 @@ async def test_on_connect(chat_namespace): chat_namespace.on_connect(sid, environ) -@pytest.mark.asyncio async def test_on_disconnect(chat_namespace): sid = "test_session_id" reason = "test_reason" @@ -28,7 +26,6 @@ async def test_on_disconnect(chat_namespace): chat_namespace.on_disconnect(sid, reason) -@pytest.mark.asyncio async def test_on_echo_message(chat_namespace): sid = "test_session_id" test_data = {"message": "Hello, World!"} From 08dd9e4d06062bc7c3bd254ed5a47259e53c3930 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Sat, 8 Feb 2025 11:11:16 +0000 Subject: [PATCH 08/11] Try manual image name --- .github/workflows/ci-pipeline.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 9d88587..4f49f9a 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -15,7 +15,7 @@ env: # Use docker.io for Docker Hub if empty REGISTRY: ghcr.io # github.repository as / - IMAGE_NAME: ${{ github.repository }} + REGISTRY_PATH: ${{ github.repository }} TEST_TAG: user/app:test jobs: @@ -30,7 +30,7 @@ jobs: - name: Setup Docker buildx uses: docker/setup-buildx-action@v3.9.0 - # Build but not push Docker image with Buildx + # Build but don't push Docker image with Buildx # https://github.com/docker/build-push-action - name: Build test image id: build-test @@ -65,6 +65,13 @@ jobs: - socketio_app - dramatiq_app steps: + # GitHub gives only repository complete in / format. + # Need some manual sheanigans + # Set IMAGE_NAME so we can push to // + - name: Set ENV variables + run: | + echo "IMAGE_NAME=${GITHUB_REPOSITORY#$GITHUB_REPOSITORY_OWNER/}" >> $GITHUB_ENV + - name: Checkout repository uses: actions/checkout@v4 @@ -95,8 +102,9 @@ jobs: uses: docker/metadata-action@v5.6.1 with: # list of Docker images to use as base name for tags + # //- images: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ env.IMAGE_NAME }}-${{ matrix.docker_target }} + ${{ env.REGISTRY }}/${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}-${{ matrix.docker_target }} # generate Docker tags based on the following events/attributes tags: | type=sha From 4a1a3b726a11bac24b04973ba905c59991e8f494 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Sat, 8 Feb 2025 11:23:39 +0000 Subject: [PATCH 09/11] Rename dockerfile targets --- .github/workflows/ci-pipeline.yml | 6 +++--- .gitlab_ci/build.yml | 4 ++-- Dockerfile | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 4f49f9a..21a11be 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -61,9 +61,9 @@ jobs: strategy: matrix: docker_target: - - http_app - - socketio_app - - dramatiq_app + - http + - socketio + - dramatiq steps: # GitHub gives only repository complete in / format. # Need some manual sheanigans diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml index 600fcb1..74b3c2d 100644 --- a/.gitlab_ci/build.yml +++ b/.gitlab_ci/build.yml @@ -17,7 +17,7 @@ build-http-app-amd64: variables: DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-http DOCKER_PLATFORM: "linux/amd64" - DOCKER_TARGET: http_app + DOCKER_TARGET: http tags: - saas-linux-small-amd64 rules: @@ -32,7 +32,7 @@ build-http-app-arm64: variables: DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-http DOCKER_PLATFORM: "linux/arm64" - DOCKER_TARGET: http_app + DOCKER_TARGET: http tags: - saas-linux-small-arm64 rules: diff --git a/Dockerfile b/Dockerfile index f8bb811..9241636 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,21 +76,21 @@ COPY --chown=nonroot:nonroot src/common ./common COPY --chown=nonroot:nonroot src/alembic.ini . # Copy the http python package and requirements from relevant builder -FROM base_app AS http_app +FROM base_app AS http COPY --from=http_builder /venv /venv COPY --chown=nonroot:nonroot src/http_app ./http_app # Run CMD using array syntax, so it uses `exec` and runs as PID1 CMD ["opentelemetry-instrument", "python", "-m", "http_app"] # Copy the socketio python package and requirements from relevant builder -FROM base_app AS socketio_app +FROM base_app AS socketio COPY --from=socketio_builder /venv /venv COPY --chown=nonroot:nonroot src/socketio_app ./socketio_app # Run CMD using array syntax, so it uses `exec` and runs as PID1 CMD ["opentelemetry-instrument", "python", "-m", "socketio_app"] # Copy the dramatiq python package and requirements from relevant builder -FROM base_app AS dramatiq_app +FROM base_app AS dramatiq COPY --from=dramatiq_builder /venv /venv COPY --chown=nonroot:nonroot src/dramatiq_worker ./dramatiq_worker # Run CMD using array syntax, so it uses `exec` and runs as PID1 From a0d9e0368d68a9d366d094e96d3e0436d60a025f Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Sat, 8 Feb 2025 11:25:29 +0000 Subject: [PATCH 10/11] Rearrange COPY in alphabetical order --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9241636..5785aa0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,10 +69,10 @@ RUN --mount=type=cache,target=~/.cache/uv \ # Create the base app with the common python packages FROM base AS base_app USER nonroot -COPY --chown=nonroot:nonroot src/migrations ./migrations +COPY --chown=nonroot:nonroot src/common ./common COPY --chown=nonroot:nonroot src/domains ./domains COPY --chown=nonroot:nonroot src/gateways ./gateways -COPY --chown=nonroot:nonroot src/common ./common +COPY --chown=nonroot:nonroot src/migrations ./migrations COPY --chown=nonroot:nonroot src/alembic.ini . # Copy the http python package and requirements from relevant builder From 5a67c91186bd6a743e13aa8c09aec44d5aa3c9ad Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Sat, 8 Feb 2025 11:32:19 +0000 Subject: [PATCH 11/11] Run back CI only on main --- .github/workflows/ci-pipeline.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 21a11be..3786ef2 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -8,8 +8,11 @@ name: CI Pipeline on: push: branches: [ "main" ] - pull_request: - branches: [ "main" ] +####### Uncomment this to test the CI pipeline in a PR +####### You'll also need to comment the rules containing {{branch}} +####### in the `Extract Docker metadata` step +# pull_request: +# branches: [ "main" ] env: # Use docker.io for Docker Hub if empty @@ -108,8 +111,8 @@ jobs: # generate Docker tags based on the following events/attributes tags: | type=sha -# type=raw,value={{branch}}-latest -# type=raw,value={{branch}}-{{date 'YYYYMMDDHHmmss'}} + type=raw,value={{branch}}-latest + type=raw,value={{branch}}-{{date 'YYYYMMDDHHmmss'}} # Build and push Docker image with Buildx # https://github.com/docker/build-push-action