From cd8216e3f7453ab3e53f1ab171b8f39d2666ff53 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Thu, 26 Dec 2024 11:10:48 +0000 Subject: [PATCH 01/62] Gitlab CI experiment --- .gitlab-ci.yml | 2 ++ .gitlab_ci/_templates.yml | 0 .gitlab_ci/base.yml | 16 ++++++++++++++++ .gitlab_ci/build.yml | 20 ++++++++++++++++++++ .gitlab_ci/release.yml | 25 +++++++++++++++++++++++++ .gitlab_ci/test.yml | 10 ++++++++++ 6 files changed, 73 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 .gitlab_ci/_templates.yml create mode 100644 .gitlab_ci/base.yml create mode 100644 .gitlab_ci/build.yml create mode 100644 .gitlab_ci/release.yml create mode 100644 .gitlab_ci/test.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..6228992c --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,2 @@ +include: + - local: .gitlab_ci/base.yml diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml new file mode 100644 index 00000000..e69de29b diff --git a/.gitlab_ci/base.yml b/.gitlab_ci/base.yml new file mode 100644 index 00000000..f03dc579 --- /dev/null +++ b/.gitlab_ci/base.yml @@ -0,0 +1,16 @@ +variables: + # Use docker.io for Docker Hub if empty + REGISTRY: registry.gitlab.com + # IMAGE_NAME is defined as / in GitLab CI/CD + IMAGE_NAME: $CI_REGISTRY_IMAGE + TEST_TAG: $REGISTRY/$CI_PROJECT_PATH:test + +stages: + - build + - test + - release + +include: + - local: /.gitlab_ci/build.yml + - local: /.gitlab_ci/test.yml + - local: /.gitlab_ci/release.yml diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml new file mode 100644 index 00000000..eaea0d82 --- /dev/null +++ b/.gitlab_ci/build.yml @@ -0,0 +1,20 @@ +# Build Docker image for test +build-test: + stage: build + image: docker:24.0.2 + services: + - docker:24.0.2-dind + variables: + DOCKER_BUILDKIT: 1 + before_script: + - apk add --no-cache qemu bash + script: + - docker buildx create --use + - docker buildx inspect --bootstrap + - docker buildx build --load --target=dev --tag=$TEST_TAG --cache-to=type=local,dest=/tmp/.buildx-cache --cache-from=type=local,src=/tmp/.buildx-cache . + artifacts: + paths: + - /tmp/.buildx-cache + expire_in: 1h + + diff --git a/.gitlab_ci/release.yml b/.gitlab_ci/release.yml new file mode 100644 index 00000000..4f50a169 --- /dev/null +++ b/.gitlab_ci/release.yml @@ -0,0 +1,25 @@ +# Production Build with Multi-Arch +release: + stage: release + image: docker:24.0.2 + only: + - main + services: + - docker:24.0.2-dind + variables: + CI_REGISTRY_USER: "$CI_REGISTRY_USER" + CI_REGISTRY_PASSWORD: "$CI_REGISTRY_PASSWORD" + DOCKER_BUILDKIT: 1 + before_script: + - apk add --no-cache qemu bash + - docker buildx create --use + - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" + script: + - docker buildx inspect --bootstrap + - | + docker buildx build --push --target=http_app \ + --platform linux/amd64,linux/arm64 \ + --tag $CI_REGISTRY/$CI_PROJECT_PATH:latest \ + --tag $CI_REGISTRY/$CI_PROJECT_PATH:$(date +%Y%m%d%H%M%S) \ + --cache-to=type=registry,ref=$CI_REGISTRY/$CI_PROJECT_PATH/cache,mode=max \ + --cache-from=type=registry,ref=$CI_REGISTRY/$CI_PROJECT_PATH/cache . diff --git a/.gitlab_ci/test.yml b/.gitlab_ci/test.yml new file mode 100644 index 00000000..42efe97a --- /dev/null +++ b/.gitlab_ci/test.yml @@ -0,0 +1,10 @@ +# Test Docker image +test: + stage: test + image: docker:24.0.2 + services: + - docker:24.0.2-dind + before_script: + - apk add --no-cache bash + script: + - docker run --rm $TEST_TAG make ci-test From 8bee95051bde0f873daafb81e53e9b8a618c5220 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Mon, 30 Dec 2024 23:22:05 +0000 Subject: [PATCH 02/62] Add git in build phase Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/build.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml index eaea0d82..53547cd4 100644 --- a/.gitlab_ci/build.yml +++ b/.gitlab_ci/build.yml @@ -7,11 +7,15 @@ build-test: variables: DOCKER_BUILDKIT: 1 before_script: - - apk add --no-cache qemu bash + - apk add --no-cache qemu bash git script: - docker buildx create --use - docker buildx inspect --bootstrap - - docker buildx build --load --target=dev --tag=$TEST_TAG --cache-to=type=local,dest=/tmp/.buildx-cache --cache-from=type=local,src=/tmp/.buildx-cache . + - | + docker buildx build --load --target=dev --tag=$TEST_TAG \ + --cache-to=type=local,dest=/tmp/.buildx-cache \ + --cache-from=type=local,src=/tmp/.buildx-cache \ + . artifacts: paths: - /tmp/.buildx-cache From 7cb8983b8126b2abf8a2cae7005dadc0bfb8cd97 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 12:39:54 +0000 Subject: [PATCH 03/62] Create multi-arch build template Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 29 +++++++++++++++++++++++++++++ .gitlab_ci/build.yml | 27 +++++++-------------------- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index e69de29b..66e25880 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -0,0 +1,29 @@ +# Build Docker image for test +.build-and-push-gitlab: + image: docker:24.0.2 + services: + - docker:24.0.2-dind + variables: + DOCKER_BUILDKIT: 1 + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME + DOCKER_IMAGE_TAG: $CI_COMMIT_SHA + DOCKER_IMAGE_TAG_SUFFIX: "" + DOCKER_PLATFORM: "linux/arm64,linux/amd64" + DOCKER_TARGET: dev + before_script: + - apk add --no-cache qemu bash git + script: + - docker buildx create --use + - docker buildx inspect --bootstrap + - | + docker buildx build --load \ + --target=$DOCKER_TARGET \ + --tag=$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX \ + --platform=$DOCKER_PLATFORM \ + --cache-to=type=local,dest=/tmp/.buildx-cache \ + --cache-from=type=local,src=/tmp/.buildx-cache \ + . + artifacts: + paths: + - /tmp/.buildx-cache + expire_in: 1h diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml index 53547cd4..e5a7f896 100644 --- a/.gitlab_ci/build.yml +++ b/.gitlab_ci/build.yml @@ -1,24 +1,11 @@ # Build Docker image for test build-test: stage: build - image: docker:24.0.2 - services: - - docker:24.0.2-dind variables: - DOCKER_BUILDKIT: 1 - before_script: - - apk add --no-cache qemu bash git - script: - - docker buildx create --use - - docker buildx inspect --bootstrap - - | - docker buildx build --load --target=dev --tag=$TEST_TAG \ - --cache-to=type=local,dest=/tmp/.buildx-cache \ - --cache-from=type=local,src=/tmp/.buildx-cache \ - . - artifacts: - paths: - - /tmp/.buildx-cache - expire_in: 1h - - + DOCKER_IMAGE_TAG_SUFFIX: "-test" + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .build-and-push-gitlab From 72d92216496e88ec5dd04d6157432c748af79b05 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 12:42:04 +0000 Subject: [PATCH 04/62] Include templates file Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab_ci/base.yml b/.gitlab_ci/base.yml index f03dc579..a1364e03 100644 --- a/.gitlab_ci/base.yml +++ b/.gitlab_ci/base.yml @@ -11,6 +11,7 @@ stages: - release include: + - local: /.gitlab_ci/_templates.yml - local: /.gitlab_ci/build.yml - local: /.gitlab_ci/test.yml - local: /.gitlab_ci/release.yml From 09ab310642bf4e37e7139b17a24d4dbf5cc2969a Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 12:46:23 +0000 Subject: [PATCH 05/62] Remove --load Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 66e25880..375191c0 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -16,7 +16,7 @@ - docker buildx create --use - docker buildx inspect --bootstrap - | - docker buildx build --load \ + docker buildx build \ --target=$DOCKER_TARGET \ --tag=$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX \ --platform=$DOCKER_PLATFORM \ From fb1f7a7b31663b51815bbb1616c609913ad5470d Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 13:06:44 +0000 Subject: [PATCH 06/62] Use registry cache and push image to registry Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 375191c0..a38eb9f4 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -5,25 +5,25 @@ - docker:24.0.2-dind variables: DOCKER_BUILDKIT: 1 - DOCKER_IMAGE_NAME: $CI_PROJECT_NAME + DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE DOCKER_IMAGE_TAG: $CI_COMMIT_SHA DOCKER_IMAGE_TAG_SUFFIX: "" - DOCKER_PLATFORM: "linux/arm64,linux/amd64" + DOCKER_PLATFORM: "linux /arm64,linux/amd64" DOCKER_TARGET: dev before_script: - apk add --no-cache qemu bash git script: - docker buildx create --use - docker buildx inspect --bootstrap + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com - | + echo "###### Building $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX - Cache from $DOCKER_IMAGE_NAME:cache ########" docker buildx build \ --target=$DOCKER_TARGET \ --tag=$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX \ --platform=$DOCKER_PLATFORM \ - --cache-to=type=local,dest=/tmp/.buildx-cache \ - --cache-from=type=local,src=/tmp/.buildx-cache \ + --cache-from type=registry,ref=$DOCKER_IMAGE_NAME:cache \ + --cache-to type=registry,ref=$DOCKER_IMAGE_NAME:cache \ + --push \ . - artifacts: - paths: - - /tmp/.buildx-cache - expire_in: 1h + # TODO: Sign image using Cosign From 6fef6a0abe5042c7578d8bc7e799c23562847da2 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 13:11:57 +0000 Subject: [PATCH 07/62] Try to fix buildx command Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index a38eb9f4..4e675ef8 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -16,14 +16,13 @@ - docker buildx create --use - docker buildx inspect --bootstrap - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com + - echo "###### Building $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX - Cache from $DOCKER_IMAGE_NAME:cache ########" - | - echo "###### Building $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX - Cache from $DOCKER_IMAGE_NAME:cache ########" - docker buildx build \ + docker buildx build --push \ --target=$DOCKER_TARGET \ --tag=$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX \ --platform=$DOCKER_PLATFORM \ --cache-from type=registry,ref=$DOCKER_IMAGE_NAME:cache \ --cache-to type=registry,ref=$DOCKER_IMAGE_NAME:cache \ - --push \ . # TODO: Sign image using Cosign From 529374f3a4c7563cb266d7d8ae5b2f6ba85b1b53 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 13:14:41 +0000 Subject: [PATCH 08/62] Try to fix buildx command 2 Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 4e675ef8..e3f0ba16 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -1,4 +1,5 @@ # Build Docker image for test +# TODO: Sign image using Cosign .build-and-push-gitlab: image: docker:24.0.2 services: @@ -25,4 +26,3 @@ --cache-from type=registry,ref=$DOCKER_IMAGE_NAME:cache \ --cache-to type=registry,ref=$DOCKER_IMAGE_NAME:cache \ . - # TODO: Sign image using Cosign From 22182d91fd5f9022f1eb1a756441c78bf12f90be Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 13:16:57 +0000 Subject: [PATCH 09/62] Remove # from echoed string Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index e3f0ba16..155a8fa4 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -17,7 +17,7 @@ - docker buildx create --use - docker buildx inspect --bootstrap - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com - - echo "###### Building $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX - Cache from $DOCKER_IMAGE_NAME:cache ########" + - echo "Building $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX - Cache from $DOCKER_IMAGE_NAME:cache" - | docker buildx build --push \ --target=$DOCKER_TARGET \ From dd630227b736f5154218315f54fc5615cbba6e3d Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 13:20:24 +0000 Subject: [PATCH 10/62] Remove = from options Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 155a8fa4..58421751 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -20,9 +20,9 @@ - echo "Building $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX - Cache from $DOCKER_IMAGE_NAME:cache" - | docker buildx build --push \ - --target=$DOCKER_TARGET \ - --tag=$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX \ - --platform=$DOCKER_PLATFORM \ + --target $DOCKER_TARGET \ + --tag $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX \ + --platform $DOCKER_PLATFORM \ --cache-from type=registry,ref=$DOCKER_IMAGE_NAME:cache \ --cache-to type=registry,ref=$DOCKER_IMAGE_NAME:cache \ . From 6d46ca594ceec975298aa1deb226b0ff71a8a383 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 13:23:11 +0000 Subject: [PATCH 11/62] Add = to all options Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 58421751..18c5a048 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -20,9 +20,9 @@ - echo "Building $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX - Cache from $DOCKER_IMAGE_NAME:cache" - | docker buildx build --push \ - --target $DOCKER_TARGET \ - --tag $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX \ - --platform $DOCKER_PLATFORM \ - --cache-from type=registry,ref=$DOCKER_IMAGE_NAME:cache \ - --cache-to type=registry,ref=$DOCKER_IMAGE_NAME:cache \ + --target=$DOCKER_TARGET \ + --tag=$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX \ + --platform=$DOCKER_PLATFORM \ + --cache-from=type=registry,ref=$DOCKER_IMAGE_NAME:cache \ + --cache-to=type=registry,ref=$DOCKER_IMAGE_NAME:cache \ . From 0a2bb66769b9d93efdc8d90b9daf04c54619ec49 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 13:29:21 +0000 Subject: [PATCH 12/62] Improve login security Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 18c5a048..6d963e9a 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -16,7 +16,7 @@ script: - docker buildx create --use - docker buildx inspect --bootstrap - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com + - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com - echo "Building $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX - Cache from $DOCKER_IMAGE_NAME:cache" - | docker buildx build --push \ From b30b53baca0ffd5f4e7a5d9905e4c8324ffd775d Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 13:29:35 +0000 Subject: [PATCH 13/62] Build and push in separate commands Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 6d963e9a..c52927a9 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -19,10 +19,12 @@ - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com - echo "Building $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX - Cache from $DOCKER_IMAGE_NAME:cache" - | - docker buildx build --push \ - --target=$DOCKER_TARGET \ - --tag=$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX \ - --platform=$DOCKER_PLATFORM \ - --cache-from=type=registry,ref=$DOCKER_IMAGE_NAME:cache \ - --cache-to=type=registry,ref=$DOCKER_IMAGE_NAME:cache \ + docker buildx build \ + --target $DOCKER_TARGET \ + --tag $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX \ + --platform $DOCKER_PLATFORM \ + --cache-from type=registry,ref=$DOCKER_IMAGE_NAME:cache \ + --cache-to type=registry,ref=$DOCKER_IMAGE_NAME:cache \ + --output type=docker \ . + - docker push $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX From 914e93e828fac53c1959dbf14dd449c1f0442a8d Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 13:31:00 +0000 Subject: [PATCH 14/62] Remove whitespace -.-' and unify build+push commands Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index c52927a9..5ff34eb1 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -9,7 +9,7 @@ DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE DOCKER_IMAGE_TAG: $CI_COMMIT_SHA DOCKER_IMAGE_TAG_SUFFIX: "" - DOCKER_PLATFORM: "linux /arm64,linux/amd64" + DOCKER_PLATFORM: "linux/arm64,linux/amd64" DOCKER_TARGET: dev before_script: - apk add --no-cache qemu bash git @@ -19,12 +19,10 @@ - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com - echo "Building $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX - Cache from $DOCKER_IMAGE_NAME:cache" - | - docker buildx build \ + docker buildx build --push \ --target $DOCKER_TARGET \ --tag $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX \ --platform $DOCKER_PLATFORM \ --cache-from type=registry,ref=$DOCKER_IMAGE_NAME:cache \ --cache-to type=registry,ref=$DOCKER_IMAGE_NAME:cache \ - --output type=docker \ . - - docker push $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX From 076104dce8c999d29395391bf3644d98c73b98c3 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:10:36 +0000 Subject: [PATCH 15/62] Use prefix instead of suffix Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 6 +++--- .gitlab_ci/build.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 5ff34eb1..04da4299 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -8,7 +8,7 @@ DOCKER_BUILDKIT: 1 DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE DOCKER_IMAGE_TAG: $CI_COMMIT_SHA - DOCKER_IMAGE_TAG_SUFFIX: "" + DOCKER_IMAGE_TAG_PREFIX: "" DOCKER_PLATFORM: "linux/arm64,linux/amd64" DOCKER_TARGET: dev before_script: @@ -17,11 +17,11 @@ - docker buildx create --use - docker buildx inspect --bootstrap - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com - - echo "Building $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX - Cache from $DOCKER_IMAGE_NAME:cache" + - echo "Building $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG - Cache from $DOCKER_IMAGE_NAME:cache" - | docker buildx build --push \ --target $DOCKER_TARGET \ - --tag $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX \ + --tag $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG \ --platform $DOCKER_PLATFORM \ --cache-from type=registry,ref=$DOCKER_IMAGE_NAME:cache \ --cache-to type=registry,ref=$DOCKER_IMAGE_NAME:cache \ diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml index e5a7f896..6a000bf8 100644 --- a/.gitlab_ci/build.yml +++ b/.gitlab_ci/build.yml @@ -2,7 +2,7 @@ build-test: stage: build variables: - DOCKER_IMAGE_TAG_SUFFIX: "-test" + DOCKER_IMAGE_TAG_PREFIX: "test-" rules: # We run the pipeline only on merge requests or the `main` branch - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH From e12acb8220f64a75dfc80ba0e058ee417c629e57 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:22:53 +0000 Subject: [PATCH 16/62] Comment release Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/release.yml | 48 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.gitlab_ci/release.yml b/.gitlab_ci/release.yml index 4f50a169..5315ff61 100644 --- a/.gitlab_ci/release.yml +++ b/.gitlab_ci/release.yml @@ -1,25 +1,25 @@ # Production Build with Multi-Arch -release: - stage: release - image: docker:24.0.2 - only: - - main - services: - - docker:24.0.2-dind - variables: - CI_REGISTRY_USER: "$CI_REGISTRY_USER" - CI_REGISTRY_PASSWORD: "$CI_REGISTRY_PASSWORD" - DOCKER_BUILDKIT: 1 - before_script: - - apk add --no-cache qemu bash - - docker buildx create --use - - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" - script: - - docker buildx inspect --bootstrap - - | - docker buildx build --push --target=http_app \ - --platform linux/amd64,linux/arm64 \ - --tag $CI_REGISTRY/$CI_PROJECT_PATH:latest \ - --tag $CI_REGISTRY/$CI_PROJECT_PATH:$(date +%Y%m%d%H%M%S) \ - --cache-to=type=registry,ref=$CI_REGISTRY/$CI_PROJECT_PATH/cache,mode=max \ - --cache-from=type=registry,ref=$CI_REGISTRY/$CI_PROJECT_PATH/cache . +#release: +# stage: release +# image: docker:24.0.2 +# only: +# - main +# services: +# - docker:24.0.2-dind +# variables: +# CI_REGISTRY_USER: "$CI_REGISTRY_USER" +# CI_REGISTRY_PASSWORD: "$CI_REGISTRY_PASSWORD" +# DOCKER_BUILDKIT: 1 +# before_script: +# - apk add --no-cache qemu bash +# - docker buildx create --use +# - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" +# script: +# - docker buildx inspect --bootstrap +# - | +# docker buildx build --push --target=http_app \ +# --platform linux/amd64,linux/arm64 \ +# --tag $CI_REGISTRY/$CI_PROJECT_PATH:latest \ +# --tag $CI_REGISTRY/$CI_PROJECT_PATH:$(date +%Y%m%d%H%M%S) \ +# --cache-to=type=registry,ref=$CI_REGISTRY/$CI_PROJECT_PATH/cache,mode=max \ +# --cache-from=type=registry,ref=$CI_REGISTRY/$CI_PROJECT_PATH/cache . From 57db965f5a602082958a23691045e84680e21e1e Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:23:49 +0000 Subject: [PATCH 17/62] Add typing job Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 19 ++++++++++++++----- .gitlab_ci/test.yml | 17 +++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 04da4299..c48d83c3 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -1,3 +1,9 @@ +variables: + DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE + DOCKER_IMAGE_TAG: $CI_COMMIT_SHA + DOCKER_IMAGE_TAG_PREFIX: "" + DOCKER_IMAGE_FULL_TAG: $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG + # Build Docker image for test # TODO: Sign image using Cosign .build-and-push-gitlab: @@ -6,9 +12,6 @@ - docker:24.0.2-dind variables: DOCKER_BUILDKIT: 1 - DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE - DOCKER_IMAGE_TAG: $CI_COMMIT_SHA - DOCKER_IMAGE_TAG_PREFIX: "" DOCKER_PLATFORM: "linux/arm64,linux/amd64" DOCKER_TARGET: dev before_script: @@ -17,12 +20,18 @@ - docker buildx create --use - docker buildx inspect --bootstrap - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com - - echo "Building $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG - Cache from $DOCKER_IMAGE_NAME:cache" + - echo "Building $DOCKER_IMAGE_FULL_TAG - Cache from $DOCKER_IMAGE_NAME:cache" - | docker buildx build --push \ --target $DOCKER_TARGET \ - --tag $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG \ + --tag $DOCKER_IMAGE_FULL_TAG \ --platform $DOCKER_PLATFORM \ --cache-from type=registry,ref=$DOCKER_IMAGE_NAME:cache \ --cache-to type=registry,ref=$DOCKER_IMAGE_NAME:cache \ . + +.python-typing: + image: $DOCKER_IMAGE_FULL_TAG + script: + - make typing + diff --git a/.gitlab_ci/test.yml b/.gitlab_ci/test.yml index 42efe97a..2111574c 100644 --- a/.gitlab_ci/test.yml +++ b/.gitlab_ci/test.yml @@ -1,10 +1,11 @@ # Test Docker image -test: +typing: stage: test - image: docker:24.0.2 - services: - - docker:24.0.2-dind - before_script: - - apk add --no-cache bash - script: - - docker run --rm $TEST_TAG make ci-test + variables: + DOCKER_IMAGE_TAG_PREFIX: "test-" + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .python-typing From 1af4e5cb5c7a730f388a896180ba78545f04826e Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:24:31 +0000 Subject: [PATCH 18/62] Remove release Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/base.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab_ci/base.yml b/.gitlab_ci/base.yml index a1364e03..10900d35 100644 --- a/.gitlab_ci/base.yml +++ b/.gitlab_ci/base.yml @@ -14,4 +14,3 @@ include: - local: /.gitlab_ci/_templates.yml - local: /.gitlab_ci/build.yml - local: /.gitlab_ci/test.yml - - local: /.gitlab_ci/release.yml From 323081820a5b3afd20a1f93e8c7fe8c31531aa26 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:30:51 +0000 Subject: [PATCH 19/62] Add note about native ARM64 build Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index c48d83c3..5a711e5a 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -6,6 +6,7 @@ variables: # Build Docker image for test # TODO: Sign image using Cosign +# TODO: Use ARM64 workers and build image without QEMU: https://docs.gitlab.com/ee/ci/runners/hosted_runners/linux.html .build-and-push-gitlab: image: docker:24.0.2 services: From 6c3c5d3ee5addbb942f409759f1d15df0f3af0dd Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:40:56 +0000 Subject: [PATCH 20/62] Add lint, format and tests Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 15 +++++++++++++++ .gitlab_ci/test.yml | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 5a711e5a..bdfe918a 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -36,3 +36,18 @@ variables: script: - make typing +.python-lint: + image: $DOCKER_IMAGE_FULL_TAG + script: + - make lint + +.python-format: + image: $DOCKER_IMAGE_FULL_TAG + script: + - make format + +.python-tests: + image: $DOCKER_IMAGE_FULL_TAG + script: + - make ci-coverage + diff --git a/.gitlab_ci/test.yml b/.gitlab_ci/test.yml index 2111574c..f901363e 100644 --- a/.gitlab_ci/test.yml +++ b/.gitlab_ci/test.yml @@ -9,3 +9,36 @@ typing: - if: $CI_PIPELINE_SOURCE == "merge_request_event" extends: - .python-typing + +lint: + stage: test + variables: + DOCKER_IMAGE_TAG_PREFIX: "test-" + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .python-lint + +format: + stage: test + variables: + DOCKER_IMAGE_TAG_PREFIX: "test-" + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .python-format + +tests: + stage: test + variables: + DOCKER_IMAGE_TAG_PREFIX: "test-" + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .python-tests From a483a24539952436235db0048d6071bfefe05d53 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:45:08 +0000 Subject: [PATCH 21/62] Use test with console coverage report Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index bdfe918a..7d594824 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -49,5 +49,5 @@ variables: .python-tests: image: $DOCKER_IMAGE_FULL_TAG script: - - make ci-coverage + - make test From 7cfb2055182cc68dfb6952fd3f7d1af9bbdd7d63 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 15:32:17 +0000 Subject: [PATCH 22/62] Remove QEMU and default to amd64 architecture Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 7d594824..d759e7bf 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -13,10 +13,10 @@ variables: - docker:24.0.2-dind variables: DOCKER_BUILDKIT: 1 - DOCKER_PLATFORM: "linux/arm64,linux/amd64" + DOCKER_PLATFORM: "linux/amd64" DOCKER_TARGET: dev before_script: - - apk add --no-cache qemu bash git + - apk add --no-cache bash git script: - docker buildx create --use - docker buildx inspect --bootstrap From de7d3e388e87ca933a54b2da45456042f720e354 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:42:13 +0000 Subject: [PATCH 23/62] Add http app native arch image Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/build.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml index 6a000bf8..d213fcd0 100644 --- a/.gitlab_ci/build.yml +++ b/.gitlab_ci/build.yml @@ -9,3 +9,32 @@ build-test: - if: $CI_PIPELINE_SOURCE == "merge_request_event" extends: - .build-and-push-gitlab + +build-http-app-amd64: + stage: build + variables: + DOCKER_IMAGE_TAG_PREFIX: "amd64-http-" + DOCKER_TARGET: http_app + tags: + - saas-linux-small-amd64 + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .build-and-push-gitlab + +build-http-app-arm64: + stage: build + variables: + DOCKER_IMAGE_TAG_PREFIX: "arm64-http-" + DOCKER_PLATFORM: "linux/arm64" + DOCKER_TARGET: http_app + tags: + - saas-linux-small-arm64 + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .build-and-push-gitlab From 9d2ba952c869337e9fe461fcf9d2f7c3a80c7f64 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 17:08:13 +0000 Subject: [PATCH 24/62] Reverse tag prefixes Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml index d213fcd0..5053590c 100644 --- a/.gitlab_ci/build.yml +++ b/.gitlab_ci/build.yml @@ -13,7 +13,7 @@ build-test: build-http-app-amd64: stage: build variables: - DOCKER_IMAGE_TAG_PREFIX: "amd64-http-" + DOCKER_IMAGE_TAG_PREFIX: "http-amd64-" DOCKER_TARGET: http_app tags: - saas-linux-small-amd64 @@ -27,7 +27,7 @@ build-http-app-amd64: build-http-app-arm64: stage: build variables: - DOCKER_IMAGE_TAG_PREFIX: "arm64-http-" + DOCKER_IMAGE_TAG_PREFIX: "http-arm64-" DOCKER_PLATFORM: "linux/arm64" DOCKER_TARGET: http_app tags: From 4fbb720cab7ad27e2e56f25edfde3da86dab909f Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Tue, 31 Dec 2024 17:42:25 +0000 Subject: [PATCH 25/62] Split prefix and suffix, create multiarch manifest Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 15 ++++++++++++++- .gitlab_ci/build.yml | 21 +++++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index d759e7bf..b0dfa87b 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -2,7 +2,8 @@ variables: DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE DOCKER_IMAGE_TAG: $CI_COMMIT_SHA DOCKER_IMAGE_TAG_PREFIX: "" - DOCKER_IMAGE_FULL_TAG: $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG + DOCKER_IMAGE_TAG_SUFFIX: "" + DOCKER_IMAGE_FULL_TAG: $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX # Build Docker image for test # TODO: Sign image using Cosign @@ -31,6 +32,18 @@ variables: --cache-to type=registry,ref=$DOCKER_IMAGE_NAME:cache \ . +# Architectures are hardcoded for multiarch, need to make this better +.multiarch-manifest-gitlab: + image: docker:24.0.2 + services: + - docker:24.0.2-dind + script: + - docker manifest create $DOCKER_IMAGE_FULL_TAG \ + --amend $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG-amd64 \ + --amend $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG-arm64 + + - docker manifest push $DOCKER_IMAGE_FULL_TAG + .python-typing: image: $DOCKER_IMAGE_FULL_TAG script: diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml index 5053590c..bb254a5d 100644 --- a/.gitlab_ci/build.yml +++ b/.gitlab_ci/build.yml @@ -13,7 +13,9 @@ build-test: build-http-app-amd64: stage: build variables: - DOCKER_IMAGE_TAG_PREFIX: "http-amd64-" + DOCKER_IMAGE_TAG_PREFIX: "http-" + DOCKER_IMAGE_TAG_SUFFIX: "-amd64" + DOCKER_PLATFORM: "linux/amd64" DOCKER_TARGET: http_app tags: - saas-linux-small-amd64 @@ -27,7 +29,8 @@ build-http-app-amd64: build-http-app-arm64: stage: build variables: - DOCKER_IMAGE_TAG_PREFIX: "http-arm64-" + DOCKER_IMAGE_TAG_PREFIX: "http-" + DOCKER_IMAGE_TAG_SUFFIX: "-arm64" DOCKER_PLATFORM: "linux/arm64" DOCKER_TARGET: http_app tags: @@ -38,3 +41,17 @@ build-http-app-arm64: - if: $CI_PIPELINE_SOURCE == "merge_request_event" extends: - .build-and-push-gitlab + +aggregate-http-manifests: + stage: build + needs: + - build-http-app-amd64 + - build-http-app-arm64 + variables: + DOCKER_IMAGE_TAG_PREFIX: "http-" + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .build-and-push-gitlab From 0275a20e034e852f42b5f48f9629b80ea7530b25 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 10:44:03 +0000 Subject: [PATCH 26/62] Reminder to group the build pipeline Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml index bb254a5d..df5ae770 100644 --- a/.gitlab_ci/build.yml +++ b/.gitlab_ci/build.yml @@ -10,6 +10,7 @@ build-test: extends: - .build-and-push-gitlab +# TODO: Group the multi-arch bild (perhaps with a nested workflow) build-http-app-amd64: stage: build variables: From f3558f2dacc6fbdda8bf9f4d01bb45cfb4f0acfe Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 11:15:59 +0000 Subject: [PATCH 27/62] 1st attempt at artifact promotion Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 19 +++++++++++++++++++ .gitlab_ci/base.yml | 3 ++- .gitlab_ci/deploy.yml | 11 +++++++++++ .gitlab_ci/release.yml | 25 ------------------------- 4 files changed, 32 insertions(+), 26 deletions(-) create mode 100644 .gitlab_ci/deploy.yml delete mode 100644 .gitlab_ci/release.yml diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index b0dfa87b..f157683e 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -44,6 +44,25 @@ variables: - docker manifest push $DOCKER_IMAGE_FULL_TAG +.promote-image: + image: docker:24.0.2 + variables: + PROMOTED_ENVIRONMENT: "dev" + DOCKER_BUILDKIT: 1 + services: + - docker:24.0.2-dind + script: + - desired_format="%s" # This format will output the Unix timestamp + - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" +"$desired_format") + - echo "Unix timestamp: $UNIX_TIMESTAMP" + - docker buildx imagetools create \ + --annotation "index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA" \ + --annotation "index:org.opencontainers.image.revision=$CI_COMMIT_SHA" \ + --annotation "index:org.opencontainers.image.source=$CI_PROJECT_URL" \ + --annotation "index:org.opencontainers.image.created=$CI_JOB_STARTED_AT" \ + --tag $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP \ + - $DOCKER_IMAGE_FULL_TAG + .python-typing: image: $DOCKER_IMAGE_FULL_TAG script: diff --git a/.gitlab_ci/base.yml b/.gitlab_ci/base.yml index 10900d35..660333f1 100644 --- a/.gitlab_ci/base.yml +++ b/.gitlab_ci/base.yml @@ -8,9 +8,10 @@ variables: stages: - build - test - - release + - deploy include: - local: /.gitlab_ci/_templates.yml - local: /.gitlab_ci/build.yml - local: /.gitlab_ci/test.yml + - local: /.gitlab_ci/deploy.yml diff --git a/.gitlab_ci/deploy.yml b/.gitlab_ci/deploy.yml new file mode 100644 index 00000000..1029f50e --- /dev/null +++ b/.gitlab_ci/deploy.yml @@ -0,0 +1,11 @@ +promote-dev: + stage: deploy + variables: + DOCKER_IMAGE_TAG_PREFIX: "http-" + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .build-and-push-gitlab + when: manual diff --git a/.gitlab_ci/release.yml b/.gitlab_ci/release.yml deleted file mode 100644 index 5315ff61..00000000 --- a/.gitlab_ci/release.yml +++ /dev/null @@ -1,25 +0,0 @@ -# Production Build with Multi-Arch -#release: -# stage: release -# image: docker:24.0.2 -# only: -# - main -# services: -# - docker:24.0.2-dind -# variables: -# CI_REGISTRY_USER: "$CI_REGISTRY_USER" -# CI_REGISTRY_PASSWORD: "$CI_REGISTRY_PASSWORD" -# DOCKER_BUILDKIT: 1 -# before_script: -# - apk add --no-cache qemu bash -# - docker buildx create --use -# - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" -# script: -# - docker buildx inspect --bootstrap -# - | -# docker buildx build --push --target=http_app \ -# --platform linux/amd64,linux/arm64 \ -# --tag $CI_REGISTRY/$CI_PROJECT_PATH:latest \ -# --tag $CI_REGISTRY/$CI_PROJECT_PATH:$(date +%Y%m%d%H%M%S) \ -# --cache-to=type=registry,ref=$CI_REGISTRY/$CI_PROJECT_PATH/cache,mode=max \ -# --cache-from=type=registry,ref=$CI_REGISTRY/$CI_PROJECT_PATH/cache . From 52dfa551f66df4594a0b47ea15b57cb8f18f4a37 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 11:28:04 +0000 Subject: [PATCH 28/62] Extend the correct template -.- Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 12 ++++++------ .gitlab_ci/deploy.yml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index f157683e..6fafb233 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -56,12 +56,12 @@ variables: - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" +"$desired_format") - echo "Unix timestamp: $UNIX_TIMESTAMP" - docker buildx imagetools create \ - --annotation "index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA" \ - --annotation "index:org.opencontainers.image.revision=$CI_COMMIT_SHA" \ - --annotation "index:org.opencontainers.image.source=$CI_PROJECT_URL" \ - --annotation "index:org.opencontainers.image.created=$CI_JOB_STARTED_AT" \ - --tag $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP \ - - $DOCKER_IMAGE_FULL_TAG + --annotation "index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA" \ + --annotation "index:org.opencontainers.image.revision=$CI_COMMIT_SHA" \ + --annotation "index:org.opencontainers.image.source=$CI_PROJECT_URL" \ + --annotation "index:org.opencontainers.image.created=$CI_JOB_STARTED_AT" \ + --tag $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP \ + - $DOCKER_IMAGE_FULL_TAG .python-typing: image: $DOCKER_IMAGE_FULL_TAG diff --git a/.gitlab_ci/deploy.yml b/.gitlab_ci/deploy.yml index 1029f50e..ebaa3d15 100644 --- a/.gitlab_ci/deploy.yml +++ b/.gitlab_ci/deploy.yml @@ -7,5 +7,5 @@ promote-dev: - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == "merge_request_event" extends: - - .build-and-push-gitlab + - .promote-image when: manual From 8cf46bd973e8e7d5bebd4b0e88bdfb3ec02aad00 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 11:30:35 +0000 Subject: [PATCH 29/62] Fix indentation Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 6fafb233..7d56c3a7 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -52,16 +52,16 @@ variables: services: - docker:24.0.2-dind script: - - desired_format="%s" # This format will output the Unix timestamp - - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" +"$desired_format") - - echo "Unix timestamp: $UNIX_TIMESTAMP" - - docker buildx imagetools create \ - --annotation "index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA" \ - --annotation "index:org.opencontainers.image.revision=$CI_COMMIT_SHA" \ - --annotation "index:org.opencontainers.image.source=$CI_PROJECT_URL" \ - --annotation "index:org.opencontainers.image.created=$CI_JOB_STARTED_AT" \ - --tag $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP \ - - $DOCKER_IMAGE_FULL_TAG + - desired_format="%s" # This format will output the Unix timestamp + - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" +"$desired_format") + - echo "Unix timestamp: $UNIX_TIMESTAMP" + - docker buildx imagetools create \ + --annotation "index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA" \ + --annotation "index:org.opencontainers.image.revision=$CI_COMMIT_SHA" \ + --annotation "index:org.opencontainers.image.source=$CI_PROJECT_URL" \ + --annotation "index:org.opencontainers.image.created=$CI_JOB_STARTED_AT" \ + --tag $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP \ + - $DOCKER_IMAGE_FULL_TAG .python-typing: image: $DOCKER_IMAGE_FULL_TAG From 13ffd1844ef26df265a67cd43d87d3574608ac8b Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 11:40:37 +0000 Subject: [PATCH 30/62] Remove comment Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 7d56c3a7..993b9d7d 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -52,7 +52,7 @@ variables: services: - docker:24.0.2-dind script: - - desired_format="%s" # This format will output the Unix timestamp + - desired_format="%s" - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" +"$desired_format") - echo "Unix timestamp: $UNIX_TIMESTAMP" - docker buildx imagetools create \ From d1230899269c5cbffe94cee692df31406246a559 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 11:42:49 +0000 Subject: [PATCH 31/62] Unhappy with : ? Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 993b9d7d..35280507 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -53,8 +53,8 @@ variables: - docker:24.0.2-dind script: - desired_format="%s" - - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" +"$desired_format") - - echo "Unix timestamp: $UNIX_TIMESTAMP" + - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" + "$desired_format") + - echo "Unix timestamp - $UNIX_TIMESTAMP" - docker buildx imagetools create \ --annotation "index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA" \ --annotation "index:org.opencontainers.image.revision=$CI_COMMIT_SHA" \ From df2a370c2475309f8373f553e5fd3aa2abc344ab Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 11:50:23 +0000 Subject: [PATCH 32/62] Disable unix-timestamp format Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 35280507..5cfebf0e 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -52,15 +52,15 @@ variables: services: - docker:24.0.2-dind script: - - desired_format="%s" - - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" + "$desired_format") - - echo "Unix timestamp - $UNIX_TIMESTAMP" +# - desired_format="%s" +# - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" + "$desired_format") +# - echo "Unix timestamp - $UNIX_TIMESTAMP" - docker buildx imagetools create \ --annotation "index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA" \ --annotation "index:org.opencontainers.image.revision=$CI_COMMIT_SHA" \ --annotation "index:org.opencontainers.image.source=$CI_PROJECT_URL" \ --annotation "index:org.opencontainers.image.created=$CI_JOB_STARTED_AT" \ - --tag $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP \ + --tag $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$CI_COMMIT_TIMESTAMP \ - $DOCKER_IMAGE_FULL_TAG .python-typing: From 771753ff27202ff12b8a494d13f744d0ae90839e Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 11:57:33 +0000 Subject: [PATCH 33/62] Remove - Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 5cfebf0e..b75184c0 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -61,7 +61,7 @@ variables: --annotation "index:org.opencontainers.image.source=$CI_PROJECT_URL" \ --annotation "index:org.opencontainers.image.created=$CI_JOB_STARTED_AT" \ --tag $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$CI_COMMIT_TIMESTAMP \ - - $DOCKER_IMAGE_FULL_TAG + $DOCKER_IMAGE_FULL_TAG .python-typing: image: $DOCKER_IMAGE_FULL_TAG From dccdfb6077f46f6bfc2d958f28aa68fd4e61ae43 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 12:06:55 +0000 Subject: [PATCH 34/62] Remove backslashes Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index b75184c0..6ae33403 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -55,12 +55,12 @@ variables: # - desired_format="%s" # - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" + "$desired_format") # - echo "Unix timestamp - $UNIX_TIMESTAMP" - - docker buildx imagetools create \ - --annotation "index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA" \ - --annotation "index:org.opencontainers.image.revision=$CI_COMMIT_SHA" \ - --annotation "index:org.opencontainers.image.source=$CI_PROJECT_URL" \ - --annotation "index:org.opencontainers.image.created=$CI_JOB_STARTED_AT" \ - --tag $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$CI_COMMIT_TIMESTAMP \ + - docker buildx imagetools create + --annotation "index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA" + --annotation "index:org.opencontainers.image.revision=$CI_COMMIT_SHA" + --annotation "index:org.opencontainers.image.source=$CI_PROJECT_URL" + --annotation "index:org.opencontainers.image.created=$CI_JOB_STARTED_AT" + --tag $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$CI_COMMIT_TIMESTAMP $DOCKER_IMAGE_FULL_TAG .python-typing: From de9798f03f29039bb2e376e7896fb5821b45979b Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 12:18:00 +0000 Subject: [PATCH 35/62] Update docker image version Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 6ae33403..365d013c 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -9,9 +9,9 @@ variables: # TODO: Sign image using Cosign # TODO: Use ARM64 workers and build image without QEMU: https://docs.gitlab.com/ee/ci/runners/hosted_runners/linux.html .build-and-push-gitlab: - image: docker:24.0.2 + image: docker:27.4 services: - - docker:24.0.2-dind + - docker:27.4-dind variables: DOCKER_BUILDKIT: 1 DOCKER_PLATFORM: "linux/amd64" @@ -34,9 +34,9 @@ variables: # Architectures are hardcoded for multiarch, need to make this better .multiarch-manifest-gitlab: - image: docker:24.0.2 + image: docker:27.4 services: - - docker:24.0.2-dind + - docker:27.4-dind script: - docker manifest create $DOCKER_IMAGE_FULL_TAG \ --amend $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG-amd64 \ @@ -45,12 +45,12 @@ variables: - docker manifest push $DOCKER_IMAGE_FULL_TAG .promote-image: - image: docker:24.0.2 + image: docker:27.4 variables: PROMOTED_ENVIRONMENT: "dev" DOCKER_BUILDKIT: 1 services: - - docker:24.0.2-dind + - docker:27.4-dind script: # - desired_format="%s" # - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" + "$desired_format") From 74f2c2ef6336e9d258e37a24a4a0597bdfcc4582 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 12:19:51 +0000 Subject: [PATCH 36/62] Use variable for docker version Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 365d013c..26102af7 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -4,14 +4,15 @@ variables: DOCKER_IMAGE_TAG_PREFIX: "" DOCKER_IMAGE_TAG_SUFFIX: "" DOCKER_IMAGE_FULL_TAG: $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX + DOCKER_VERSION: 27.4 # Build Docker image for test # TODO: Sign image using Cosign # TODO: Use ARM64 workers and build image without QEMU: https://docs.gitlab.com/ee/ci/runners/hosted_runners/linux.html .build-and-push-gitlab: - image: docker:27.4 + image: docker:$DOCKER_VERSION services: - - docker:27.4-dind + - docker:$DOCKER_VERSION-dind variables: DOCKER_BUILDKIT: 1 DOCKER_PLATFORM: "linux/amd64" @@ -34,9 +35,9 @@ variables: # Architectures are hardcoded for multiarch, need to make this better .multiarch-manifest-gitlab: - image: docker:27.4 + image: docker:$DOCKER_VERSION services: - - docker:27.4-dind + - docker:$DOCKER_VERSION-dind script: - docker manifest create $DOCKER_IMAGE_FULL_TAG \ --amend $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG-amd64 \ @@ -45,12 +46,12 @@ variables: - docker manifest push $DOCKER_IMAGE_FULL_TAG .promote-image: - image: docker:27.4 + image: docker:$DOCKER_VERSION variables: PROMOTED_ENVIRONMENT: "dev" DOCKER_BUILDKIT: 1 services: - - docker:27.4-dind + - docker:$DOCKER_VERSION-dind script: # - desired_format="%s" # - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" + "$desired_format") From 08edaf958b918df16e3c660d9aa210a44f423719 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 12:28:47 +0000 Subject: [PATCH 37/62] Add quotes Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 26102af7..3272051d 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -61,8 +61,8 @@ variables: --annotation "index:org.opencontainers.image.revision=$CI_COMMIT_SHA" --annotation "index:org.opencontainers.image.source=$CI_PROJECT_URL" --annotation "index:org.opencontainers.image.created=$CI_JOB_STARTED_AT" - --tag $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$CI_COMMIT_TIMESTAMP - $DOCKER_IMAGE_FULL_TAG + --tag "$DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$CI_COMMIT_TIMESTAMP" + "$DOCKER_IMAGE_FULL_TAG" .python-typing: image: $DOCKER_IMAGE_FULL_TAG From d4ca0730b95f6a3afade2573f9811537acfc74f0 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 12:37:29 +0000 Subject: [PATCH 38/62] Remove TODO Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 3272051d..5e77b289 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -8,7 +8,6 @@ variables: # Build Docker image for test # TODO: Sign image using Cosign -# TODO: Use ARM64 workers and build image without QEMU: https://docs.gitlab.com/ee/ci/runners/hosted_runners/linux.html .build-and-push-gitlab: image: docker:$DOCKER_VERSION services: From 1baada2c3feab486e5c07d9e430a6faac3f47729 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 12:44:06 +0000 Subject: [PATCH 39/62] Prepare timestamp and add debug printout Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 5e77b289..023d05fa 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -52,9 +52,9 @@ variables: services: - docker:$DOCKER_VERSION-dind script: -# - desired_format="%s" -# - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" + "$desired_format") +# - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" -D "%Y-%m-%dT%H:%M:%SZ"+%s) # - echo "Unix timestamp - $UNIX_TIMESTAMP" + - echo "Tagging $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$CI_COMMIT_TIMESTAMP from $DOCKER_IMAGE_FULL_TAG" - docker buildx imagetools create --annotation "index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA" --annotation "index:org.opencontainers.image.revision=$CI_COMMIT_SHA" From f9a6feec5cd025f243bcd9d73695cc0a3986fad3 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 12:45:46 +0000 Subject: [PATCH 40/62] Temporarily disable test stage Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/base.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab_ci/base.yml b/.gitlab_ci/base.yml index 660333f1..dcc361ae 100644 --- a/.gitlab_ci/base.yml +++ b/.gitlab_ci/base.yml @@ -7,11 +7,11 @@ variables: stages: - build - - test +# - test - deploy include: - local: /.gitlab_ci/_templates.yml - local: /.gitlab_ci/build.yml - - local: /.gitlab_ci/test.yml +# - local: /.gitlab_ci/test.yml - local: /.gitlab_ci/deploy.yml From 4819e394956cb7cd6e5f04c6a2250d1b5f2fa49f Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 13:07:18 +0000 Subject: [PATCH 41/62] Transform date in unix timestamp Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 023d05fa..9cac8512 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -52,15 +52,18 @@ variables: services: - docker:$DOCKER_VERSION-dind script: -# - export UNIX_TIMESTAMP=$(date -d "$CI_COMMIT_TIMESTAMP" -D "%Y-%m-%dT%H:%M:%SZ"+%s) -# - echo "Unix timestamp - $UNIX_TIMESTAMP" - - echo "Tagging $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$CI_COMMIT_TIMESTAMP from $DOCKER_IMAGE_FULL_TAG" + # Remove the UTC offset, not supported by `date` in docker image (busybox) + - export CLEAN_DATETIME=$(echo "$CI_COMMIT_TIMESTAMP" | sed 's/+00:00//' | sed 's/Z//') + # Transform in unix timestamp + - export UNIX_TIMESTAMP=$(date -d "$CLEAN_DATETIME" -D "%Y-%m-%dT%H:%M:%SZ"+%s) + - echo "Unix timestamp - $UNIX_TIMESTAMP" + - echo "Tagging $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP from $DOCKER_IMAGE_FULL_TAG" - docker buildx imagetools create --annotation "index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA" --annotation "index:org.opencontainers.image.revision=$CI_COMMIT_SHA" --annotation "index:org.opencontainers.image.source=$CI_PROJECT_URL" --annotation "index:org.opencontainers.image.created=$CI_JOB_STARTED_AT" - --tag "$DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$CI_COMMIT_TIMESTAMP" + --tag "$DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP" "$DOCKER_IMAGE_FULL_TAG" .python-typing: From 6859da5cbba658d17f1611b93dd96fe0e2e62a3a Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 13:13:33 +0000 Subject: [PATCH 42/62] Login into registry Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 9cac8512..dc128eb2 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -52,6 +52,7 @@ variables: services: - docker:$DOCKER_VERSION-dind script: + - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com # Remove the UTC offset, not supported by `date` in docker image (busybox) - export CLEAN_DATETIME=$(echo "$CI_COMMIT_TIMESTAMP" | sed 's/+00:00//' | sed 's/Z//') # Transform in unix timestamp From a9480e4f9ee57d4d152d53d8b5e7fdee19584994 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 13:18:59 +0000 Subject: [PATCH 43/62] Fix date format Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index dc128eb2..a19d1832 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -56,7 +56,7 @@ variables: # Remove the UTC offset, not supported by `date` in docker image (busybox) - export CLEAN_DATETIME=$(echo "$CI_COMMIT_TIMESTAMP" | sed 's/+00:00//' | sed 's/Z//') # Transform in unix timestamp - - export UNIX_TIMESTAMP=$(date -d "$CLEAN_DATETIME" -D "%Y-%m-%dT%H:%M:%SZ"+%s) + - export UNIX_TIMESTAMP=$(date -d "$CLEAN_DATETIME" -D "%Y-%m-%dT%H:%M:%S"+%s) - echo "Unix timestamp - $UNIX_TIMESTAMP" - echo "Tagging $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP from $DOCKER_IMAGE_FULL_TAG" - docker buildx imagetools create From 8890f26d91ed946b2c1e018273842cd0e8336bb9 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 13:25:19 +0000 Subject: [PATCH 44/62] Missing space Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index a19d1832..befcce3b 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -56,7 +56,7 @@ variables: # Remove the UTC offset, not supported by `date` in docker image (busybox) - export CLEAN_DATETIME=$(echo "$CI_COMMIT_TIMESTAMP" | sed 's/+00:00//' | sed 's/Z//') # Transform in unix timestamp - - export UNIX_TIMESTAMP=$(date -d "$CLEAN_DATETIME" -D "%Y-%m-%dT%H:%M:%S"+%s) + - export UNIX_TIMESTAMP=$(date -d "$CLEAN_DATETIME" -D "%Y-%m-%dT%H:%M:%S" +%s) - echo "Unix timestamp - $UNIX_TIMESTAMP" - echo "Tagging $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP from $DOCKER_IMAGE_FULL_TAG" - docker buildx imagetools create From bf5a00fc4c9c86b44b789a45176001b00674ca6a Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 14:21:21 +0000 Subject: [PATCH 45/62] Use variable for login Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index befcce3b..6d4452cb 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -21,7 +21,7 @@ variables: script: - docker buildx create --use - docker buildx inspect --bootstrap - - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_PASSWORD --password-stdin $CI_REGISTRY - echo "Building $DOCKER_IMAGE_FULL_TAG - Cache from $DOCKER_IMAGE_NAME:cache" - | docker buildx build --push \ @@ -52,7 +52,7 @@ variables: services: - docker:$DOCKER_VERSION-dind script: - - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_PASSWORD --password-stdin $CI_REGISTRY # Remove the UTC offset, not supported by `date` in docker image (busybox) - export CLEAN_DATETIME=$(echo "$CI_COMMIT_TIMESTAMP" | sed 's/+00:00//' | sed 's/Z//') # Transform in unix timestamp From 462e601f028b20c1c6328ffdd53ecb321cf8e84e Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 14:47:08 +0000 Subject: [PATCH 46/62] Rearrange variables Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 16 ++++++++-------- .gitlab_ci/build.yml | 12 +++++------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 6d4452cb..60cdb793 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -1,9 +1,7 @@ variables: - DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE DOCKER_IMAGE_TAG: $CI_COMMIT_SHA - DOCKER_IMAGE_TAG_PREFIX: "" - DOCKER_IMAGE_TAG_SUFFIX: "" - DOCKER_IMAGE_FULL_TAG: $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG$DOCKER_IMAGE_TAG_SUFFIX + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME + DOCKER_IMAGE_FULL_TAG: $CI_REGISTRY_IMAGE/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG DOCKER_VERSION: 27.4 # Build Docker image for test @@ -15,6 +13,7 @@ variables: variables: DOCKER_BUILDKIT: 1 DOCKER_PLATFORM: "linux/amd64" + # TODO: Make target optional DOCKER_TARGET: dev before_script: - apk add --no-cache bash git @@ -23,10 +22,12 @@ variables: - docker buildx inspect --bootstrap - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_PASSWORD --password-stdin $CI_REGISTRY - echo "Building $DOCKER_IMAGE_FULL_TAG - Cache from $DOCKER_IMAGE_NAME:cache" + # remove \ from platform to + - export SUFFIX=$(echo $DOCKER_PLATFORM | sed 's/\///') - | docker buildx build --push \ --target $DOCKER_TARGET \ - --tag $DOCKER_IMAGE_FULL_TAG \ + --tag $DOCKER_IMAGE_FULL_TAG-$SUFFIX \ --platform $DOCKER_PLATFORM \ --cache-from type=registry,ref=$DOCKER_IMAGE_NAME:cache \ --cache-to type=registry,ref=$DOCKER_IMAGE_NAME:cache \ @@ -39,9 +40,8 @@ variables: - docker:$DOCKER_VERSION-dind script: - docker manifest create $DOCKER_IMAGE_FULL_TAG \ - --amend $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG-amd64 \ - --amend $DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG_PREFIX$DOCKER_IMAGE_TAG-arm64 - + --amend $DOCKER_IMAGE_FULL_TAG-linuxamd64 \ + --amend $DOCKER_IMAGE_FULL_TAG-linuxarm64 - docker manifest push $DOCKER_IMAGE_FULL_TAG .promote-image: diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml index df5ae770..945370e4 100644 --- a/.gitlab_ci/build.yml +++ b/.gitlab_ci/build.yml @@ -2,7 +2,7 @@ build-test: stage: build variables: - DOCKER_IMAGE_TAG_PREFIX: "test-" + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-test rules: # We run the pipeline only on merge requests or the `main` branch - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH @@ -10,12 +10,11 @@ build-test: extends: - .build-and-push-gitlab -# TODO: Group the multi-arch bild (perhaps with a nested workflow) +# TODO: Make the multi-arch build in a single job (perhaps with a nested workflow) build-http-app-amd64: stage: build variables: - DOCKER_IMAGE_TAG_PREFIX: "http-" - DOCKER_IMAGE_TAG_SUFFIX: "-amd64" + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-http DOCKER_PLATFORM: "linux/amd64" DOCKER_TARGET: http_app tags: @@ -30,8 +29,7 @@ build-http-app-amd64: build-http-app-arm64: stage: build variables: - DOCKER_IMAGE_TAG_PREFIX: "http-" - DOCKER_IMAGE_TAG_SUFFIX: "-arm64" + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-http DOCKER_PLATFORM: "linux/arm64" DOCKER_TARGET: http_app tags: @@ -49,7 +47,7 @@ aggregate-http-manifests: - build-http-app-amd64 - build-http-app-arm64 variables: - DOCKER_IMAGE_TAG_PREFIX: "http-" + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-http rules: # We run the pipeline only on merge requests or the `main` branch - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH From 14477c42e8115acb7242d8b0643036d732b726cf Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 14:47:36 +0000 Subject: [PATCH 47/62] Fix multiarch step extends Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml index 945370e4..89d08302 100644 --- a/.gitlab_ci/build.yml +++ b/.gitlab_ci/build.yml @@ -53,4 +53,4 @@ aggregate-http-manifests: - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == "merge_request_event" extends: - - .build-and-push-gitlab + - .multiarch-manifest-gitlab From 1310c545906373f84e183331bd294b9f78a1fb94 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 14:53:16 +0000 Subject: [PATCH 48/62] Fix docker login Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 60cdb793..e2d1aa00 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -20,7 +20,7 @@ variables: script: - docker buildx create --use - docker buildx inspect --bootstrap - - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_PASSWORD --password-stdin $CI_REGISTRY + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - echo "Building $DOCKER_IMAGE_FULL_TAG - Cache from $DOCKER_IMAGE_NAME:cache" # remove \ from platform to - export SUFFIX=$(echo $DOCKER_PLATFORM | sed 's/\///') @@ -39,6 +39,7 @@ variables: services: - docker:$DOCKER_VERSION-dind script: + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - docker manifest create $DOCKER_IMAGE_FULL_TAG \ --amend $DOCKER_IMAGE_FULL_TAG-linuxamd64 \ --amend $DOCKER_IMAGE_FULL_TAG-linuxarm64 @@ -52,7 +53,7 @@ variables: services: - docker:$DOCKER_VERSION-dind script: - - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_PASSWORD --password-stdin $CI_REGISTRY + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY # Remove the UTC offset, not supported by `date` in docker image (busybox) - export CLEAN_DATETIME=$(echo "$CI_COMMIT_TIMESTAMP" | sed 's/+00:00//' | sed 's/Z//') # Transform in unix timestamp From cccd9031f4799aa7d41d80fb9ff279d71834b18e Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 15:12:23 +0000 Subject: [PATCH 49/62] Make target and platform optional Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index e2d1aa00..720cc9b9 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -12,9 +12,8 @@ variables: - docker:$DOCKER_VERSION-dind variables: DOCKER_BUILDKIT: 1 - DOCKER_PLATFORM: "linux/amd64" - # TODO: Make target optional - DOCKER_TARGET: dev + DOCKER_PLATFORM: "" + DOCKER_TARGET: "" before_script: - apk add --no-cache bash git script: @@ -22,13 +21,16 @@ variables: - docker buildx inspect --bootstrap - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - echo "Building $DOCKER_IMAGE_FULL_TAG - Cache from $DOCKER_IMAGE_NAME:cache" + - if [[ -n "$DOCKER_TARGET" ]]; then export TARGET_ARG="--target $DOCKER_TARGET"; fi; + - if [[ -n "$DOCKER_PLATFORM" ]]; then export PLATFORM_ARG="--platform $DOCKER_PLATFORM"; fi; + - if [[ -n "$DOCKER_PLATFORM" ]]; then export PLATFORM_SUFFIX="-$(echo $DOCKER_PLATFORM | sed 's/\///')"; fi; # remove \ from platform to - export SUFFIX=$(echo $DOCKER_PLATFORM | sed 's/\///') - | docker buildx build --push \ - --target $DOCKER_TARGET \ - --tag $DOCKER_IMAGE_FULL_TAG-$SUFFIX \ - --platform $DOCKER_PLATFORM \ + $TARGET_ARG \ + --tag $DOCKER_IMAGE_FULL_TAG$PLATFORM_SUFFIX \ + $PLATFORM_ARG \ --cache-from type=registry,ref=$DOCKER_IMAGE_NAME:cache \ --cache-to type=registry,ref=$DOCKER_IMAGE_NAME:cache \ . From dbafef47eb8b66f78225b05a484b83455e6cb180 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 15:17:09 +0000 Subject: [PATCH 50/62] Template for docker login Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 720cc9b9..a174f0d4 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -4,6 +4,9 @@ variables: DOCKER_IMAGE_FULL_TAG: $CI_REGISTRY_IMAGE/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG DOCKER_VERSION: 27.4 +.docker-gitlab-login: &docker-gitlab-login + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + # Build Docker image for test # TODO: Sign image using Cosign .build-and-push-gitlab: @@ -19,7 +22,7 @@ variables: script: - docker buildx create --use - docker buildx inspect --bootstrap - - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + - *docker-gitlab-login - echo "Building $DOCKER_IMAGE_FULL_TAG - Cache from $DOCKER_IMAGE_NAME:cache" - if [[ -n "$DOCKER_TARGET" ]]; then export TARGET_ARG="--target $DOCKER_TARGET"; fi; - if [[ -n "$DOCKER_PLATFORM" ]]; then export PLATFORM_ARG="--platform $DOCKER_PLATFORM"; fi; @@ -41,7 +44,7 @@ variables: services: - docker:$DOCKER_VERSION-dind script: - - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + - *docker-gitlab-login - docker manifest create $DOCKER_IMAGE_FULL_TAG \ --amend $DOCKER_IMAGE_FULL_TAG-linuxamd64 \ --amend $DOCKER_IMAGE_FULL_TAG-linuxarm64 @@ -55,7 +58,7 @@ variables: services: - docker:$DOCKER_VERSION-dind script: - - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + - *docker-gitlab-login # Remove the UTC offset, not supported by `date` in docker image (busybox) - export CLEAN_DATETIME=$(echo "$CI_COMMIT_TIMESTAMP" | sed 's/+00:00//' | sed 's/Z//') # Transform in unix timestamp From a0d4c4c12157c8cd84b5dfd27a2506565b9cd6c8 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 15:19:18 +0000 Subject: [PATCH 51/62] Fix cache Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index a174f0d4..0e807c74 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -17,13 +17,14 @@ variables: DOCKER_BUILDKIT: 1 DOCKER_PLATFORM: "" DOCKER_TARGET: "" + DOCKER_CACHE_FULL_TAG: $CI_REGISTRY_IMAGE/$DOCKER_IMAGE_NAME:cache before_script: - apk add --no-cache bash git script: - docker buildx create --use - docker buildx inspect --bootstrap - *docker-gitlab-login - - echo "Building $DOCKER_IMAGE_FULL_TAG - Cache from $DOCKER_IMAGE_NAME:cache" + - echo "Building $DOCKER_IMAGE_FULL_TAG - Cache from $DOCKER_CACHE_FULL_TAG" - if [[ -n "$DOCKER_TARGET" ]]; then export TARGET_ARG="--target $DOCKER_TARGET"; fi; - if [[ -n "$DOCKER_PLATFORM" ]]; then export PLATFORM_ARG="--platform $DOCKER_PLATFORM"; fi; - if [[ -n "$DOCKER_PLATFORM" ]]; then export PLATFORM_SUFFIX="-$(echo $DOCKER_PLATFORM | sed 's/\///')"; fi; @@ -34,8 +35,8 @@ variables: $TARGET_ARG \ --tag $DOCKER_IMAGE_FULL_TAG$PLATFORM_SUFFIX \ $PLATFORM_ARG \ - --cache-from type=registry,ref=$DOCKER_IMAGE_NAME:cache \ - --cache-to type=registry,ref=$DOCKER_IMAGE_NAME:cache \ + --cache-from type=registry,ref=$DOCKER_CACHE_FULL_TAG \ + --cache-to type=registry,ref=$DOCKER_CACHE_FULL_TAG \ . # Architectures are hardcoded for multiarch, need to make this better From fb7f4d7804fa4b92063754a6fefffdd05d4e89c5 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 15:26:25 +0000 Subject: [PATCH 52/62] Add debug line Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 0e807c74..7fd0e36c 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -46,6 +46,7 @@ variables: - docker:$DOCKER_VERSION-dind script: - *docker-gitlab-login + - echo "Building $DOCKER_IMAGE_FULL_TAG multiarch manifest" - docker manifest create $DOCKER_IMAGE_FULL_TAG \ --amend $DOCKER_IMAGE_FULL_TAG-linuxamd64 \ --amend $DOCKER_IMAGE_FULL_TAG-linuxarm64 From 6ada7421cd5457f55dec348b9be1985f0dd20b7f Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 15:42:48 +0000 Subject: [PATCH 53/62] Remvoe other multiline \ Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 7fd0e36c..b4955066 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -28,15 +28,14 @@ variables: - if [[ -n "$DOCKER_TARGET" ]]; then export TARGET_ARG="--target $DOCKER_TARGET"; fi; - if [[ -n "$DOCKER_PLATFORM" ]]; then export PLATFORM_ARG="--platform $DOCKER_PLATFORM"; fi; - if [[ -n "$DOCKER_PLATFORM" ]]; then export PLATFORM_SUFFIX="-$(echo $DOCKER_PLATFORM | sed 's/\///')"; fi; - # remove \ from platform to + # remove \ from platform variable - export SUFFIX=$(echo $DOCKER_PLATFORM | sed 's/\///') - - | - docker buildx build --push \ - $TARGET_ARG \ - --tag $DOCKER_IMAGE_FULL_TAG$PLATFORM_SUFFIX \ - $PLATFORM_ARG \ - --cache-from type=registry,ref=$DOCKER_CACHE_FULL_TAG \ - --cache-to type=registry,ref=$DOCKER_CACHE_FULL_TAG \ + - docker buildx build --push + $TARGET_ARG + --tag $DOCKER_IMAGE_FULL_TAG$PLATFORM_SUFFIX + $PLATFORM_ARG + --cache-from type=registry,ref=$DOCKER_CACHE_FULL_TAG + --cache-to type=registry,ref=$DOCKER_CACHE_FULL_TAG . # Architectures are hardcoded for multiarch, need to make this better @@ -47,8 +46,8 @@ variables: script: - *docker-gitlab-login - echo "Building $DOCKER_IMAGE_FULL_TAG multiarch manifest" - - docker manifest create $DOCKER_IMAGE_FULL_TAG \ - --amend $DOCKER_IMAGE_FULL_TAG-linuxamd64 \ + - docker manifest create $DOCKER_IMAGE_FULL_TAG + --amend $DOCKER_IMAGE_FULL_TAG-linuxamd64 --amend $DOCKER_IMAGE_FULL_TAG-linuxarm64 - docker manifest push $DOCKER_IMAGE_FULL_TAG From e91b6d17b817eca30e22926baba78edb7f5d7249 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 16:06:32 +0000 Subject: [PATCH 54/62] Aggregate manifests using buildx Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 7 +++---- .gitlab_ci/build.yml | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index b4955066..04179367 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -46,10 +46,9 @@ variables: script: - *docker-gitlab-login - echo "Building $DOCKER_IMAGE_FULL_TAG multiarch manifest" - - docker manifest create $DOCKER_IMAGE_FULL_TAG - --amend $DOCKER_IMAGE_FULL_TAG-linuxamd64 - --amend $DOCKER_IMAGE_FULL_TAG-linuxarm64 - - docker manifest push $DOCKER_IMAGE_FULL_TAG + - docker buildx imagetools create $DOCKER_IMAGE_FULL_TAG + $DOCKER_IMAGE_FULL_TAG-linuxamd64 + $DOCKER_IMAGE_FULL_TAG-linuxarm64 .promote-image: image: docker:$DOCKER_VERSION diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml index 89d08302..099274da 100644 --- a/.gitlab_ci/build.yml +++ b/.gitlab_ci/build.yml @@ -54,3 +54,4 @@ aggregate-http-manifests: - if: $CI_PIPELINE_SOURCE == "merge_request_event" extends: - .multiarch-manifest-gitlab + From 20b627c6093830d9ad3f1c64616f440686f9b764 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 16:10:04 +0000 Subject: [PATCH 55/62] Fix manifest aggregation Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 04179367..98b7429c 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -46,7 +46,8 @@ variables: script: - *docker-gitlab-login - echo "Building $DOCKER_IMAGE_FULL_TAG multiarch manifest" - - docker buildx imagetools create $DOCKER_IMAGE_FULL_TAG + - docker buildx imagetools create + --tag $DOCKER_IMAGE_FULL_TAG $DOCKER_IMAGE_FULL_TAG-linuxamd64 $DOCKER_IMAGE_FULL_TAG-linuxarm64 From 395ac9dfaa13a53f87e7f5ec29a8f5b81f4a6e93 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 16:17:15 +0000 Subject: [PATCH 56/62] Remove unnecessary quotes Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 98b7429c..5d9a0e42 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -67,12 +67,12 @@ variables: - echo "Unix timestamp - $UNIX_TIMESTAMP" - echo "Tagging $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP from $DOCKER_IMAGE_FULL_TAG" - docker buildx imagetools create - --annotation "index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA" - --annotation "index:org.opencontainers.image.revision=$CI_COMMIT_SHA" - --annotation "index:org.opencontainers.image.source=$CI_PROJECT_URL" - --annotation "index:org.opencontainers.image.created=$CI_JOB_STARTED_AT" - --tag "$DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP" - "$DOCKER_IMAGE_FULL_TAG" + --annotation index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA + --annotation index:org.opencontainers.image.revision=$CI_COMMIT_SHA + --annotation index:org.opencontainers.image.source=$CI_PROJECT_URL + --annotation index:org.opencontainers.image.created=$CI_JOB_STARTED_AT + --tag $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP + $DOCKER_IMAGE_FULL_TAG .python-typing: image: $DOCKER_IMAGE_FULL_TAG From dcd1e8253656b49f7cbe85142141789d73ba7d87 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 16:17:46 +0000 Subject: [PATCH 57/62] Use DOCKER_IMAGE_NAME in promote step Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab_ci/deploy.yml b/.gitlab_ci/deploy.yml index ebaa3d15..154b8b04 100644 --- a/.gitlab_ci/deploy.yml +++ b/.gitlab_ci/deploy.yml @@ -1,7 +1,7 @@ promote-dev: stage: deploy variables: - DOCKER_IMAGE_TAG_PREFIX: "http-" + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-http rules: # We run the pipeline only on merge requests or the `main` branch - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH From 29c7b82027453f5fa5f6361ce40f0ec075e79aa0 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 16:22:36 +0000 Subject: [PATCH 58/62] Fix registry in promote step Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 5d9a0e42..15345e6e 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -65,13 +65,13 @@ variables: # Transform in unix timestamp - export UNIX_TIMESTAMP=$(date -d "$CLEAN_DATETIME" -D "%Y-%m-%dT%H:%M:%S" +%s) - echo "Unix timestamp - $UNIX_TIMESTAMP" - - echo "Tagging $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP from $DOCKER_IMAGE_FULL_TAG" + - echo "Tagging $CI_REGISTRY_IMAGE/$DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP from $DOCKER_IMAGE_FULL_TAG" - docker buildx imagetools create --annotation index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA --annotation index:org.opencontainers.image.revision=$CI_COMMIT_SHA --annotation index:org.opencontainers.image.source=$CI_PROJECT_URL --annotation index:org.opencontainers.image.created=$CI_JOB_STARTED_AT - --tag $DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP + --tag $CI_REGISTRY_IMAGE/$DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP $DOCKER_IMAGE_FULL_TAG .python-typing: From 23327a7c9ea9ab5579ad3913f83b6ea690bc9f95 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 16:32:35 +0000 Subject: [PATCH 59/62] Create image using job datetime so it can be executed again Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/_templates.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml index 15345e6e..a4f1a913 100644 --- a/.gitlab_ci/_templates.yml +++ b/.gitlab_ci/_templates.yml @@ -61,7 +61,7 @@ variables: script: - *docker-gitlab-login # Remove the UTC offset, not supported by `date` in docker image (busybox) - - export CLEAN_DATETIME=$(echo "$CI_COMMIT_TIMESTAMP" | sed 's/+00:00//' | sed 's/Z//') + - export CLEAN_DATETIME=$(echo "$CI_JOB_STARTED_AT" | sed 's/+00:00//' | sed 's/Z//') # Transform in unix timestamp - export UNIX_TIMESTAMP=$(date -d "$CLEAN_DATETIME" -D "%Y-%m-%dT%H:%M:%S" +%s) - echo "Unix timestamp - $UNIX_TIMESTAMP" From 6e52de937d89e134e5c7154f82a18b29bd80c630 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 16:42:08 +0000 Subject: [PATCH 60/62] Re-enable tests Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/base.yml | 4 ++-- .gitlab_ci/test.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab_ci/base.yml b/.gitlab_ci/base.yml index dcc361ae..660333f1 100644 --- a/.gitlab_ci/base.yml +++ b/.gitlab_ci/base.yml @@ -7,11 +7,11 @@ variables: stages: - build -# - test + - test - deploy include: - local: /.gitlab_ci/_templates.yml - local: /.gitlab_ci/build.yml -# - local: /.gitlab_ci/test.yml + - local: /.gitlab_ci/test.yml - local: /.gitlab_ci/deploy.yml diff --git a/.gitlab_ci/test.yml b/.gitlab_ci/test.yml index f901363e..6fe1e40b 100644 --- a/.gitlab_ci/test.yml +++ b/.gitlab_ci/test.yml @@ -2,7 +2,7 @@ typing: stage: test variables: - DOCKER_IMAGE_TAG_PREFIX: "test-" + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-test rules: # We run the pipeline only on merge requests or the `main` branch - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH @@ -13,7 +13,7 @@ typing: lint: stage: test variables: - DOCKER_IMAGE_TAG_PREFIX: "test-" + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-test rules: # We run the pipeline only on merge requests or the `main` branch - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH @@ -24,7 +24,7 @@ lint: format: stage: test variables: - DOCKER_IMAGE_TAG_PREFIX: "test-" + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-test rules: # We run the pipeline only on merge requests or the `main` branch - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH @@ -35,7 +35,7 @@ format: tests: stage: test variables: - DOCKER_IMAGE_TAG_PREFIX: "test-" + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-test rules: # We run the pipeline only on merge requests or the `main` branch - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH From 59199a16de47c6984f50fe1bc6a46d2bec67a0d1 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 16:47:02 +0000 Subject: [PATCH 61/62] Use dev target for tests Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- .gitlab_ci/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml index 099274da..600fcb14 100644 --- a/.gitlab_ci/build.yml +++ b/.gitlab_ci/build.yml @@ -3,6 +3,7 @@ build-test: stage: build variables: DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-test + DOCKER_TARGET: dev rules: # We run the pipeline only on merge requests or the `main` branch - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH From 1b907bb9122976022c665cbee549045fc86759a2 Mon Sep 17 00:00:00 2001 From: Federico Busetti <729029+febus982@users.noreply.github.com> Date: Wed, 1 Jan 2025 17:13:26 +0000 Subject: [PATCH 62/62] Mention github pipeline in the README.md Signed-off-by: Federico Busetti <729029+febus982@users.noreply.github.com> --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ec7bbe72..58d53843 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ This template provides out of the box some commonly used functionalities: * Repository pattern for databases using [SQLAlchemy](https://www.sqlalchemy.org/) and [SQLAlchemy bind manager](https://febus982.github.io/sqlalchemy-bind-manager/stable/) * Database migrations using [Alembic](https://alembic.sqlalchemy.org/en/latest/) (configured supporting both sync and async SQLAlchemy engines) * Authentication and Identity Provider using [ORY Zero Trust architecture](https://www.ory.sh/docs/kratos/guides/zero-trust-iap-proxy-identity-access-proxy) +* Example CI/CD deployment pipeline for GitLab (The focus for this repository is still GitHub but, in case you want to use GitLab 🤷) * [TODO] Producer and consumer to emit and consume events using [CloudEvents](https://cloudevents.io/) format on [Confluent Kafka](https://docs.confluent.io/kafka-clients/python/current/overview.html) ## Documentation