diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..6228992c --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,2 @@ +include: + - local: .gitlab_ci/base.yml diff --git a/.gitlab_ci/_templates.yml b/.gitlab_ci/_templates.yml new file mode 100644 index 00000000..a4f1a913 --- /dev/null +++ b/.gitlab_ci/_templates.yml @@ -0,0 +1,96 @@ +variables: + DOCKER_IMAGE_TAG: $CI_COMMIT_SHA + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME + DOCKER_IMAGE_FULL_TAG: $CI_REGISTRY_IMAGE/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_TAG + DOCKER_VERSION: 27.4 + +.docker-gitlab-login: &docker-gitlab-login + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + +# Build Docker image for test +# TODO: Sign image using Cosign +.build-and-push-gitlab: + image: docker:$DOCKER_VERSION + services: + - docker:$DOCKER_VERSION-dind + variables: + DOCKER_BUILDKIT: 1 + DOCKER_PLATFORM: "" + DOCKER_TARGET: "" + DOCKER_CACHE_FULL_TAG: $CI_REGISTRY_IMAGE/$DOCKER_IMAGE_NAME:cache + before_script: + - apk add --no-cache bash git + script: + - docker buildx create --use + - docker buildx inspect --bootstrap + - *docker-gitlab-login + - echo "Building $DOCKER_IMAGE_FULL_TAG - Cache from $DOCKER_CACHE_FULL_TAG" + - if [[ -n "$DOCKER_TARGET" ]]; then export TARGET_ARG="--target $DOCKER_TARGET"; fi; + - if [[ -n "$DOCKER_PLATFORM" ]]; then export PLATFORM_ARG="--platform $DOCKER_PLATFORM"; fi; + - if [[ -n "$DOCKER_PLATFORM" ]]; then export PLATFORM_SUFFIX="-$(echo $DOCKER_PLATFORM | sed 's/\///')"; fi; + # remove \ from platform variable + - export SUFFIX=$(echo $DOCKER_PLATFORM | sed 's/\///') + - docker buildx build --push + $TARGET_ARG + --tag $DOCKER_IMAGE_FULL_TAG$PLATFORM_SUFFIX + $PLATFORM_ARG + --cache-from type=registry,ref=$DOCKER_CACHE_FULL_TAG + --cache-to type=registry,ref=$DOCKER_CACHE_FULL_TAG + . + +# Architectures are hardcoded for multiarch, need to make this better +.multiarch-manifest-gitlab: + image: docker:$DOCKER_VERSION + services: + - docker:$DOCKER_VERSION-dind + script: + - *docker-gitlab-login + - echo "Building $DOCKER_IMAGE_FULL_TAG multiarch manifest" + - docker buildx imagetools create + --tag $DOCKER_IMAGE_FULL_TAG + $DOCKER_IMAGE_FULL_TAG-linuxamd64 + $DOCKER_IMAGE_FULL_TAG-linuxarm64 + +.promote-image: + image: docker:$DOCKER_VERSION + variables: + PROMOTED_ENVIRONMENT: "dev" + DOCKER_BUILDKIT: 1 + services: + - docker:$DOCKER_VERSION-dind + script: + - *docker-gitlab-login + # Remove the UTC offset, not supported by `date` in docker image (busybox) + - export CLEAN_DATETIME=$(echo "$CI_JOB_STARTED_AT" | sed 's/+00:00//' | sed 's/Z//') + # Transform in unix timestamp + - export UNIX_TIMESTAMP=$(date -d "$CLEAN_DATETIME" -D "%Y-%m-%dT%H:%M:%S" +%s) + - echo "Unix timestamp - $UNIX_TIMESTAMP" + - echo "Tagging $CI_REGISTRY_IMAGE/$DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP from $DOCKER_IMAGE_FULL_TAG" + - docker buildx imagetools create + --annotation index:org.opencontainers.image.version=$CI_COMMIT_SHORT_SHA + --annotation index:org.opencontainers.image.revision=$CI_COMMIT_SHA + --annotation index:org.opencontainers.image.source=$CI_PROJECT_URL + --annotation index:org.opencontainers.image.created=$CI_JOB_STARTED_AT + --tag $CI_REGISTRY_IMAGE/$DOCKER_IMAGE_NAME:$PROMOTED_ENVIRONMENT-$UNIX_TIMESTAMP + $DOCKER_IMAGE_FULL_TAG + +.python-typing: + image: $DOCKER_IMAGE_FULL_TAG + script: + - make typing + +.python-lint: + image: $DOCKER_IMAGE_FULL_TAG + script: + - make lint + +.python-format: + image: $DOCKER_IMAGE_FULL_TAG + script: + - make format + +.python-tests: + image: $DOCKER_IMAGE_FULL_TAG + script: + - make test + diff --git a/.gitlab_ci/base.yml b/.gitlab_ci/base.yml new file mode 100644 index 00000000..660333f1 --- /dev/null +++ b/.gitlab_ci/base.yml @@ -0,0 +1,17 @@ +variables: + # Use docker.io for Docker Hub if empty + REGISTRY: registry.gitlab.com + # IMAGE_NAME is defined as / in GitLab CI/CD + IMAGE_NAME: $CI_REGISTRY_IMAGE + TEST_TAG: $REGISTRY/$CI_PROJECT_PATH:test + +stages: + - build + - test + - deploy + +include: + - local: /.gitlab_ci/_templates.yml + - local: /.gitlab_ci/build.yml + - local: /.gitlab_ci/test.yml + - local: /.gitlab_ci/deploy.yml diff --git a/.gitlab_ci/build.yml b/.gitlab_ci/build.yml new file mode 100644 index 00000000..600fcb14 --- /dev/null +++ b/.gitlab_ci/build.yml @@ -0,0 +1,58 @@ +# Build Docker image for test +build-test: + stage: build + variables: + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-test + DOCKER_TARGET: dev + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .build-and-push-gitlab + +# TODO: Make the multi-arch build in a single job (perhaps with a nested workflow) +build-http-app-amd64: + stage: build + variables: + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-http + DOCKER_PLATFORM: "linux/amd64" + DOCKER_TARGET: http_app + tags: + - saas-linux-small-amd64 + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .build-and-push-gitlab + +build-http-app-arm64: + stage: build + variables: + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-http + DOCKER_PLATFORM: "linux/arm64" + DOCKER_TARGET: http_app + tags: + - saas-linux-small-arm64 + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .build-and-push-gitlab + +aggregate-http-manifests: + stage: build + needs: + - build-http-app-amd64 + - build-http-app-arm64 + variables: + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-http + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .multiarch-manifest-gitlab + diff --git a/.gitlab_ci/deploy.yml b/.gitlab_ci/deploy.yml new file mode 100644 index 00000000..154b8b04 --- /dev/null +++ b/.gitlab_ci/deploy.yml @@ -0,0 +1,11 @@ +promote-dev: + stage: deploy + variables: + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-http + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .promote-image + when: manual diff --git a/.gitlab_ci/test.yml b/.gitlab_ci/test.yml new file mode 100644 index 00000000..6fe1e40b --- /dev/null +++ b/.gitlab_ci/test.yml @@ -0,0 +1,44 @@ +# Test Docker image +typing: + stage: test + variables: + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-test + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .python-typing + +lint: + stage: test + variables: + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-test + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .python-lint + +format: + stage: test + variables: + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-test + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .python-format + +tests: + stage: test + variables: + DOCKER_IMAGE_NAME: $CI_PROJECT_NAME-test + rules: + # We run the pipeline only on merge requests or the `main` branch + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + extends: + - .python-tests diff --git a/README.md b/README.md index ec7bbe72..58d53843 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ This template provides out of the box some commonly used functionalities: * Repository pattern for databases using [SQLAlchemy](https://www.sqlalchemy.org/) and [SQLAlchemy bind manager](https://febus982.github.io/sqlalchemy-bind-manager/stable/) * Database migrations using [Alembic](https://alembic.sqlalchemy.org/en/latest/) (configured supporting both sync and async SQLAlchemy engines) * Authentication and Identity Provider using [ORY Zero Trust architecture](https://www.ory.sh/docs/kratos/guides/zero-trust-iap-proxy-identity-access-proxy) +* Example CI/CD deployment pipeline for GitLab (The focus for this repository is still GitHub but, in case you want to use GitLab 🤷) * [TODO] Producer and consumer to emit and consume events using [CloudEvents](https://cloudevents.io/) format on [Confluent Kafka](https://docs.confluent.io/kafka-clients/python/current/overview.html) ## Documentation