.gitlab-ci.yml

image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:git

variables:
  DOCKER_REPOSITORY: 'mendersoftware/mender-test-containers'
  DOCKER_HUB_USERNAME: 'menderbuildsystem'
  DOCKER_BUILDKIT: 1
  DOCKER_VERSION: "27.3"

.mender-dist-packages-image-matrix:
  parallel:
    matrix:
      - BUILD: crosscompile
        DISTRO: debian
        RELEASE: [bookworm, trixie]
        ARCH: [amd64, armhf, arm64]
      - BUILD: crosscompile
        DISTRO: ubuntu
        RELEASE: [jammy, noble]
        ARCH: [amd64, armhf, arm64]

.tag_n_push_to_gitlab_registry: &tag_n_push_to_gitlab_registry
  - docker pull ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID}
  - docker tag ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID} ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_COMMIT_BRANCH}
  - docker push ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_COMMIT_BRANCH}

include:
  - project: Northern.tech/Mender/mendertesting
    file:
      - qa-common/runner.yml
      - qa-common/retry.yml
  - project: 'Northern.tech/Mender/mendertesting'
    file: '.gitlab-ci-check-commits-signoffs.yml'
  - project: 'Northern.tech/Mender/mendertesting'
    file: '.gitlab-ci-github-status-updates.yml'
  - project: 'Northern.tech/Mender/mendertesting'
    file: '.gitlab-ci-check-python3-format.yml'

default:
  tags: !reference [.qa-common-default-runner, tags]
  retry: !reference [.qa-common-default-retry, retry]

.requires-docker: &requires-docker
  - DOCKER_RETRY_SLEEP_S=10 # wait longer for k8s workers
  - DOCKER_RUNNING=false
  - for try in 4 3 2 1; do
  -   docker ps && DOCKER_RUNNING=true
  -   if [ "${DOCKER_RUNNING}" == "true" ]; then
  -     echo "DEBUG - docker is running - continue"
  -     break
  -   fi
  -   sleep "${DOCKER_RETRY_SLEEP_S}"
  - done
  - if [ "${DOCKER_RUNNING}" != "true" ]; then
  -   echo "DEBUG - docker is not running - exiting"
  -   exit 192
  - fi

.dind-login: &dind-login
  - mkdir -p $HOME/.docker && echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json

stages:
  - build
  - test
  - publish

build:gui-e2e-testing:
  tags:
    - hetzner-amd-beefy
  stage: build
  needs: []
  variables:
    CONTAINER_TAG: "gui-e2e-testing"
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  before_script:
    - *requires-docker
    - apk add --no-cache aws-cli curl
    - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)"
    - *dind-login
  script:
    - echo "INFO - Building and Pushing ${CONTAINER_TAG} to the registry ${CI_REGISTRY_IMAGE}"
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin
    - docker build
        --cache-from ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-master
        -t ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID}
        -f gui-e2e-testing/Dockerfile
        --push
        gui-e2e-testing
    - docker save ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID} > guiE2eTestingImage.tar
    # Upload to temporary S3 bucket left for backwards compatibility
    - mender_ci_save_tmp_artifact guiE2eTestingImage.tar
  artifacts:
    paths:
      - checksums
  retry:
    max: 2
    exit_codes:
      - 2
      - 137
      - 192

build:mender-client-acceptance-testing:
  tags:
    # KVM support requires the same type of GCP machine as the tests will use
    - mender-qa-worker-client-acceptance-tests
  stage: build
  needs: []
  variables:
    CONTAINER_TAG: "mender-client-acceptance-testing"
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  before_script:
    - *requires-docker
    - apk add --no-cache aws-cli curl
    - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)"
  script:
    - echo "INFO - Building and Pushing ${CONTAINER_TAG} to the registry ${CI_REGISTRY_IMAGE}"
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin
    - docker build
        --cache-from ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-master
        -t ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID}
        -f mender-client-acceptance-testing/Dockerfile
        --push
        mender-client-acceptance-testing
    - docker save ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID} > qaTestingImage.tar
    # Upload to temporary S3 bucket
    - mender_ci_save_tmp_artifact qaTestingImage.tar
  artifacts:
    paths:
      - checksums
  retry:
    max: 2
    exit_codes:
      - 2
      - 137
      - 192

build:aws-k8s-pipeline-toolbox:
  tags:
    - hetzner-amd-beefy
  stage: build
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker
  needs: []
  variables:
    CONTAINER_TAG: "aws-k8s-v1"
    CI_KUBECTL_VERSION: "1.32.9"
    CI_HELM_VERSION: "v3.18.6"
    CI_KUBECONFORM_VERSION: "0.6.3"
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  before_script:
    - *requires-docker
    - *dind-login
  script:
    - echo "INFO - Building and Pushing ${CONTAINER_TAG}-${CI_PIPELINE_ID} to the registry ${CI_REGISTRY_IMAGE}"
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin
    - docker build
        --cache-from ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-master
        -t ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID}
        --build-arg KUBECTL_VERSION=${CI_KUBECTL_VERSION}
        --build-arg HELM_VERSION=${CI_HELM_VERSION}
        --build-arg KUBECONFORM_VERSION=${CI_KUBECONFORM_VERSION}
        -f aws-k8s-toolbox/Dockerfile
        --push
        aws-k8s-toolbox
  retry:
    max: 2
    exit_codes:
      - 2
      - 137
      - 192

.build:mender-dist-packages-image:
  tags:
    - hetzner-amd-beefy-privileged
  stage: build
  needs: []
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:git
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  before_script:
    - *requires-docker
    - *dind-login
  script:
    - apk add bash
    - CONTAINER_TAG=mender-dist-packages-builder-${BUILD}-${DISTRO}-${RELEASE}-${ARCH}
    - echo "INFO - Building and Pushing ${CONTAINER_TAG}-${CI_PIPELINE_ID} to the registry ${CI_REGISTRY_IMAGE}"
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin
    - cd mender-dist-packages-building
    - ./build.sh
         --container-tag ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}
         --distro ${DISTRO}
         --release ${RELEASE}
         --arch ${ARCH}
         --ci-pipeline-id ${CI_PIPELINE_ID}
  parallel: !reference [.mender-dist-packages-image-matrix, parallel]
  retry:
    max: 2
    exit_codes:
      - 2
      - 137
      - 192

build:mender-dist-packages-image:
  rules:
    - if: $CI_COMMIT_BRANCH == "master"
  extends: .build:mender-dist-packages-image

build:mender-dist-packages-image:manual:
  when: manual
  extends: .build:mender-dist-packages-image

build:docker-multiplatform-buildx:
  tags:
    - hetzner-amd-beefy
  stage: build
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker
  needs: []
  variables:
    CONTAINER_TAG: "docker-multiplatform-buildx-v1"
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  before_script:
    - *requires-docker
    - *dind-login
  script:
    - echo "INFO - Building and Pushing ${CONTAINER_TAG}-${CI_PIPELINE_ID} to the registry ${CI_REGISTRY_IMAGE}"
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin
    - docker build
        --cache-from ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-master
        -t ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID}
        -f docker-multiplatform-buildx/Dockerfile
        --push
        docker-multiplatform-buildx
  retry:
    max: 2
    exit_codes:
      - 2
      - 137
      - 192

build:goveralls:
  tags:
    - hetzner-amd-beefy
  stage: build
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker
  needs: []
  variables:
    CONTAINER_TAG: "goveralls"
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  before_script:
    - *requires-docker
    - *dind-login
  script:
    - echo "INFO - Building and Pushing ${CONTAINER_TAG}-${CI_PIPELINE_ID} to the registry ${CI_REGISTRY_IMAGE}"
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin
    - docker build
        --cache-from ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-master
        -t ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID}
        -f goveralls/Dockerfile
        --push
        goveralls
  retry:
    max: 2
    exit_codes:
      - 2
      - 137
      - 192

build:mongodb-backup-runner:
  tags:
    - hetzner-amd-beefy
  stage: build
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker
  needs: []
  variables:
    CONTAINER_TAG: "mongodb-backup-runner"
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  before_script:
    - *requires-docker
    - *dind-login
  script:
    - echo "INFO - Building and Pushing ${CONTAINER_TAG}-${CI_PIPELINE_ID} to the registry ${CI_REGISTRY_IMAGE}"
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin
    - docker build
        --cache-from ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-master
        -t ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID}
        -f mongodb-backup-runner/Dockerfile
        --push
        mongodb-backup-runner
  retry:
    max: 2
    exit_codes:
      - 2
      - 137
      - 192

build:terragrunt-trivy-toolbox:
  tags:
    - hetzner-amd-beefy
  stage: build
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}
  needs: []
  variables:
    CONTAINER_TAG: "terragrunt-trivy-v1"
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  before_script:
    - *requires-docker
    - *dind-login
  script:
    - echo "INFO - Building and Pushing ${CONTAINER_TAG}-${CI_PIPELINE_ID} to the registry ${CI_REGISTRY_IMAGE}"
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin
    - docker build
        --cache-from ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-master
        -t ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID}
        -f terragrunt-trivy-toolbox/Dockerfile
        --push
        terragrunt-trivy-toolbox
  retry:
    max: 2
    exit_codes:
      - 2
      - 137
      - 192

build:release-please:
  tags:
    - hetzner-amd-beefy
  stage: build
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}
  needs: []
  variables:
    CONTAINER_TAG: "release-please-v1"
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  before_script:
    - *requires-docker
    - *dind-login
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin
    - docker context create builder
    - docker buildx create builder --use --driver-opt "image=moby/buildkit:v0.17.3,network=host" --buildkitd-flags '--debug --allow-insecure-entitlement network.host'
  script:
    - echo "INFO - Building and Pushing ${CONTAINER_TAG}-${CI_PIPELINE_ID} to the registry ${CI_REGISTRY_IMAGE}"
    - apk add --no-cache git
    - cd release-please
    - docker buildx build
        --cache-to type=registry,ref=${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}_ci_cache,mode=max
        --cache-from type=registry,ref=${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}_ci_cache
        --tag ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID}
        --file ${DOCKER_DIR:-.}/${DOCKERFILE:-Dockerfile}
        --provenance false
        --push
        ${DOCKER_DIR:-.}
  retry:
    max: 2
    exit_codes:
      - 2
      - 137
      - 192

.build:base-image:
  tags:
    - k8s
  stage: build
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}
  needs: []
  variables:
    DOCKER_HOST: "tcp://docker:2376"
    DOCKER_TLS_CERTDIR: "/certs"
    DOCKER_TLS_VERIFY: 1
    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  before_script:
    - *requires-docker
    - *dind-login
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin
    - docker context create builder
    - docker buildx create builder --use --driver-opt "image=moby/buildkit:v0.17.3,network=host" --buildkitd-flags '--debug --allow-insecure-entitlement network.host'
  script:
    - echo "INFO - Building and Pushing ${CONTAINER_TAG}-${CI_PIPELINE_ID} and ${CONTAINER_TAG}-${CI_COMMIT_BRANCH} to the registry ${CI_REGISTRY_IMAGE}"
    - cd ${BASE_IMAGE_DIR}
    - docker buildx build
        --platform linux/amd64,linux/arm64
        --cache-to type=registry,ref=${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}_ci_cache,mode=max
        --cache-from type=registry,ref=${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}_ci_cache
        --tag ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID}
        --tag ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_COMMIT_BRANCH}
        --provenance false
        --push
        .
  retry:
    max: 2
    exit_codes:
      - 2
      - 137
      - 192
    when:
      - runner_system_failure
      - stuck_or_timeout_failure

build:base-debian:
  extends: .build:base-image
  variables:
    CONTAINER_TAG: "base-debian"
    BASE_IMAGE_DIR: "base-debian"
  rules:
    - if: $CI_COMMIT_BRANCH == "master"
      changes:
        - base-debian/Dockerfile
    - when: manual
      allow_failure: true

build:base-ubuntu:
  extends: .build:base-image
  variables:
    CONTAINER_TAG: "base-ubuntu"
    BASE_IMAGE_DIR: "base-ubuntu"
  rules:
    - if: $CI_COMMIT_BRANCH == "master"
      changes:
        - base-ubuntu/Dockerfile
    - when: manual
      allow_failure: true

build:base-alpine:
  extends: .build:base-image
  variables:
    CONTAINER_TAG: "base-alpine"
    BASE_IMAGE_DIR: "base-alpine"
  rules:
    - if: $CI_COMMIT_BRANCH == "master"
      changes:
        - base-alpine/Dockerfile
    - when: manual
      allow_failure: true

.template:publish:
  tags:
    - hetzner-amd-beefy
  stage: publish
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  rules:
    - if: $CI_COMMIT_BRANCH == "master"
  before_script:
    - *requires-docker
    - echo -n $DOCKER_HUB_PASSWORD | docker login -u $DOCKER_HUB_USERNAME --password-stdin
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin
  retry:
    max: 2
    exit_codes:
      - 2
      - 137
      - 192

publish:gui-e2e-testing:
  extends: .template:publish
  needs:
    - job: build:gui-e2e-testing
      artifacts: true
  variables:
    CONTAINER_TAG: "gui-e2e-testing"
  script:
    # backward compatibility: pushing to the Docker Hub
    - apk add --no-cache aws-cli curl
    - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)"
    # Fetch from temporary S3 bucket
    - mender_ci_load_tmp_artifact guiE2eTestingImage.tar
    - echo "publishing image to Docker Hub"
    - docker load -i guiE2eTestingImage.tar
    - docker tag ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID} $DOCKER_REPOSITORY:$CONTAINER_TAG
    - docker push $DOCKER_REPOSITORY:$CONTAINER_TAG
    # Gitlab registry
    - *tag_n_push_to_gitlab_registry

publish:mender-client-acceptance-testing:
  extends: .template:publish
  needs:
    - job: build:mender-client-acceptance-testing
      artifacts: true
  variables:
    CONTAINER_TAG: "mender-client-acceptance-testing"
  script:
    # backward compatibility: pushing to the Docker Hub
    - apk add --no-cache aws-cli curl
    - eval "$(curl https://raw.githubusercontent.com/mendersoftware/mendertesting/master/mender-ci-common.sh)"
    # Fetch from temporary S3 bucket
    - mender_ci_load_tmp_artifact qaTestingImage.tar
    - echo "publishing image to Docker Hub"
    - docker load -i qaTestingImage.tar
    - docker tag ${CI_REGISTRY_IMAGE}:${CONTAINER_TAG}-${CI_PIPELINE_ID} $DOCKER_REPOSITORY:$CONTAINER_TAG
    - docker push $DOCKER_REPOSITORY:$CONTAINER_TAG
    # Gitlab registry
    - *tag_n_push_to_gitlab_registry

publish:aws-k8s-pipeline-toolbox:
  extends: .template:publish
  needs:
    - job: build:aws-k8s-pipeline-toolbox
      artifacts: true
  variables:
    CONTAINER_TAG: "aws-k8s-v1"
  script:
    - *tag_n_push_to_gitlab_registry

publish:docker-multiplatform-buildx:
  extends: .template:publish
  needs:
    - job: build:docker-multiplatform-buildx
      artifacts: true
  variables:
    CONTAINER_TAG: "docker-multiplatform-buildx-v1"
  script:
    - *tag_n_push_to_gitlab_registry

publish:mender-dist-packages-image:
  extends: .template:publish
  dependencies:
    # NOTE: We should depend on each of the jobs individually with:
    # - "build:mender-dist-packages-image: [${BUILD}, ${DISTRO}, ${RELEASE}, ${ARCH}]"
    # However GitLab does not seem to expand these variables on a dependencies
    # key. Use arbitrary one of them and assume that they all passed (using
    # depends instead of needs ensures that the whole previous build stage
    # needs to pass, so it should be safe).
    - "build:mender-dist-packages-image: [crosscompile, debian, trixie, amd64]"
  rules:
    - if: $CI_COMMIT_BRANCH == "master"
  script:
    - CONTAINER_TAG=mender-dist-packages-builder-${BUILD}-${DISTRO}-${RELEASE}-${ARCH}
    - *tag_n_push_to_gitlab_registry
  parallel: !reference [.mender-dist-packages-image-matrix, parallel]

publish:goveralls:
  extends: .template:publish
  needs:
    - job: build:goveralls
      artifacts: true
  variables:
    CONTAINER_TAG: "goveralls"
  script:
    - *tag_n_push_to_gitlab_registry

publish:mongodb-backup-runner:
  extends: .template:publish
  needs:
    - job: build:mongodb-backup-runner
      artifacts: true
  variables:
    CONTAINER_TAG: "mongodb-backup-runner"
  script:
    - *tag_n_push_to_gitlab_registry

publish:terragrunt-trivy-toolbox:
  extends: .template:publish
  needs:
    - job: build:aws-k8s-pipeline-toolbox
      artifacts: true
  variables:
    CONTAINER_TAG: "terragrunt-trivy-v1"
  script:
    - *tag_n_push_to_gitlab_registry

publish:release-please:
  extends: .template:publish
  needs:
    - job: build:release-please
      artifacts: true
  variables:
    CONTAINER_TAG: "release-please-v1"
  script:
    - *tag_n_push_to_gitlab_registry