continuous-delivery.yml

# This workflow executes the E2E Test Suite for a series of combinations that
# represent different execution environments
name: continuous-delivery
on:
  issue_comment:
    types: [created]
  # Manually or triggered by another workflow
  workflow_dispatch:
    inputs:
      depth:
        description: 'Depth (push, pull_request, main (default), schedule)'
        required: true
        default: 'main'
      limit:
        description: 'Limit to the specified engines list (kind, k3d, eks, aks, gke, openshift)'
        required: false
      test_level:
        description: 'Test level: 0(highest) to 4(lowest). Default is 4.'
        required: false
        default: '4'
      feature_type:
        description: >
          Feature Type (backup-restore, basic, cluster-metadata, declarative-databases, disruptive,
          image-volume-extensions, importing-databases, maintenance, no-openshift, observability, operator,
          performance, plugin, pod-scheduling, postgres-configuration, postgres-major-upgrade, publication-subscription,
          recovery, replication, security, self-healing, service-connectivity, smoke, snapshot, storage, tablespaces,
          upgrade)
        required: false
      log_level:
        description: 'Log level for operator (error, warning, info, debug(default), trace)'
        required: false
        default: 'debug'
  schedule:
    - cron:  '0 1 * * *'

permissions: read-all

# set up environment variables to be used across all the jobs
env:
  KUBEBUILDER_VERSION: "2.3.1"
  # renovate: datasource=github-tags depName=kubernetes-sigs/kind versioning=semver
  KIND_VERSION: "v0.31.0"
  # renovate: datasource=github-releases depName=k3d-io/k3d versioning=semver
  K3D_VERSION: "v5.8.3"
  # renovate: datasource=github-releases depName=rook/rook versioning=loose
  ROOK_VERSION: "v1.19.2"
  # renovate: datasource=github-releases depName=kubernetes-csi/external-snapshotter
  EXTERNAL_SNAPSHOTTER_VERSION: "v8.5.0"
  OPERATOR_IMAGE_NAME: "ghcr.io/${{ github.repository }}-testing"
  BUILD_PUSH_PROVENANCE: ""
  BUILD_PUSH_CACHE_FROM: ""
  BUILD_PUSH_CACHE_TO: ""
  REGISTRY: "ghcr.io"
  REGISTRY_USER: ${{ github.actor }}
  REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
  REPOSITORY_OWNER: "cloudnative-pg"
  SIGN_IMAGES: "true"
  SLACK_USERNAME: "cnpg-bot"
  BUILD_MANAGER_RELEASE_ARGS: "build --skip=validate --clean --id manager"
  # Keep in mind that adding more platforms (architectures) will increase the building
  # time even if we use the ghcache for the building process.
  PLATFORMS: "linux/amd64,linux/arm64"
  E2E_SUFFIX: "cnpge2e"

defaults:
  run:
    # default failure handling for shell scripts in 'run' steps
    shell: 'bash -Eeuo pipefail -x {0}'

jobs:
  # Trigger the workflow on release-* branches for smoke testing whenever it's a scheduled run.
  # Note: this is a workaround since we can't directly schedule-run a workflow from a non default branch
  smoke_test_release_branches:
    runs-on: ubuntu-24.04
    permissions:
      actions: write
    name: smoke test release-* branches when it's a scheduled run
    if: github.event_name == 'schedule'
    strategy:
      fail-fast: false
      matrix:
        branch: [release-1.25, release-1.27, release-1.28]
    env:
      BRANCH: ${{ matrix.branch }}
    steps:
      - name: Invoke workflow with inputs
        uses: benc-uk/workflow-dispatch@7a027648b88c2413826b6ddd6c76114894dc5ec4 # v1
        with:
          workflow: continuous-delivery
          ref: ${{ env.BRANCH }}
          inputs: '{ "depth": "push", "limit": "kind", "test_level": "4", "log_level": "debug" }'

  check_commenter:
    if: |
      github.event_name == 'issue_comment' &&
      github.event.action == 'created' &&
      github.event.issue.pull_request &&
      startsWith(github.event.comment.body, '/test')
    name: Retrieve command
    permissions:
      pull-requests: write
      contents: read
      issues: read
    runs-on: ubuntu-24.04
    outputs:
      github_ref: ${{ steps.refs.outputs.head_sha }}
      depth: ${{ env.DEPTH }}
      limit: ${{ env.LIMIT }}
      test_level: ${{ env.TEST_LEVEL }}
      feature_type: ${{ env.FEATURE_TYPE }}
      log_level: ${{ env.LOG_LEVEL }}
    steps:
      - name: Check for Command
        id: command
        uses: xt0rted/slash-command-action@bf51f8f5f4ea3d58abc7eca58f77104182b23e88 # v2
        continue-on-error: false
        with:
          command: test
          reaction: "true"
          reaction-type: "eyes"
          allow-edits: "false"
          permission-level: write

      - name: Process arguments
        id: args
        env:
          ARGS: ${{ steps.command.outputs.command-arguments }}
        run: |
          # Set the defaults
          DEPTH="main"
          LIMIT="kind"
          TEST_LEVEL="4"
          FEATURE_TYPE=""
          LOG_LEVEL="debug"

          for ARG in $ARGS; do
            IFS='=' read name value <<< $ARG
            case "${name}" in
              "depth"|"d")
              DEPTH="${value}"
              ;;
              "limit"|"l")
              LIMIT="${value}"
              ;;
              "test_level"|"level"|"tl")
              TEST_LEVEL="${value}"
              ;;
              "feature_type"|"type"|"ft")
              FEATURE_TYPE="${value}"
              ;;
              "log_level"|"ll")
              LOG_LEVEL="${value}"
              ;;
              *)
              ;;
            esac
          done

          echo "DEPTH=${DEPTH}" >> $GITHUB_ENV
          echo "LIMIT=${LIMIT}" >> $GITHUB_ENV
          echo "TEST_LEVEL=${TEST_LEVEL}" >> $GITHUB_ENV
          echo "FEATURE_TYPE=${FEATURE_TYPE}" >> $GITHUB_ENV
          echo "LOG_LEVEL=${LOG_LEVEL}" >> $GITHUB_ENV

      - name: Resolve Git reference
        uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3
        id: refs

      - name: Create comment
        uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          repository: ${{ github.repository }}
          issue-number: ${{ github.event.issue.number }}
          body: |
            @${{ github.actor }}, here's the link to the E2E on CNPG workflow run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

  test_arguments:
    name: Parse arguments
    if: |
      github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
    runs-on: ubuntu-24.04
    outputs:
      github_ref: ${{ github.ref }}
      depth: ${{ env.DEPTH }}
      limit: ${{ env.LIMIT }}
      test_level: ${{ env.TEST_LEVEL }}
      feature_type: ${{ env.FEATURE_TYPE }}
      log_level: ${{ env.LOG_LEVEL }}
    steps:
      - name: Parse input to env
        env:
          EVENT_NAME: ${{ github.event_name }}
          INPUT_DEPTH: ${{ github.event.inputs.depth }}
          INPUT_LIMIT: ${{ github.event.inputs.limit }}
          INPUT_TEST_LEVEL: ${{ github.event.inputs.test_level }}
          INPUT_FEATURE_TYPE: ${{ github.event.inputs.feature_type }}
          INPUT_LOG_LEVEL: ${{ github.event.inputs.log_level }}
        run: |
          # Set the defaults for workflow dispatch
          if [[ "${EVENT_NAME}" == 'workflow_dispatch' ]]; then
            DEPTH="${INPUT_DEPTH}"
            LIMIT="${INPUT_LIMIT}"
            TEST_LEVEL="${INPUT_TEST_LEVEL}"
            FEATURE_TYPE="${INPUT_FEATURE_TYPE}"
            LOG_LEVEL="${INPUT_LOG_LEVEL}"
          fi
          # Set the defaults for schedule dispatch
          if [[ "${EVENT_NAME}" == 'schedule' ]]; then
            DEPTH="schedule"
            LIMIT=""
            TEST_LEVEL="4"
            FEATURE_TYPE=""
            LOG_LEVEL="debug"
          fi
          echo "DEPTH=${DEPTH}" >> $GITHUB_ENV
          echo "LIMIT=${LIMIT}" >> $GITHUB_ENV
          echo "TEST_LEVEL=${TEST_LEVEL}" >> $GITHUB_ENV
          echo "FEATURE_TYPE=${FEATURE_TYPE}" >> $GITHUB_ENV
          echo "LOG_LEVEL=${LOG_LEVEL}" >> $GITHUB_ENV

  evaluate_options:
    name: Evaluate workflow options
    needs:
      - check_commenter
      - test_arguments
    runs-on: ubuntu-24.04
    if: |
      (
        needs.check_commenter.result == 'success' ||
        needs.test_arguments.result == 'success'
      ) &&
      !cancelled()
    outputs:
      git_ref: ${{ env.GITHUB_REF }}
      depth: ${{ env.DEPTH }}
      limit: ${{ env.LIMIT }}
      test_level: ${{ env.TEST_LEVEL }}
      feature_type: ${{ env.FEATURE_TYPE }}
      log_level: ${{ env.LOG_LEVEL }}
    steps:
      - name: From command
        env:
          EVENT_NAME: ${{ github.event_name }}
          WD_GITHUB_REF: ${{ needs.test_arguments.outputs.github_ref }}
          WD_DEPTH: ${{ needs.test_arguments.outputs.depth }}
          WD_LIMIT: ${{ needs.test_arguments.outputs.limit }}
          WD_TEST_LEVEL: ${{ needs.test_arguments.outputs.test_level }}
          WD_FEATURE_TYPE: ${{ needs.test_arguments.outputs.feature_type }}
          WD_LOG_LEVEL: ${{ needs.test_arguments.outputs.log_level }}
          IC_GITHUB_REF: ${{ needs.check_commenter.outputs.github_ref }}
          IC_DEPTH: ${{ needs.check_commenter.outputs.depth }}
          IC_LIMIT: ${{ needs.check_commenter.outputs.limit }}
          IC_TEST_LEVEL: ${{ needs.check_commenter.outputs.test_level }}
          IC_FEATURE_TYPE: ${{ needs.check_commenter.outputs.feature_type }}
          IC_LOG_LEVEL: ${{ needs.check_commenter.outputs.log_level }}
        run: |
          if [[ "${EVENT_NAME}" == 'workflow_dispatch' ]] || [[ "${EVENT_NAME}" == 'schedule' ]]; then
            echo "GITHUB_REF=${WD_GITHUB_REF}" >> $GITHUB_ENV
            echo "DEPTH=${WD_DEPTH}" >> $GITHUB_ENV
            echo "LIMIT=${WD_LIMIT}" >> $GITHUB_ENV
            echo "TEST_LEVEL=${WD_TEST_LEVEL}" >> $GITHUB_ENV
            echo "FEATURE_TYPE=${WD_FEATURE_TYPE}" >> $GITHUB_ENV
            echo "LOG_LEVEL=${WD_LOG_LEVEL}" >> $GITHUB_ENV
          fi
          if [[ "${EVENT_NAME}" == 'issue_comment' ]]; then
            echo "GITHUB_REF=${IC_GITHUB_REF}" >> $GITHUB_ENV
            echo "DEPTH=${IC_DEPTH}" >> $GITHUB_ENV
            echo "LIMIT=${IC_LIMIT}" >> $GITHUB_ENV
            echo "TEST_LEVEL=${IC_TEST_LEVEL}" >> $GITHUB_ENV
            echo "FEATURE_TYPE=${IC_FEATURE_TYPE}" >> $GITHUB_ENV
            echo "LOG_LEVEL=${IC_LOG_LEVEL}" >> $GITHUB_ENV
          fi

  buildx:
    name: Build containers
    needs:
      - check_commenter
      - test_arguments
      - evaluate_options
    if: |
      always() && !cancelled() &&
      needs.evaluate_options.result == 'success'
    runs-on: ubuntu-24.04
    permissions:
      contents: read
      packages: write
      pull-requests: read
      id-token: write
    outputs:
      # 'branch_name' is used in 'GetMostRecentReleaseTag' in the Go code
      branch_name: ${{ steps.build-meta.outputs.branch_name }}
      upload_artifacts: ${{ steps.build-meta.outputs.upload_artifacts }}
      commit_msg: ${{ steps.build-meta.outputs.commit_msg }}
      commit_sha: ${{ steps.build-meta.outputs.commit_sha }}
      author_name: ${{ steps.build-meta.outputs.author_name }}
      author_email: ${{ steps.build-meta.outputs.author_email }}
      controller_img: ${{ env.CONTROLLER_IMG }}
      controller_img_digest: ${{ fromJSON(steps.bake-push.outputs.metadata)['distroless']['containerimage.digest'] }}
      controller_img_prime_digest: ${{ steps.build-prime.outputs.digest }}
      controller_img_ubi: ${{ env.CONTROLLER_IMG_UBI }}
      index_img: ${{ env.INDEX_IMG }}
      bundle_img: ${{ env.BUNDLE_IMG }}
      catalog_img: ${{ env.CATALOG_IMG }}
    steps:
      -
        name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ needs.evaluate_options.outputs.git_ref }}
          # To identify the commit we need the history and all the tags.
          fetch-depth: 0
      -
        name: Install Go
        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
        with:
          go-version-file: go.mod
          check-latest: true
      -
        name: Build meta
        id: build-meta
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          COMMIT_REF: ${{ needs.evaluate_options.outputs.git_ref }}
          EVENT_NAME: ${{ github.event_name }}
          ISSUE_NUMBER: ${{ github.event.issue.number }}
        run: |
          commit_sha="${COMMIT_REF}"
          commit_date=$(git log -1 --pretty=format:'%ad' --date short "${commit_sha}" || : )
          # use git describe to get the nearest tag and use that to build the version (e.g. 1.4.0-dev24 or 1.4.0)
          commit_version=$(git describe --tags --match 'v*' "${commit_sha}"| sed -e 's/^v//; s/-g[0-9a-f]\+$//; s/-\([0-9]\+\)$/-dev\1/')

          # shortened commit sha
          commit_short=$(git rev-parse --short "${commit_sha}")

          # multiline strings are weird
          commit_message=$(git show -s --format=%B "${commit_sha}")
          commit_message=${commit_message//$'%'/'%25'}
          commit_message=${commit_message//$'\n'/'%0A'}
          commit_message=${commit_message//$'\r'/'%0D'}

          # get git user and email
          author_name=$(git show -s --format='%an' "${commit_sha}" | head -n1)
          author_email=$(git show -s --format='%ae' "${commit_sha}" | head -n1)

          # extract branch name
          if [[ "${EVENT_NAME}" == 'workflow_dispatch' ]] || [[ "${EVENT_NAME}" == 'schedule' ]]
          then
            branch_name=${GITHUB_REF#refs/heads/}
          fi
          if [[ "${EVENT_NAME}" == 'issue_comment' ]]
          then
            branch_name=$(gh pr view "${ISSUE_NUMBER}" --json headRefName -q '.headRefName' 2>/dev/null)
          fi

          # extract tag from branch name
          tag_name=$(echo "$branch_name" | tr / -)

          upload_artifacts=false
          if [[ ${branch_name} == main || ${branch_name} =~ ^release- ]]; then
            upload_artifacts=true
          fi

          echo "DATE=${commit_date}" >> $GITHUB_ENV
          echo "VERSION=${commit_version}" >> $GITHUB_ENV
          echo "COMMIT=${commit_short}" >> $GITHUB_ENV
          echo "IMAGE_TAG=${tag_name,,}" >> $GITHUB_ENV
          echo "REPO_OWNER=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_ENV
          echo "commit_sha=${commit_sha}" >> $GITHUB_OUTPUT
          echo "commit_msg=${commit_message}" >> $GITHUB_OUTPUT
          echo "author_name=${author_name}" >> $GITHUB_OUTPUT
          echo "author_email=${author_email}" >> $GITHUB_OUTPUT
          echo "branch_name=${branch_name}" >> $GITHUB_OUTPUT
          echo "upload_artifacts=${upload_artifacts}" >> $GITHUB_OUTPUT
      -
        name: Set GoReleaser environment
        run: |
          echo GOPATH=$(go env GOPATH) >> $GITHUB_ENV
          echo PWD=$(pwd) >> $GITHUB_ENV
      -
        name: Run GoReleaser
        uses: goreleaser/goreleaser-action@ec59f474b9834571250b370d4735c50f8e2d1e29 # v7
        with:
          distribution: goreleaser
          version: v2
          args: ${{ env.BUILD_MANAGER_RELEASE_ARGS }}
        env:
          DATE: ${{ env.DATE }}
          COMMIT: ${{ env.COMMIT }}
          VERSION: ${{ env.VERSION }}
      -
        name: Set up QEMU
        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4
        with:
          platforms: ${{ env.PLATFORMS }}
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
      -
        name: Login into docker registry
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ env.REGISTRY_USER }}
          password: ${{ env.REGISTRY_PASSWORD }}
      -
        name: Build and push
        uses: docker/bake-action@82490499d2e5613fcead7e128237ef0b0ea210f7 # v7
        id: bake-push
        env:
          BUILDX_METADATA_PROVENANCE: disabled
          environment: "testing"
          buildVersion: ${{ env.VERSION }}
          tag: ${{ env.IMAGE_TAG }}
          registry: ${{ env.REGISTRY }}/${{ env.REPO_OWNER }}
          revision: ${{ env.COMMIT }}
        with:
          source: .
          push: true
          no-cache: true
          targets: "default"
      -
        name: Install cosign
        if: env.SIGN_IMAGES == 'true'
        uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3
        # See https://github.blog/security/supply-chain-security/safeguard-container-signing-capability-actions/
        # and https://github.com/actions/starter-workflows/blob/main/ci/docker-publish.yml for more details on
        # how to use cosign.
      -
        name: Sign images
        if: env.SIGN_IMAGES == 'true'
        env:
          BAKE_METADATA: ${{ steps.bake-push.outputs.metadata }}
        run: |
          images=$(echo "${BAKE_METADATA}" |
            jq -r '.[] | (."image.name" | sub(",.*";"" )) + "@" + ."containerimage.digest"'
          )
          cosign sign --yes ${images}
      -
        name: Output images
        env:
          DISTROLESS: ${{ fromJSON(steps.bake-push.outputs.metadata)['distroless']['image.name'] }}
          UBI: ${{ fromJSON(steps.bake-push.outputs.metadata)['ubi']['image.name'] }}
        run: |
          echo "CONTROLLER_IMG=${DISTROLESS}" >> $GITHUB_ENV
          echo "CONTROLLER_IMG_UBI=${UBI}" >> $GITHUB_ENV
          echo "BUNDLE_IMG=${UBI}-bundle" >> $GITHUB_ENV
          echo "INDEX_IMG=${UBI}-index" >> $GITHUB_ENV
          echo "CATALOG_IMG=${UBI}-catalog" >> $GITHUB_ENV
      -
        name: Generate manifest for operator deployment
        id: generate-manifest
        env:
          CONTROLLER_IMG: ${{ env.CONTROLLER_IMG }}
          CONTROLLER_IMG_DIGEST: ${{ fromJSON(steps.bake-push.outputs.metadata)['distroless']['containerimage.digest'] }}
        run: |
          make generate-manifest
      -
        name: Upload the operator manifest as artifact in workflow
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: operator-manifest.yaml
          path: dist/operator-manifest.yaml
          retention-days: 7
      -
      # In order to test the case of upgrading from the current operator
      # to a future one, we build and push an image with a different VERSION
      # to force a different hash for the manager binary.
      # (Otherwise the ONLINE upgrade won't trigger)
      #
      # NOTE: we only fire this in TEST DEPTH = 4, as that is the level of the
      # upgrade test
        name: Build binary for upgrade test
        uses: goreleaser/goreleaser-action@ec59f474b9834571250b370d4735c50f8e2d1e29 # v7
        if: |
          always() && !cancelled() &&
          needs.evaluate_options.outputs.test_level == '4'
        with:
          distribution: goreleaser
          version: v2
          args: ${{ env.BUILD_MANAGER_RELEASE_ARGS }}
        env:
          DATE: ${{ env.DATE }}
          COMMIT: ${{ env.COMMIT }}
          VERSION: ${{ env.VERSION }}-prime
      -
      # In order to test the case of upgrading from the current operator
      # to a future one, we build and push an image with a different VERSION
      # to force a different hash for the manager binary.
      # (Otherwise the ONLINE upgrade won't trigger)
      #
      # We push the "prime" binary using a tag with the suffix "-prime"
      # NOTE: we only fire this in TEST DEPTH = 4, as that is the level of the
      # upgrade test
        name: Build and push image for upgrade test
        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
        id: build-prime
        if: |
          always() && !cancelled() &&
          needs.evaluate_options.outputs.test_level == '4'
        with:
          platforms: ${{ env.PLATFORMS }}
          context: .
          file: Dockerfile
          push: true
          build-args: |
            VERSION=${{ env.VERSION }}-prime
          tags: ${{ env.CONTROLLER_IMG }}-prime
          provenance: ${{ env.BUILD_PUSH_PROVENANCE }}
          cache-from: ${{ env.BUILD_PUSH_CACHE_FROM }}
          cache-to: ${{ env.BUILD_PUSH_CACHE_TO }}

  # This will only execute in cloudnative-pg org
  publish-artifacts:
    name: Publish artifacts
    needs:
      - buildx
    if: |
      (always() && !cancelled()) &&
      needs.buildx.result == 'success' &&
      needs.buildx.outputs.upload_artifacts == 'true' &&
      github.repository_owner == 'cloudnative-pg'
    runs-on: ubuntu-24.04
    env:
      BRANCH: ${{ needs.buildx.outputs.branch_name }}
    steps:
      -
        name: Checkout artifact
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          repository: cloudnative-pg/artifacts
          token: ${{ secrets.REPO_GHA_PAT }}
          ref: main
          fetch-depth: 0
      -
        name: Configure git user
        env:
          AUTHOR_EMAIL: ${{ needs.buildx.outputs.author_email }}
          AUTHOR_NAME: ${{ needs.buildx.outputs.author_name }}
        run: |
          git config user.email "${AUTHOR_EMAIL}"
          git config user.name "${AUTHOR_NAME}"
      -
        name: Switch to or create the right branch
        run: |
          git checkout "${BRANCH}" 2>/dev/null || git checkout -b "${BRANCH}"

          # Remove the previous operator manifest if present because the next
          # step doesn't overwrite existing files
          rm -fr manifests/operator-manifest.yaml
      -
        name: Prepare the operator manifest
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: operator-manifest.yaml
          path: manifests
      -
        name: Prepare the commit
        env:
          COMMIT_MESSAGE: |
            ${{ needs.buildx.outputs.commit_msg }}

            https://github.com/cloudnative-pg/cloudnative-pg/commit/${{ needs.buildx.outputs.commit_sha }}
        run: |
          # Skip creating the commit if there are no changes
          [ -n "$(git status -s)" ] || exit 0

          git add .
          git commit -m "${COMMIT_MESSAGE}"
      -
        name: Push changes
        uses: ad-m/github-push-action@77c5b412c50b723d2a4fbc6d71fb5723bcd439aa # v1.0.0
        with:
          github_token: ${{ secrets.REPO_GHA_PAT }}
          repository: cloudnative-pg/artifacts
          branch: ${{ env.BRANCH }}

  generate-jobs:
    name: Generate jobs for E2E tests
    needs:
      - buildx
      - evaluate_options
    # We try to avoid running the E2E Test Suite in general, to reduce load on
    # GitHub resources.
    # Currently, it's executed in the following cases:
    # - When dispatched via chatops commands
    # - On a push in main and release branches
    # - On scheduled executions
    if: |
      (always() && !cancelled()) &&
      needs.buildx.result == 'success'
    runs-on: ubuntu-24.04
    outputs:
      controller_img: ${{ needs.buildx.outputs.controller_img }}
      kindMatrix: ${{ steps.generate-jobs.outputs.kindMatrix }}
      kindEnabled: ${{ steps.generate-jobs.outputs.kindEnabled }}
      kindTimeout: ${{ steps.generate-jobs.outputs.kindE2ETimeout }}
      k3dMatrix: ${{ steps.generate-jobs.outputs.k3dMatrix }}
      k3dEnabled: ${{ steps.generate-jobs.outputs.k3dEnabled }}
      k3dTimeout: ${{ steps.generate-jobs.outputs.k3dE2ETimeout }}
      eksMatrix: ${{ steps.generate-jobs.outputs.eksMatrix }}
      eksEnabled: ${{ steps.generate-jobs.outputs.eksEnabled }}
      eksTimeout: ${{ steps.generate-jobs.outputs.eksE2ETimeout }}
      aksMatrix: ${{ steps.generate-jobs.outputs.aksMatrix }}
      aksEnabled: ${{ steps.generate-jobs.outputs.aksEnabled }}
      aksTimeout: ${{ steps.generate-jobs.outputs.aksE2ETimeout }}
      gkeMatrix: ${{ steps.generate-jobs.outputs.gkeMatrix }}
      gkeEnabled: ${{ steps.generate-jobs.outputs.gkeEnabled }}
      gkeTimeout: ${{ steps.generate-jobs.outputs.gkeE2ETimeout }}
      openshiftMatrix: ${{ steps.generate-jobs.outputs.openshiftMatrix }}
      openshiftEnabled: ${{ steps.generate-jobs.outputs.openshiftEnabled }}
      openshiftTimeout: ${{ steps.generate-jobs.outputs.openshiftE2ETimeout }}
    steps:
      -
        name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ needs.evaluate_options.outputs.git_ref }}
      -
        id: generate-jobs
        # Generates the jobs that will become different matrix branches,
        # according to the event, or to the "depth" parameter if set manually
        name: Generate Jobs
        shell: bash
        env:
          DEPTH: ${{ needs.evaluate_options.outputs.depth }}
          LIMIT: ${{ needs.evaluate_options.outputs.limit }}
        run: |
          python .github/e2e-matrix-generator.py \
            -m "${DEPTH}" \
            -l "${LIMIT}"

  e2e-kind:
    name: Run E2E on kind executors
    if: |
      (always() && !cancelled()) &&
      needs.generate-jobs.outputs.kindEnabled == 'true' &&
      needs.generate-jobs.result == 'success'
    needs:
      - buildx
      - generate-jobs
      - evaluate_options
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.generate-jobs.outputs.kindMatrix) }}
    runs-on: ubuntu-24.04
    env:
      # TEST_DEPTH determines the maximum test level the suite should be running
      TEST_DEPTH: ${{ needs.evaluate_options.outputs.test_level }}
      # FEATURE_TYPE, when defined, determines the subset of E2E tests that will be executed, divided by feature type
      FEATURE_TYPE: ${{ needs.evaluate_options.outputs.feature_type }}

      K8S_VERSION: "${{ matrix.k8s_version }}"
      POSTGRES_VERSION: ${{ matrix.postgres_version }}
      POSTGRES_KIND: ${{ matrix.postgres_kind }}
      MATRIX: ${{ matrix.id }}
      POSTGRES_IMG: "${{ matrix.postgres_img }}"
      # The version of operator to upgrade FROM, in the rolling upgrade E2E test
      E2E_PRE_ROLLING_UPDATE_IMG: "${{ matrix.postgres_pre_img }}"
      TEST_TIMEOUTS: ${{ needs.generate-jobs.outputs.kindTimeout }}
      BRANCH_NAME: ${{ needs.buildx.outputs.branch_name }}
      GIT_REF: ${{ needs.evaluate_options.outputs.git_ref }}

      DEBUG: "true"
      BUILD_IMAGE: "false"
      CONTROLLER_IMG: ${{ needs.generate-jobs.outputs.controller_img }}
      CONTROLLER_IMG_DIGEST: ${{ needs.buildx.outputs.controller_img_digest }}
      CONTROLLER_IMG_PRIME_DIGEST: ${{ needs.buildx.outputs.controller_img_prime_digest }}
      E2E_DEFAULT_STORAGE_CLASS: standard
      E2E_CSI_STORAGE_CLASS: csi-hostpath-sc
      E2E_DEFAULT_VOLUMESNAPSHOT_CLASS: csi-hostpath-snapclass
      LOG_DIR: ${{ github.workspace }}/kind-logs/
      DOCKER_REGISTRY_MIRROR: https://mirror.gcr.io
      TEST_CLOUD_VENDOR: "kind"
    steps:
      -
        name: Cleanup Disk
        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          tool-cache: true
          large-packages: false
          swap-storage: false
      -
        name: Cleanup docker cache
        run: |
          echo "-------------Disk info before cleanup----------------"
          df -h
          echo "-----------------------------------------------------"
          docker system prune -a -f
          echo "-------------Disk info after cleanup----------------"
          df -h
          echo "-----------------------------------------------------"
      -
        name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ env.GIT_REF }}
      -
        name: Install Go
        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
        with:
          go-version-file: go.mod
          check-latest: true
      -
        ## In case hack/setup-cluster.sh need pull operand image from registry
        name: Login into docker registry
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ env.REGISTRY_USER }}
          password: ${{ env.REGISTRY_PASSWORD }}
      -
        name: Install Kind
        uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0
        with:
          install_only: true
          version: ${{ env.KIND_VERSION }}
          kubectl_version: ${{ env.K8S_VERSION }}
      -
        name: Prepare patch for customization
        env:
          ## the following variable all need be set if we use env_override_customized.yaml.template
          ## this is customization for local kind
          LEADER_ELECTION: "true"
          LEADER_LEASE_DURATION: 15
          LEADER_RENEW_DEADLINE: 10
          LIVENESS_PROBE_THRESHOLD: 3
          LOG_LEVEL: ${{ needs.evaluate_options.outputs.log_level }}
        run: |
          LOG_LEVEL=${LOG_LEVEL:-info}
          envsubst < hack/e2e/env_override_customized.yaml.template > config/manager/env_override.yaml
          cat config/manager/env_override.yaml
      -
        name: Run Kind End-to-End tests
        env:
          ENABLE_APISERVER_AUDIT: true
        run:
          make e2e-test-kind
      -
        # Summarize the failed E2E test cases if there are any
        name: Report failed E2E tests
        if: failure()
        run: |
          set +x
          chmod +x .github/report-failed-test.sh
          ./.github/report-failed-test.sh
      -
        # Create an individual artifact for each E2E test, which will be used to
        # generate E2E test summary in the follow-up job 'summarize-e2e-tests'
        name: Create individual artifact for each E2E test
        if: (always() && !cancelled())
        env:
          RUNNER: "kind"
          RUN_ID: ${{ github.run_id }}
          REPOSITORY: ${{ github.repository }}
        run: |
          set +x
          python .github/generate-test-artifacts.py \
            -o testartifacts-${{ env.MATRIX }} \
            -f tests/e2e/out/report.json \
            --environment=true
          if [ -f tests/e2e/out/upgrade_report.json ]; then
            python .github/generate-test-artifacts.py \
              -o testartifacts-${{ env.MATRIX }} \
              -f tests/e2e/out/upgrade_report.json \
              --environment=true
          fi
      -
        name: Archive test artifacts
        if: (always() && !cancelled())
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: testartifacts-${{ env.MATRIX }}
          path: testartifacts-${{ env.MATRIX }}/
          retention-days: 7
      -
        name: Cleanup test artifacts
        if: always()
        run:
          rm -rf testartifacts-${{ env.MATRIX }}/
      -
        name: Cleanup ginkgo JSON report
        # Delete report.json after the analysis. File should always exist.
        # Delete upgrade_report.json. It may not exist depending on test level.
        if: always()
        run: |
          if [ -f tests/e2e/out/upgrade_report.json ]; then
            rm tests/e2e/out/upgrade_report.json
          fi
          if [ -f tests/e2e/out/report.json ]; then
            rm tests/e2e/out/report.json
          fi
      -
        # Archive logs for failed test cases if there are any
        name: Archive Kind logs
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: kind-logs-${{ env.MATRIX }}
          path: kind-logs/
          retention-days: 7
      -
        name: Archive e2e failure contexts
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: test-failure-contexts-${{ env.MATRIX }}
          path: |
            tests/*/out/
          retention-days: 7
          if-no-files-found: ignore

      -
        name: Archive e2e logs
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: cluster-logs-${{ env.MATRIX }}
          path: |
            tests/e2e/cluster_logs/**
          retention-days: 7
          if-no-files-found: ignore

  e2e-k3d:
    name: Run E2E on K3D
    if: |
      (always() && !cancelled()) &&
      github.event_name == 'workflow_dispatch' &&
      needs.generate-jobs.outputs.k3dEnabled == 'true' &&
      needs.generate-jobs.result == 'success'
    needs:
      - buildx
      - generate-jobs
      - evaluate_options
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.generate-jobs.outputs.k3dMatrix) }}
    runs-on: ubuntu-24.04
    env:
      # TEST_DEPTH determines the maximum test level the suite should be running
      TEST_DEPTH: ${{ needs.evaluate_options.outputs.test_level }}
      # FEATURE_TYPE, when defined, determines the subset of E2E tests that will be executed, divided by feature type
      FEATURE_TYPE: ${{ needs.evaluate_options.outputs.feature_type }}

      K8S_VERSION: "${{ matrix.k8s_version }}"
      POSTGRES_VERSION: ${{ matrix.postgres_version }}
      POSTGRES_KIND: ${{ matrix.postgres_kind }}
      MATRIX: ${{ matrix.id }}
      POSTGRES_IMG: "${{ matrix.postgres_img }}"
      # The version of operator to upgrade FROM, in the rolling upgrade E2E test
      E2E_PRE_ROLLING_UPDATE_IMG: "${{ matrix.postgres_pre_img }}"
      TEST_TIMEOUTS: ${{ needs.generate-jobs.outputs.k3dTimeout }}
      BRANCH_NAME: ${{ needs.buildx.outputs.branch_name }}
      GIT_REF: ${{ needs.evaluate_options.outputs.git_ref }}

      DEBUG: "true"
      BUILD_IMAGE: "false"
      CONTROLLER_IMG: ${{ needs.generate-jobs.outputs.controller_img }}
      CONTROLLER_IMG_DIGEST: ${{ needs.buildx.outputs.controller_img_digest }}
      CONTROLLER_IMG_PRIME_DIGEST: ${{ needs.buildx.outputs.controller_img_prime_digest }}
      E2E_DEFAULT_STORAGE_CLASS: local-path  # K3D uses local-path by default
      E2E_CSI_STORAGE_CLASS: csi-hostpath-sc
      E2E_DEFAULT_VOLUMESNAPSHOT_CLASS: csi-hostpath-snapclass
      LOG_DIR: ${{ github.workspace }}/k3d-logs/
      DOCKER_REGISTRY_MIRROR: https://mirror.gcr.io
      TEST_CLOUD_VENDOR: "k3d"
    steps:
      - name: Cleanup Disk
        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
        with:
          android: true
          dotnet: true
          haskell: true
          tool-cache: true
          large-packages: false
          swap-storage: false
      - name: Cleanup docker cache
        run: |
          echo "-------------Disk info before cleanup----------------"
          df -h
          echo "-----------------------------------------------------"
          docker system prune -a -f
          echo "-------------Disk info after cleanup----------------"
          df -h
          echo "-----------------------------------------------------"
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ env.GIT_REF }}
      - name: Install Go
        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
        with:
          go-version-file: go.mod
          check-latest: true
      - ## In case hack/setup-cluster.sh needs to pull operand image from registry
        name: Login into docker registry
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ env.REGISTRY_USER }}
          password: ${{ env.REGISTRY_PASSWORD }}
      - name: Install K3D
        run: |
          K3D_URL="https://github.com/k3d-io/k3d/releases/download/${{ env.K3D_VERSION }}"
          curl -sL "${K3D_URL}/k3d-linux-amd64" -o /tmp/k3d
          curl -sL "${K3D_URL}/checksums.txt" -o /tmp/k3d-checksums.txt
          grep '_dist/k3d-linux-amd64$' /tmp/k3d-checksums.txt | sed 's|_dist/k3d-linux-amd64|/tmp/k3d|' | sha256sum -c -
          install /tmp/k3d /usr/local/bin/k3d
          k3d version
      - name: Install kubectl
        uses: azure/setup-kubectl@776406bce94f63e41d621b960d78ee25c8b76ede # v4.0.1
        with:
          version: ${{ env.K8S_VERSION }}
      - name: Prepare patch for customization
        env:
          ## the following variables all need to be set if we use env_override_customized.yaml.template
          ## this is customization for local k3d
          LEADER_ELECTION: "true"
          LEADER_LEASE_DURATION: 15
          LEADER_RENEW_DEADLINE: 10
          LIVENESS_PROBE_THRESHOLD: 3
          LOG_LEVEL: ${{ needs.evaluate_options.outputs.log_level }}
        run: |
          LOG_LEVEL=${LOG_LEVEL:-info}
          envsubst < hack/e2e/env_override_customized.yaml.template > config/manager/env_override.yaml
          cat config/manager/env_override.yaml
      - name: Run K3D End-to-End tests
        env:
          ENABLE_APISERVER_AUDIT: true
        run: |
          make e2e-test-k3d
      - # Summarize the failed E2E test cases if there are any
        name: Report failed E2E tests
        if: failure()
        run: |
          set +x
          chmod +x .github/report-failed-test.sh
          ./.github/report-failed-test.sh
      - # Create an individual artifact for each E2E test
        name: Create individual artifact for each E2E test
        if: (always() && !cancelled())
        env:
          RUNNER: "k3d"
          RUN_ID: ${{ github.run_id }}
          REPOSITORY: ${{ github.repository }}
        run: |
          set +x
          python .github/generate-test-artifacts.py \
            -o testartifacts-${{ env.MATRIX }} \
            -f tests/e2e/out/report.json \
            --environment=true
          if [ -f tests/e2e/out/upgrade_report.json ]; then
            python .github/generate-test-artifacts.py \
              -o testartifacts-${{ env.MATRIX }} \
              -f tests/e2e/out/upgrade_report.json \
              --environment=true
          fi
      - name: Archive test artifacts
        if: (always() && !cancelled())
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: testartifacts-${{ env.MATRIX }}
          path: testartifacts-${{ env.MATRIX }}/
          retention-days: 7
      - name: Cleanup test artifacts
        if: always()
        run: |
          rm -rf testartifacts-${{ env.MATRIX }}/
      - name: Cleanup ginkgo JSON report
        if: always()
        run: |
          if [ -f tests/e2e/out/upgrade_report.json ]; then
            rm tests/e2e/out/upgrade_report.json
          fi
          if [ -f tests/e2e/out/report.json ]; then
            rm tests/e2e/out/report.json
          fi
      - # Archive logs for failed test cases if there are any
        name: Archive K3D logs
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: k3d-logs-${{ env.MATRIX }}
          path: k3d-logs/
          retention-days: 7
      - name: Archive e2e failure contexts
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: test-failure-contexts-${{ env.MATRIX }}
          path: |
            tests/*/out/
          retention-days: 7
          if-no-files-found: ignore
      - name: Archive e2e logs
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: cluster-logs-${{ env.MATRIX }}
          path: |
            tests/e2e/cluster_logs/**
          retention-days: 7
          if-no-files-found: ignore

  # AKS Secrets required
  # secrets.AZURE_CREDENTIALS
  # secrets.AZURE_SUBSCRIPTION
  # secrets.AZURE_RESOURCEGROUP
  # secrets.AZURE_RESOURCENAME
  # secrets.AZURE_WORKSPACE_RESOURCE_ID

  e2e-aks-setup:
    name: Setup shared resources for Microsoft AKS E2Es
    if: |
      (always() && !cancelled()) &&
      vars.AKS_ENABLED == 'true' &&
      needs.generate-jobs.outputs.aksEnabled == 'true' &&
      needs.generate-jobs.result == 'success'
    needs:
      - buildx
      - generate-jobs
      - evaluate_options
    runs-on: ubuntu-24.04
    outputs:
      azure_storage_account: ${{ steps.setup.outputs.azure_storage_account }}
    steps:
      -
        name: Azure Login
        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      -
        name: Create AKS shared resources
        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
        id: setup
        with:
          timeout_minutes: 10
          max_attempts: 3
          command: |
            az extension add --allow-preview true --name aks-preview
            az account set --subscription ${{ secrets.AZURE_SUBSCRIPTION }}

            AZURE_STORAGE_ACCOUNT="${GITHUB_RUN_NUMBER}${E2E_SUFFIX}"
            az storage account create \
              --resource-group ${{ secrets.AZURE_RESOURCEGROUP }} \
              --name ${AZURE_STORAGE_ACCOUNT} \
              --sku Standard_LRS -o none

            # Output storage account name
            echo "azure_storage_account=${AZURE_STORAGE_ACCOUNT}" >> $GITHUB_OUTPUT

  e2e-aks:
    name: Run E2E on Microsoft AKS
    if: |
      (always() && !cancelled()) &&
      vars.AKS_ENABLED == 'true' &&
      needs.generate-jobs.outputs.aksEnabled == 'true' &&
      needs.generate-jobs.result == 'success' &&
      needs.e2e-aks-setup.result == 'success'
    needs:
      - buildx
      - generate-jobs
      - evaluate_options
      - e2e-aks-setup
    strategy:
      fail-fast: false
      max-parallel: 8
      matrix: ${{ fromJSON(needs.generate-jobs.outputs.aksMatrix) }}
    runs-on: ubuntu-24.04
    env:
      # TEST_DEPTH determines the maximum test level the suite should be running
      TEST_DEPTH: ${{ needs.evaluate_options.outputs.test_level }}
      # FEATURE_TYPE, when defined, determines the subset of E2E tests that will be executed, divided by feature type
      FEATURE_TYPE: ${{ needs.evaluate_options.outputs.feature_type }}

      K8S_VERSION: "${{ matrix.k8s_version }}"
      POSTGRES_VERSION: ${{ matrix.postgres_version }}
      POSTGRES_KIND: ${{ matrix.postgres_kind }}
      MATRIX: ${{ matrix.id }}
      POSTGRES_IMG: "${{ matrix.postgres_img }}"
      # The version of operator to upgrade FROM, in the rolling upgrade E2E test
      E2E_PRE_ROLLING_UPDATE_IMG: "${{ matrix.postgres_pre_img }}"
      TEST_TIMEOUTS: ${{ needs.generate-jobs.outputs.aksTimeout }}
      BRANCH_NAME: ${{ needs.buildx.outputs.branch_name }}
      GIT_REF: ${{ needs.evaluate_options.outputs.git_ref }}

      AZURE_STORAGE_ACCOUNT: ${{ needs.e2e-aks-setup.outputs.azure_storage_account }}
      # AZURE_STORAGE_KEY: this one is gathered during a subsequent step

      DEBUG: "true"
      BUILD_IMAGE: "false"
      CONTROLLER_IMG: ${{ needs.generate-jobs.outputs.controller_img }}
      CONTROLLER_IMG_DIGEST: ${{ needs.buildx.outputs.controller_img_digest }}
      CONTROLLER_IMG_PRIME_DIGEST: ${{ needs.buildx.outputs.controller_img_prime_digest }}
      E2E_DEFAULT_STORAGE_CLASS: rook-ceph-block
      E2E_CSI_STORAGE_CLASS: rook-ceph-block
      E2E_DEFAULT_VOLUMESNAPSHOT_CLASS: csi-rbdplugin-snapclass
      TEST_CLOUD_VENDOR: "aks"

    steps:
      -
        name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ env.GIT_REF }}
      -
        name: Install Go
        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
        with:
          go-version-file: go.mod
          check-latest: true
      -
        name: Prepare the environment
        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
        with:
          timeout_seconds: 300
          max_attempts: 3
          command: |
            sudo apt-get update
            sudo apt-get install -y gettext-base
      -
        name: Install ginkgo
        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
        with:
          timeout_minutes: 1
          max_attempts: 3
          command: |
            # renovate: datasource=github-releases depName=onsi/ginkgo
            go install github.com/onsi/ginkgo/v2/ginkgo@v2.28.1
      -
        ## In case hack/setup-cluster.sh need pull operand image from registry
        name: Login into docker registry
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ env.REGISTRY_USER }}
          password: ${{ env.REGISTRY_PASSWORD }}
      -
        name: Azure Login
        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      -
        name: Install kubectl
        uses: azure/setup-kubectl@776406bce94f63e41d621b960d78ee25c8b76ede # v4
        with:
          version: v${{ env.K8S_VERSION }}
      -
        name: Create AKS cluster
        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
        with:
          timeout_minutes: 10
          max_attempts: 3
          command: |
            az extension add --allow-preview true --name aks-preview
            az account set --subscription ${{ secrets.AZURE_SUBSCRIPTION }}

            # name of the AKS cluster
            AZURE_AKS="${{ secrets.AZURE_RESOURCENAME }}-${GITHUB_RUN_NUMBER}-$( echo "${MATRIX}" | tr -d '_.-' )"
            echo "AZURE_AKS=${AZURE_AKS}" >> $GITHUB_ENV

            # gather the storage account Key
            AZURE_STORAGE_KEY=$(az storage account keys list -g "${{ secrets.AZURE_RESOURCEGROUP }}" -n "${AZURE_STORAGE_ACCOUNT}" --query "[0].value" -o tsv)
            echo "::add-mask::$AZURE_STORAGE_KEY"
            echo "AZURE_STORAGE_KEY=${AZURE_STORAGE_KEY}" >> $GITHUB_ENV

            # name of the cluster's blob container in the storage account
            AZURE_BLOB_CONTAINER="$( echo "${MATRIX}" | tr -d '_.-' | tr '[:upper:]' '[:lower:]' )"
            echo "AZURE_BLOB_CONTAINER=${AZURE_BLOB_CONTAINER}" >> $GITHUB_ENV

            # create and login to the AKS cluster
            az aks create --resource-group ${{ secrets.AZURE_RESOURCEGROUP }} \
              --name ${AZURE_AKS} \
              --tier standard \
              --node-count 3 -k v${K8S_VERSION} --generate-ssh-keys --enable-addons monitoring \
              --workspace-resource-id ${{ secrets.AZURE_WORKSPACE_RESOURCE_ID }} \
              --aks-custom-headers EnableAzureDiskFileCSIDriver=true
            az aks get-credentials --resource-group ${{ secrets.AZURE_RESOURCEGROUP }} \
              --name ${AZURE_AKS}

            # create diagnostic settings for monitoring kube-apiserver logs
            AKS_CLUSTER_RESOURCE_ID=$(az aks show --resource-group ${{ secrets.AZURE_RESOURCEGROUP }} --name ${AZURE_AKS} --query id -o tsv --only-show-errors)
            az monitor diagnostic-settings create \
              --resource-group ${{ secrets.AZURE_RESOURCEGROUP }} \
              --resource ${AKS_CLUSTER_RESOURCE_ID} \
              --name diagnostic-kube-apiserver-logs \
              --workspace ${{ secrets.AZURE_WORKSPACE_RESOURCE_ID }} \
              --logs '[ { "category": "kube-apiserver", "enabled": true } ]'
      -
        # Azure is slow in provisioning disks, and we can't wait two minutes
        # every time we create a pod, otherwise all the tests will time out.
        # We set up a few large disks now, we run Rook on top of them and we
        # use rook to get the small PV we use in the tests.
        # It can still take a while to deploy rook.
        name: Set up Rook
        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
        with:
          timeout_minutes: 27
          max_attempts: 1
          command: |
            STORAGECLASSNAME=default
            # renovate: datasource=github-releases depName=mikefarah/yq
            go install github.com/mikefarah/yq/v4@v4
            ROOK_BASE_URL=https://raw.githubusercontent.com/rook/rook/${{ env.ROOK_VERSION }}/deploy/examples
            kubectl apply -f ${ROOK_BASE_URL}/crds.yaml
            kubectl apply -f ${ROOK_BASE_URL}/common.yaml
            kubectl apply -f ${ROOK_BASE_URL}/operator.yaml
            kubectl apply -f ${ROOK_BASE_URL}/csi-operator.yaml
            curl ${ROOK_BASE_URL}/cluster-on-pvc.yaml | \
              sed '/^ *#/d;/^ *$/d' | \
              yq e ".spec.storage.storageClassDeviceSets[].volumeClaimTemplates[].spec.resources.requests.storage = \"50Gi\" |
                .spec.storage.storageClassDeviceSets[].volumeClaimTemplates[].spec.storageClassName = \"${STORAGECLASSNAME}\" |
                .spec.mon.volumeClaimTemplate.spec.storageClassName = \"${STORAGECLASSNAME}\" " - | \
              kubectl apply -f -
            while true; do
              output=$( kubectl get deploy -n rook-ceph -l app=rook-ceph-osd --no-headers -o name )
              if [[ $(wc -w <<< $output)  == 3 ]]; then
                break
              fi
            done
            echo "Waiting for Rook OSDs to be available"
            kubectl wait deploy -n rook-ceph --for condition=available --timeout 480s -l app=rook-ceph-osd
            kubectl apply -f ${ROOK_BASE_URL}/csi/rbd/storageclass.yaml
            kubectl apply -f ${ROOK_BASE_URL}/csi/rbd/snapshotclass.yaml
            kubectl annotate storageclass ${{env.E2E_DEFAULT_STORAGE_CLASS}} storage.kubernetes.io/default-snapshot-class=${{env.E2E_DEFAULT_VOLUMESNAPSHOT_CLASS}} --overwrite
      -
        name: Prepare patch for customization
        env:
          ## the following variable all need be set if we use env_override_customized.yaml.template
          ## this is customization for aks
          LEADER_ELECTION: "true"
          LEADER_LEASE_DURATION: 15
          LEADER_RENEW_DEADLINE: 10
          LIVENESS_PROBE_THRESHOLD: 3
          LOG_LEVEL: ${{ needs.evaluate_options.outputs.log_level }}
        run: |
          LOG_LEVEL=${LOG_LEVEL:-info}
          envsubst < hack/e2e/env_override_customized.yaml.template > config/manager/env_override.yaml
          cat config/manager/env_override.yaml
      -
        name: Run E2E tests
        run: hack/e2e/run-e2e.sh
      -
        # Summarize the failed E2E test cases if there are any
        name: Report failed E2E tests
        if: failure()
        run: |
          set +x
          chmod +x .github/report-failed-test.sh
          ./.github/report-failed-test.sh
      -
        # Create an individual artifact for each E2E test, which will be used to
        # generate E2E test summary in the follow-up job 'summarize-e2e-tests'
        name: Create individual artifact for each E2E test
        if: (always() && !cancelled())
        env:
          RUNNER: "aks"
          RUN_ID: ${{ github.run_id }}
          REPOSITORY: ${{ github.repository }}
        run: |
          set +x
          python .github/generate-test-artifacts.py \
            -o testartifacts-${{ env.MATRIX }} \
            -f tests/e2e/out/report.json \
            --environment=true
          if [ -f tests/e2e/out/upgrade_report.json ]; then
            python .github/generate-test-artifacts.py \
              -o testartifacts-${{ env.MATRIX }} \
              -f tests/e2e/out/upgrade_report.json \
              --environment=true
          fi
      -
        name: Archive test artifacts
        if: (always() && !cancelled())
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: testartifacts-${{ env.MATRIX }}
          path: testartifacts-${{ env.MATRIX }}/
          retention-days: 7
      -
        name: Cleanup test artifacts
        if: always()
        run:
          rm -rf testartifacts-${{ env.MATRIX }}/
      -
        name: Cleanup ginkgo JSON report
        # Delete report.json after the analysis. File should always exist.
        # Delete upgrade_report.json. It may not exist depending on test level.
        if: always()
        run: |
          if [ -f tests/e2e/out/upgrade_report.json ]; then
            rm tests/e2e/out/upgrade_report.json
          fi
          if [ -f tests/e2e/out/report.json ]; then
            rm tests/e2e/out/report.json
          fi
      -
        name: Archive e2e failure contexts
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: test-failure-contexts-${{ env.MATRIX }}
          path: |
            tests/*/out/
          retention-days: 7
          if-no-files-found: ignore
      -
        name: Archive e2e logs
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: cluster-logs-${{ env.MATRIX }}
          path: |
            tests/e2e/cluster_logs/**
          retention-days: 7
          if-no-files-found: ignore
      -
        name: Clean up
        if: always()
        run: |
          set +e
          az extension add --allow-preview true --name monitor-control-service
          az account set --subscription ${{ secrets.AZURE_SUBSCRIPTION }}
          attempt=1
          max_attempts=3
          while [ "${attempt}" -le "${max_attempts}" ]; do
              echo "Deleting cluster. Attempt ${attempt} of ${max_attempts}"
              az aks delete --resource-group ${{ secrets.AZURE_RESOURCEGROUP }} -y --name ${{ env.AZURE_AKS }}
              status=$?
              if [[ $status == 0 ]]; then
                echo "AKS cluster deleted"
                break
              fi
              echo "Failed deleting cluster ${{ env.AZURE_AKS }}, retrying"
              sleep 5
              attempt=$((attempt+1))
          done
          attempt=1
          AZURE_RESOURCEGROUP_LOCATION="$( az group show --resource-group ${{ secrets.AZURE_RESOURCEGROUP }} --query location -o tsv --only-show-errors )"
          DATA_COLL_RULE_NAME="MSCI-${AZURE_RESOURCEGROUP_LOCATION}-${{ env.AZURE_AKS }}"
          while [ "${attempt}" -le "${max_attempts}" ]; do
              echo "Deleting data-collection rule ${DATA_COLL_RULE_NAME}. Attempt ${attempt} of ${max_attempts}"
              az monitor data-collection rule show --name ${DATA_COLL_RULE_NAME} --resource-group ${{ secrets.AZURE_RESOURCEGROUP }} --query name
              # if not found, let it go
              status=$?
              if [[ $status != 0 ]]; then
                echo "AKS data-collection rule not found"
                break
              fi
              az monitor data-collection rule delete -y --name ${DATA_COLL_RULE_NAME} --resource-group ${{ secrets.AZURE_RESOURCEGROUP }}
              status=$?
              if [[ $status == 0 ]]; then
                echo "AKS data-collection rule deleted"
                break
              fi
              echo "Failed deleting data-collection rule ${DATA_COLL_RULE_NAME}, retrying"
              sleep 5
              attempt=$((attempt+1))
          done

  e2e-aks-teardown:
    name: Teardown Microsoft AKS shared resources
    if: |
      always() &&
      vars.AKS_ENABLED == 'true' &&
      needs.generate-jobs.outputs.aksEnabled == 'true' &&
      needs.generate-jobs.result == 'success' &&
      needs.e2e-aks-setup.result == 'success'
    needs:
      - buildx
      - generate-jobs
      - e2e-aks-setup
      - e2e-aks
    runs-on: ubuntu-24.04
    env:
      AZURE_STORAGE_ACCOUNT: ${{ needs.e2e-aks-setup.outputs.azure_storage_account }}
    steps:
      -
        name: Azure Login
        if: always()
        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      -
        name: Teardown AKS shared resources
        if: always()
        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
        with:
          timeout_minutes: 5
          max_attempts: 3
          command: |
            az account set --subscription ${{ secrets.AZURE_SUBSCRIPTION }}
            az storage account delete -y --resource-group ${{ secrets.AZURE_RESOURCEGROUP }} --name ${{ env.AZURE_STORAGE_ACCOUNT }}

  # EKS Secrets required
  # secrets.AWS_EKS_ADMIN_IAM_ROLES
  # secrets.AWS_ACCESS_KEY_ID
  # secrets.AWS_SECRET_ACCESS_KEY

  e2e-eks:
    name: Run E2E on Amazon EKS
    if: |
      (always() && !cancelled()) &&
      vars.EKS_ENABLED == 'true' &&
      needs.generate-jobs.outputs.eksEnabled == 'true' &&
      needs.generate-jobs.result == 'success'
    needs:
      - buildx
      - generate-jobs
      - evaluate_options
    strategy:
      fail-fast: false
      max-parallel: 6
      matrix: ${{ fromJSON(needs.generate-jobs.outputs.eksMatrix) }}
    runs-on: ubuntu-24.04
    env:
      # TEST_DEPTH determines the maximum test level the suite should be running
      TEST_DEPTH: ${{ needs.evaluate_options.outputs.test_level }}
      # FEATURE_TYPE, when defined, determines the subset of E2E tests that will be executed, divided by feature type
      FEATURE_TYPE: ${{ needs.evaluate_options.outputs.feature_type }}

      K8S_VERSION: "${{ matrix.k8s_version }}"
      POSTGRES_VERSION: ${{ matrix.postgres_version }}
      POSTGRES_KIND: ${{ matrix.postgres_kind }}
      MATRIX: ${{ matrix.id }}
      POSTGRES_IMG: "${{ matrix.postgres_img }}"
      # The version of operator to upgrade FROM, in the rolling upgrade E2E test
      E2E_PRE_ROLLING_UPDATE_IMG: "${{ matrix.postgres_pre_img }}"
      TEST_TIMEOUTS: ${{ needs.generate-jobs.outputs.eksTimeout }}
      BRANCH_NAME: ${{ needs.buildx.outputs.branch_name }}
      GIT_REF: ${{ needs.evaluate_options.outputs.git_ref }}

      DEBUG: "true"
      BUILD_IMAGE: "false"
      CONTROLLER_IMG: ${{ needs.generate-jobs.outputs.controller_img }}
      CONTROLLER_IMG_DIGEST: ${{ needs.buildx.outputs.controller_img_digest }}
      CONTROLLER_IMG_PRIME_DIGEST: ${{ needs.buildx.outputs.controller_img_prime_digest }}
      E2E_DEFAULT_STORAGE_CLASS: gp3
      E2E_CSI_STORAGE_CLASS: gp3
      E2E_DEFAULT_VOLUMESNAPSHOT_CLASS: ebs-csi-snapclass

      AWS_REGION: eu-central-1
      AWS_EKS_ADMIN_IAM_ROLES: ${{ secrets.AWS_EKS_ADMIN_IAM_ROLES }}
      TEST_CLOUD_VENDOR: "eks"

    steps:
      -
        name: Set cluster name
        run: |
          echo "CLUSTER_NAME=${E2E_SUFFIX}-test-${GITHUB_RUN_NUMBER}-$( echo "${MATRIX}" | tr -d '_.-' )" >> $GITHUB_ENV
      -
        name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ env.GIT_REF }}
      -
        name: Install Go
        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
        with:
          go-version-file: go.mod
          check-latest: true
      -
        ## In case hack/setup-cluster.sh need pull operand image from registry
        name: Login into docker registry
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ env.REGISTRY_USER }}
          password: ${{ env.REGISTRY_PASSWORD }}
      -
        name: Prepare the environment
        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
        with:
          timeout_seconds: 300
          max_attempts: 3
          command: |
            sudo apt-get update
            sudo apt-get install -y gettext-base
      -
        name: Install ginkgo
        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
        with:
          timeout_minutes: 1
          max_attempts: 3
          command: |
            # renovate: datasource=github-releases depName=onsi/ginkgo
            go install github.com/onsi/ginkgo/v2/ginkgo@v2.28.1
      -
        name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ env.AWS_REGION }}
      -
        name: Install eksctl
        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
        with:
          timeout_minutes: 1
          max_attempts: 3
          command: |
            mkdir -p "$HOME/.local/bin"
            curl -sL "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" \
              | tar xz -C $HOME/.local/bin
            echo "$HOME/.local/bin" >> $GITHUB_PATH
      -
        name: Configure EKS setup
        run: |
          envsubst < hack/e2e/eks-cluster.yaml.template > hack/e2e/eks-cluster.yaml
      -
        name: Setup EKS
        run: |
          # Setting up EKS cluster
          echo "create cluster"
          eksctl create cluster -f hack/e2e/eks-cluster.yaml

          # Create iamidentitymapping
          echo "$AWS_EKS_ADMIN_IAM_ROLES" | while read role
          do
            # Masking variables to hide values
            echo "::add-mask::$role"
            eksctl create iamidentitymapping --cluster "${CLUSTER_NAME}" --region="${AWS_REGION}" --arn "${role}" --group system:masters --username admin
          done

          # Updating .kubeconfig to use the correct version of client.authentication.k8s.io API
          aws eks update-kubeconfig --name ${CLUSTER_NAME} --region ${AWS_REGION}

          # Installing CRD for support volumeSnapshot
          SNAPSHOTTER_BASE_URL=https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${{env.EXTERNAL_SNAPSHOTTER_VERSION}}
          kubectl apply -f ${SNAPSHOTTER_BASE_URL}/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
          kubectl apply -f ${SNAPSHOTTER_BASE_URL}/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
          kubectl apply -f ${SNAPSHOTTER_BASE_URL}/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml

          ## Controller
          kubectl apply -f ${SNAPSHOTTER_BASE_URL}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml
          kubectl apply -f ${SNAPSHOTTER_BASE_URL}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml

          # Install volume snapshot class
          kubectl apply -f hack/e2e/volumesnapshotclass-ebs-csi.yaml
          kubectl get volumesnapshotclass

          # Change to use gp3 as default storage account
          kubectl annotate storageclass gp2 storageclass.kubernetes.io/is-default-class=false --overwrite
          kubectl apply -f hack/e2e/storage-class-gp3.yaml
          kubectl annotate storageclass ${{env.E2E_DEFAULT_STORAGE_CLASS}} storage.kubernetes.io/default-snapshot-class=${{env.E2E_DEFAULT_VOLUMESNAPSHOT_CLASS}} --overwrite
          kubectl get storageclass
      -
        name: Setup Velero
        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
        env:
          # renovate: datasource=github-releases depName=vmware-tanzu/velero
          VELERO_VERSION: "v1.18.0"
          # renovate: datasource=github-releases depName=vmware-tanzu/velero-plugin-for-aws
          VELERO_AWS_PLUGIN_VERSION: "v1.14.0"
        with:
          timeout_minutes: 10
          max_attempts: 3
          on_retry_command: |
            # Clean up buckets
            output=$( aws s3api delete-bucket --bucket "${VELERO_BUCKET_NAME}" --region "${AWS_REGION}" 2>&1 )
            status=$?
            if [[ $status == 0 ]]; then
              echo "S3 Bucket deleted"
              break
            fi
            if ( grep "NoSuchBucket" <<< "$output" ); then
              echo "S3 Bucket doesn't exist, nothing to remove"
              break
            fi

            # Uninstall Velero
            kubectl delete namespace/velero clusterrolebinding/velero
            kubectl delete crds -l component=velero
          command: |
            VELERO_BUCKET_NAME="${CLUSTER_NAME,,}-velero"
            echo "VELERO_BUCKET_NAME=${VELERO_BUCKET_NAME}" >> $GITHUB_ENV

            # Create S3 bucket
            aws s3api create-bucket \
              --bucket "${VELERO_BUCKET_NAME}" \
              --region "${AWS_REGION}" \
              --create-bucket-configuration LocationConstraint="${AWS_REGION}"

            # Download Velero, extract and place it in $PATH
            curl -sL "https://github.com/vmware-tanzu/velero/releases/download/${VELERO_VERSION}/velero-${VELERO_VERSION}-linux-amd64.tar.gz" | tar xz
            mv velero-${VELERO_VERSION}-linux-amd64/velero $HOME/.local/bin

            # Set Velero-specific credentials
            echo -e "[default]\naws_access_key_id=${{ secrets.AWS_ACCESS_KEY_ID }}\naws_secret_access_key=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> credentials-velero

            # Install Velero
            velero install \
              --provider aws \
              --plugins velero/velero-plugin-for-aws:${VELERO_AWS_PLUGIN_VERSION} \
              --bucket "${VELERO_BUCKET_NAME}" \
              --backup-location-config region="${AWS_REGION}" \
              --snapshot-location-config region="${AWS_REGION}" \
              --secret-file ./credentials-velero \
              --wait
      -
        name: Prepare patch for customization
        env:
          ## the following variable all need be set if we use env_override_customized.yaml.template
          ## this is customization for eks
          LEADER_ELECTION: "true"
          LEADER_LEASE_DURATION: 15
          LEADER_RENEW_DEADLINE: 10
          LIVENESS_PROBE_THRESHOLD: 3
          LOG_LEVEL: ${{ needs.evaluate_options.outputs.log_level }}
        run: |
          LOG_LEVEL=${LOG_LEVEL:-info}
          envsubst < hack/e2e/env_override_customized.yaml.template > config/manager/env_override.yaml
          cat config/manager/env_override.yaml
      -
        name: Run E2E tests
        run: hack/e2e/run-e2e.sh
      -
        # Summarize the failed E2E test cases if there are any
        name: Report failed E2E tests
        if: failure()
        run: |
          set +x
          chmod +x .github/report-failed-test.sh
          ./.github/report-failed-test.sh
      -
        # Create an individual artifact for each E2E test, which will be used to
        # generate E2E test summary in the follow-up job 'summarize-e2e-tests'
        name: Create individual artifact for each E2E test
        if: (always() && !cancelled())
        env:
          RUNNER: "eks"
          RUN_ID: ${{ github.run_id }}
          REPOSITORY: ${{ github.repository }}
        run: |
          set +x
          python .github/generate-test-artifacts.py \
            -o testartifacts-${{ env.MATRIX }} \
            -f tests/e2e/out/report.json \
            --environment=true
          if [ -f tests/e2e/out/upgrade_report.json ]; then
            python .github/generate-test-artifacts.py \
              -o testartifacts-${{ env.MATRIX }} \
              -f tests/e2e/out/upgrade_report.json \
              --environment=true
          fi
      -
        name: Archive test artifacts
        if: (always() && !cancelled())
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: testartifacts-${{ env.MATRIX }}
          path: testartifacts-${{ env.MATRIX }}/
          retention-days: 7
      -
        name: Cleanup test artifacts
        if: always()
        run:
          rm -rf testartifacts-${{ env.MATRIX }}/
      -
        name: Cleanup ginkgo JSON report
        # Delete report.json after the analysis. File should always exist.
        # Delete upgrade_report.json. It may not exist depending on test level.
        if: always()
        run: |
          if [ -f tests/e2e/out/upgrade_report.json ]; then
            rm tests/e2e/out/upgrade_report.json
          fi
          if [ -f tests/e2e/out/report.json ]; then
            rm tests/e2e/out/report.json
          fi
      -
        name: Archive e2e failure contexts
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: test-failure-contexts-${{ env.MATRIX }}
          path: |
            tests/*/out/
          retention-days: 7
          if-no-files-found: ignore
      -
        name: Archive e2e logs
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: cluster-logs-${{ env.MATRIX }}
          path: |
            tests/e2e/cluster_logs/**
          retention-days: 7
          if-no-files-found: ignore
      -
        name: Clean up
        if: always()
        run: |
          set +e
          CLUSTER_NAME="${{ env.CLUSTER_NAME }}"
          REGION_NAME="${{ env.AWS_REGION }}"
          STACK_NAME="eksctl-${CLUSTER_NAME}-cluster"
          CLOUDFORMATION_STATUS_CURRENT=$(aws cloudformation describe-stacks --stack-name "${STACK_NAME}" | jq -r '.Stacks[].StackStatus')
          if [[ -z "${CLOUDFORMATION_STATUS_CURRENT}" ]]; then
            echo "CloudFormation stack not found. Nothing to cleanup."
            exit 0
          fi

          # Attempt to remove any leftover PDB (and Cluster that would recreate it)
          # that could prevent the EKS cluster deletion
          kubectl delete cluster --all --all-namespaces --now --timeout=30s || true
          kubectl delete pdb --all --all-namespaces --now --timeout=30s || true
          kubectl delete pvc --all --all-namespaces --now --timeout=30s || true
          # Remove any LoadBalancer service
          kubectl get service --all-namespaces -o json | jq -r '.items[] | select(.spec.type=="LoadBalancer") | .metadata | "kubectl delete service --now --timeout=30s -n " + .namespace + " " + .name' | xargs -rI X bash -c X || true

          NODEGROUP_STACK_NAMES=$(eksctl get nodegroup --cluster "${CLUSTER_NAME}" -o json | jq -r '.[].StackName' || true)

          attempt=1
          bucket_attempt=1
          max_attempts=3

          # Attempting three times to remove the Velero S3 bucket
          VELERO_BUCKET_NAME=${VELERO_BUCKET_NAME:-"${CLUSTER_NAME,,}-velero"}
          while [ "${bucket_attempt}" -le "${max_attempts}" ]; do
              echo "Deleting S3 Bucket. Attempt ${bucket_attempt} of ${max_attempts}"
              output=$( aws s3api delete-bucket --bucket "${VELERO_BUCKET_NAME}" --region "${AWS_REGION}" 2>&1 )
              status=$?
              if [[ $status == 0 ]]; then
                  echo "S3 Bucket deleted"
                  break
              fi
              if ( grep "NoSuchBucket" <<< "$output" ); then
                  echo "S3 Bucket doesn't exist, nothing to remove"
                  break
              fi
              echo "Failed deleting S3 Bucket ${VELERO_BUCKET_NAME}, retrying"
              sleep 5
              bucket_attempt=$((bucket_attempt+1))
          done

          # Attempting three times to cleanly remove the cluster via eksctl
          while [ "${attempt}" -le "${max_attempts}" ]; do
              echo "Deleting cluster. Attempt ${attempt} of ${max_attempts}"
              output=$( eksctl delete cluster -n "${CLUSTER_NAME}" -r "${REGION_NAME}" --wait --force 2>&1 )
              status=$?
              if [[ $status == 0 ]]; then
                  echo "EKS cluster deleted"
                  break
              fi
              if ( grep "ResourceNotFoundException: No cluster found for name: ${CLUSTER_NAME}" <<< "$output" ); then
                  echo "EKS cluster doesn't exist, nothing to remove"
                  break
              fi
              echo "Failed deleting cluster ${CLUSTER_NAME}, retrying"
              sleep 5
              attempt=$((attempt+1))
          done

          # Recheck if something got stuck, and use harder methods to clean up
          CLOUDFORMATION_STATUS_CURRENT=$(aws cloudformation describe-stacks --stack-name "${STACK_NAME}" | jq -r '.Stacks[].StackStatus')
          if [ -n "${CLOUDFORMATION_STATUS_CURRENT}" ] ; then
            echo "::warning file=continuous-delivery.yml::eksctl failed deleting a cluster cleanly"
            # When the status of CloudFormation stack managed by eksctl reports an error, try to delete resources directly with AWS CLI
            pip install -r .github/workflow-requirements.txt
            for vpc_id in $(aws ec2 describe-vpcs | jq -r '.Vpcs[] | select(.Tags?[]? | .Key == "Name" and (.Value | contains("'"${STACK_NAME}"'"))).VpcId'); do
              python .github/vpc_destroy.py --vpc_id "${vpc_id}" --region "${REGION_NAME}" --services ec2
            done
            # Then we try to delete the cluster cleanly and the cloudformation
            if aws eks describe-cluster --name "${CLUSTER_NAME}" --region "${REGION_NAME}" ; then
               eksctl delete cluster -n "${CLUSTER_NAME}" -r "${REGION_NAME}" --wait --force
            fi
            if [ -n "${NODEGROUP_STACK_NAMES}" ] ; then
              for NODEGROUP_STACK_NAME in ${NODEGROUP_STACK_NAMES}; do
                if aws cloudformation describe-stacks --stack-name "${NODEGROUP_STACK_NAME}" --region "${REGION_NAME}" ; then
                   aws cloudformation delete-stack --stack-name "${NODEGROUP_STACK_NAME}" --region "${REGION_NAME}"
                fi
              done
            fi
            if aws cloudformation describe-stacks --stack-name "${STACK_NAME}" --region "${REGION_NAME}" ; then
               aws cloudformation delete-stack --stack-name "${STACK_NAME}" --region "${REGION_NAME}"
            fi
          fi

          # Clear up leftover volumes
          while read -r volume; do
            echo "Deleting $volume of cluster $CLUSTER_NAME ..."
            if ! aws ec2 delete-volume --region "${REGION_NAME}" --volume-id "$volume" ; then
                echo "::warning file=continuous-delivery.yml::Failed deleting $volume of cluster $CLUSTER_NAME"
            fi
          done < <(aws ec2 describe-volumes --region "${REGION_NAME}" --query 'Volumes[?not_null(Tags[?Key == `kubernetes.io/cluster/'"$CLUSTER_NAME"'` && Value == `owned`].Value)].VolumeId' | jq -r '.[]' || true)

  # GKE Secrets required
  # secrets.GCP_SERVICE_ACCOUNT
  # secrets.GCP_PROJECT_ID

  e2e-gke:
    name: Run E2E on Google GKE
    if: |
      (always() && !cancelled()) &&
      vars.GKE_ENABLED == 'true' &&
      needs.generate-jobs.outputs.gkeEnabled == 'true' &&
      needs.generate-jobs.result == 'success'
    needs:
      - buildx
      - generate-jobs
      - evaluate_options
    strategy:
      fail-fast: false
      max-parallel: 6
      matrix: ${{ fromJSON(needs.generate-jobs.outputs.gkeMatrix) }}
    runs-on: ubuntu-24.04
    env:
      # TEST_DEPTH determines the maximum test level the suite should be running
      TEST_DEPTH: ${{ needs.evaluate_options.outputs.test_level }}
      # FEATURE_TYPE, when defined, determines the subset of E2E tests that will be executed, divided by feature type
      FEATURE_TYPE: ${{ needs.evaluate_options.outputs.feature_type }}

      K8S_VERSION: "${{ matrix.k8s_version }}"
      POSTGRES_VERSION: ${{ matrix.postgres_version }}
      POSTGRES_KIND: ${{ matrix.postgres_kind }}
      MATRIX: ${{ matrix.id }}
      POSTGRES_IMG: "${{ matrix.postgres_img }}"
      # The version of operator to upgrade FROM, in the rolling upgrade E2E test
      E2E_PRE_ROLLING_UPDATE_IMG: "${{ matrix.postgres_pre_img }}"
      TEST_TIMEOUTS: ${{ needs.generate-jobs.outputs.gkeTimeout }}
      BRANCH_NAME: ${{ needs.buildx.outputs.branch_name }}
      GIT_REF: ${{ needs.evaluate_options.outputs.git_ref }}

      DEBUG: "true"
      BUILD_IMAGE: "false"
      CONTROLLER_IMG: ${{ needs.generate-jobs.outputs.controller_img }}
      CONTROLLER_IMG_DIGEST: ${{ needs.buildx.outputs.controller_img_digest }}
      CONTROLLER_IMG_PRIME_DIGEST: ${{ needs.buildx.outputs.controller_img_prime_digest }}
      E2E_DEFAULT_STORAGE_CLASS: standard-rwo
      E2E_CSI_STORAGE_CLASS: standard-rwo
      E2E_DEFAULT_VOLUMESNAPSHOT_CLASS: pd-csi-snapclass

      REGION: europe-west3
      TEST_CLOUD_VENDOR: "gke"

    steps:
      -
        name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ env.GIT_REF }}
      -
        name: Install Go
        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
        with:
          go-version-file: go.mod
          check-latest: true
      -
        ## In case hack/setup-cluster.sh need pull operand image from registry
        name: Login into docker registry
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ env.REGISTRY_USER }}
          password: ${{ env.REGISTRY_PASSWORD }}
      -
        name: Prepare the environment
        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
        with:
          timeout_seconds: 300
          max_attempts: 3
          command: |
            sudo apt-get update
            sudo apt-get install -y gettext-base
      -
        name: Install ginkgo
        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
        with:
          timeout_seconds: 120
          max_attempts: 3
          command: |
            # renovate: datasource=github-releases depName=onsi/ginkgo
            go install github.com/onsi/ginkgo/v2/ginkgo@v2.28.1
      -
        name: Set cluster name
        run: |
          # GKE cluster names rules:
          # only lowercase alphanumerics and '-' allowed, must start with a letter and end with an alphanumeric,
          # and must be no longer than 40 characters
          # We need to shorten the name and lower the case
          SHORT_ID=$( echo "${MATRIX}" | tr -d '_.-' | tr '[:upper:]' '[:lower:]')
          echo "CLUSTER_NAME=${E2E_SUFFIX}-${GITHUB_RUN_NUMBER}-${SHORT_ID}" >> $GITHUB_ENV
      -
        name: Authenticate to Google Cloud
        id: 'auth'
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
        with:
          credentials_json: '${{ secrets.GCP_SERVICE_ACCOUNT }}'
      -
        name: Set up Cloud SDK and kubectl
        uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
          install_components: 'kubectl,gke-gcloud-auth-plugin'
      -
        name: Create GKE cluster
        run: |
          set +e
          # We may go over the amount of API requests allowed
          # by Google when creating all the clusters at the same time.
          # We give a few attempts at creating the cluster before giving up.
          # The following command will create a 3 nodes cluster, with each
          # node deployed in its own availability zone.
          for i in `seq 1 5`; do
            if gcloud container clusters create ${{ env.CLUSTER_NAME }} \
              --num-nodes=1 \
              --cluster-version=${{ env.K8S_VERSION }} \
              --region=${{ env.REGION }} \
              --disk-size=20 \
              --machine-type=e2-standard-2 \
              --labels=cluster=${{ env.CLUSTER_NAME }}
            then
              exit 0
            fi
            echo "Couldn't create the cluster. Retrying in 100s."
            sleep 100
          done
          echo "Couldn't create the cluster. Failing."
          exit 1
      -
        name: Get GKE kubeconfig credentials
        env:
          USE_GKE_GCLOUD_AUTH_PLUGIN: "True"
        run: |
          gcloud container clusters get-credentials ${{ env.CLUSTER_NAME }} --region ${{ env.REGION }} --project ${{ secrets.GCP_PROJECT_ID }}
      -
        name: Configure Storage
        run: |
          # Even if the cluster is ready, it might take some time before the
          # volumesnapshot CRDs are available
          attempt=1
          max_attempts=10
          while [ "${attempt}" -le "${max_attempts}" ]; do
            if kubectl get crd volumesnapshotclasses.snapshot.storage.k8s.io >/dev/null 2>&1; then
              echo "VolumeSnapshot CRDs are available."
              break
            fi
            echo "Waiting for snapshot CRDs..."
            sleep 20
            attempt=$((attempt+1))
          done
          # Install volume snapshot class
          kubectl apply -f hack/e2e/volumesnapshotclass-pd-csi.yaml
          # Change to use standard-rwo as default storage account
          kubectl annotate storageclass ${{env.E2E_DEFAULT_STORAGE_CLASS}} storage.kubernetes.io/default-snapshot-class=${{env.E2E_DEFAULT_VOLUMESNAPSHOT_CLASS}} --overwrite
          kubectl get storageclass
      -
        name: Prepare patch for customization
        env:
          ## the following variable all need be set if we use env_override_customized.yaml.template
          ## this is customization for gke
          LEADER_ELECTION: "false"
          LEADER_LEASE_DURATION: 240
          LEADER_RENEW_DEADLINE: 230
          LIVENESS_PROBE_THRESHOLD: 9
          LOG_LEVEL: ${{ needs.evaluate_options.outputs.log_level }}
        run: |
          LOG_LEVEL=${LOG_LEVEL:-info}
          envsubst < hack/e2e/env_override_customized.yaml.template > config/manager/env_override.yaml
          cat config/manager/env_override.yaml
      -
        name: Run E2E tests
        run: hack/e2e/run-e2e.sh
      -
        name: Report failed E2E tests
        if: failure()
        run: |
          set +x
          chmod +x .github/report-failed-test.sh
          ./.github/report-failed-test.sh
      -
        # Create an individual artifact for each E2E test, which will be used to
        # generate E2E test summary in the follow-up job 'summarize-e2e-tests'
        name: Create individual artifact for each E2E test
        if: (always() && !cancelled())
        env:
          RUNNER: "gke"
          RUN_ID: ${{ github.run_id }}
          REPOSITORY: ${{ github.repository }}
        run: |
          set +x
          python .github/generate-test-artifacts.py \
            -o testartifacts-${{ env.MATRIX }} \
            -f tests/e2e/out/report.json \
            --environment=true
          if [ -f tests/e2e/out/upgrade_report.json ]; then
            python .github/generate-test-artifacts.py \
              -o testartifacts-${{ env.MATRIX }} \
              -f tests/e2e/out/upgrade_report.json \
              --environment=true
          fi
      -
        name: Archive test artifacts
        if: (always() && !cancelled())
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: testartifacts-${{ env.MATRIX }}
          path: testartifacts-${{ env.MATRIX }}/
          retention-days: 7
      -
        name: Cleanup test artifacts
        if: always()
        run:
          rm -rf testartifacts-${{ env.MATRIX }}/
      -
        name: Cleanup ginkgo JSON report
        # Delete report.json after the analysis. File should always exist.
        # Delete upgrade_report.json. It may not exist depending on test level.
        if: always()
        run: |
          if [ -f tests/e2e/out/upgrade_report.json ]; then
            rm tests/e2e/out/upgrade_report.json
          fi
          if [ -f tests/e2e/out/report.json ]; then
            rm tests/e2e/out/report.json
          fi
      -
        name: Archive e2e failure contexts
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: test-failure-contexts-${{ env.MATRIX }}
          path: |
            tests/*/out/
          retention-days: 7
          if-no-files-found: ignore
      -
        name: Archive e2e logs
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: cluster-logs-${{ env.MATRIX }}
          path: |
            tests/e2e/cluster_logs/**
          retention-days: 7
          if-no-files-found: ignore
      -
        name: Clean up
        if: always()
        run: |
          set +e
          # Attempt to remove any leftover resource
          kubectl delete cluster --all --all-namespaces --now --timeout=30s || true
          kubectl delete pdb --all --all-namespaces --now --timeout=30s || true
          kubectl delete pvc --all --all-namespaces --now --timeout=30s || true

          # Wait until all the PVs provisioned are actually reclaimed
          kubectl wait --for delete --all pv --timeout=60s || true

          attempt=1
          max_attempts=3
          while [ "${attempt}" -le "${max_attempts}" ]; do
            gcloud container clusters delete ${{ env.CLUSTER_NAME }} --region=${{ env.REGION }} --quiet
            status=$?
            if [[ $status == 0 ]]; then
                echo "GKS cluster ${{ env.CLUSTER_NAME }} deleted from region ${{ env.REGION }}"
                break
            fi
            echo "Failed deleting cluster ${{ env.CLUSTER_NAME }} from region ${{ env.REGION }}, retrying"
            sleep 5
            attempt=$((attempt+1))
          done

          # The node's disks are not automatically deleted when the cluster is removed.
          # We delete all the disks tagged with the name of the cluster that are not
          # owned by anyone.

          attempt=1
          max_attempts=3
          while [ "${attempt}" -le "${max_attempts}" ]; do
            IDS=$(gcloud compute disks list --filter="labels.cluster=${{ env.CLUSTER_NAME }} AND region:${{ env.REGION }} AND -users:*" --format="value(id)")
            amount="$(echo $IDS | awk '{print NF}')"
            if [[ "$amount" == 3 ]]; then
              echo -e "Found the 3 disks to be removed:\n$IDS"
              break
            fi
            echo "Expected 3 disks to delete but found $amount, waiting and retrying"
            sleep 20
            attempt=$((attempt+1))
          done

          for ID in ${IDS}
          do
            attempt=1
            max_attempts=3
            while [ "${attempt}" -le "${max_attempts}" ]; do
               gcloud compute disks delete --region "${{ env.REGION }}" --quiet "${ID}"
               status=$?
               if [[ $status == 0 ]]; then
                  echo "computer disk ${ID} deleted"
                  break
               fi
               echo "Failed deleting disk ${ID} from region ${{ env.REGION }}, retrying"
               sleep 5
               attempt=$((attempt+1))
            done
          done

  # OpenShift Secrets required
  # secrets.AWS_EKS_ADMIN_IAM_ROLES
  # secrets.AWS_ACCESS_KEY_ID
  # secrets.AWS_SECRET_ACCESS_KEY

  e2e-openshift:
    name: Run E2E on OpenShift
    if: |
      always() && !cancelled() &&
      vars.OPENSHIFT_ENABLED == 'true' &&
      needs.generate-jobs.outputs.openshiftEnabled == 'true' &&
      needs.generate-jobs.result == 'success'
    needs:
      - buildx
      - generate-jobs
      - evaluate_options
    strategy:
      fail-fast: false
      max-parallel: 6
      matrix: ${{ fromJSON(needs.generate-jobs.outputs.openshiftMatrix) }}
    runs-on: ubuntu-24.04
    permissions:
      contents: read
      packages: write
    env:
      # TEST_DEPTH determines the maximum test level the suite should be running
      TEST_DEPTH: ${{ needs.evaluate_options.outputs.test_level }}
      # FEATURE_TYPE, when defined, determines the subset of E2E tests that will be executed, divided by feature type
      FEATURE_TYPE: ${{ needs.evaluate_options.outputs.feature_type }}

      K8S_VERSION: "${{ matrix.k8s_version }}"
      POSTGRES_VERSION: ${{ matrix.postgres_version }}
      POSTGRES_KIND: ${{ matrix.postgres_kind }}
      MATRIX: ${{ matrix.id }}
      POSTGRES_IMG: "${{ matrix.postgres_img }}"
      # The version of operator to upgrade FROM, in the rolling upgrade E2E test
      E2E_PRE_ROLLING_UPDATE_IMG: "${{ matrix.postgres_pre_img }}"
      TEST_TIMEOUTS: ${{ needs.generate-jobs.outputs.openshiftTimeout }}
      BRANCH_NAME: ${{ needs.buildx.outputs.branch_name }}
      GIT_REF: ${{ needs.evaluate_options.outputs.git_ref }}

      DEBUG: "true"
      BUILD_IMAGE: "false"
      CONTROLLER_IMG: ${{ needs.generate-jobs.outputs.controller_img }}
      E2E_DEFAULT_STORAGE_CLASS: gp3-csi
      E2E_CSI_STORAGE_CLASS: gp3-csi
      E2E_DEFAULT_VOLUMESNAPSHOT_CLASS: csi-aws-vsc

      TEST_CLOUD_VENDOR: "ocp"

      # AWS configuration
      AWS_BASE_DOMAIN: ${{ secrets.AWS_BASE_DOMAIN }}
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_REGION: eu-central-1
      AWS_EKS_ADMIN_IAM_ROLES: ${{ secrets.AWS_EKS_ADMIN_IAM_ROLES }}

      REDHAT_PULL: ${{ secrets.REDHAT_PULL }}
      SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }}

    steps:
      -
        name: Set cluster name
        env:
          MATRIX_K8S_VERSION: ${{ matrix.k8s_version }}
        run: |
          echo "CLUSTER_NAME=${E2E_SUFFIX}-ocp-${GITHUB_RUN_NUMBER}-$( echo "${MATRIX_K8S_VERSION}" | tr -d '.' )" >> $GITHUB_ENV
      -
        name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ env.GIT_REF }}
          fetch-depth: 0
      -
        name: Install Go
        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6
        with:
          go-version-file: go.mod
          check-latest: true
      -
        name: Set up QEMU
        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4
        with:
          platforms: ${{ env.PLATFORMS }}
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
      -
        ## In case hack/setup-cluster.sh need pull operand image from registry
        name: Login into docker registry
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ env.REGISTRY_USER }}
          password: ${{ env.REGISTRY_PASSWORD }}
      -
        name: Build and push the operator and catalog
        env:
          CONTROLLER_IMG: ${{ needs.buildx.outputs.controller_img_ubi }}
          BUNDLE_IMG: ${{ needs.buildx.outputs.bundle_img }}
          INDEX_IMG: ${{ needs.buildx.outputs.index_img }}
          CATALOG_IMG: ${{ needs.buildx.outputs.catalog_img }}
        run: |
          make olm-catalog
      -
        name: Install OC Installer and client
        uses: redhat-actions/openshift-tools-installer@144527c7d98999f2652264c048c7a9bd103f8a82 # v1
        with:
          source: "mirror"
          openshift-install: ${{ env.K8S_VERSION }}
          oc: ${{ env.K8S_VERSION }}
      -
        name: Install OpenShift Cluster ${{ env.K8S_VERSION }}
        timeout-minutes: 60
        run: |
          envsubst < hack/install-config.yaml.template > hack/install-config.yaml
          openshift-install create cluster --dir hack/ --log-level debug
      -
        name: Install operator-sdk
        run: |
          make operator-sdk
      -
        name: Install preflight
        run: |
          make preflight
      -
        name: Create Secret
        run: |
          export KUBECONFIG=$(pwd)/hack/auth/kubeconfig
          oc create ns cloudnative-pg
          oc -n cloudnative-pg create secret docker-registry cnpg-pull-secret \
            --docker-server="$REGISTRY" --docker-username="$REGISTRY_USER" --docker-password="$REGISTRY_PASSWORD"
      -
        name: Run preflight operator test
        timeout-minutes: 30
        env:
          BUNDLE_IMG: ${{ needs.buildx.outputs.bundle_img }}
          PFLT_INDEXIMAGE: ${{ needs.buildx.outputs.index_img }}
          PFLT_SCORECARD_WAIT_TIME: "1200"
          PFLT_ARTIFACTS: "preflight_operator_results"
        run: |
          export PATH=$(pwd)/bin/:${PATH}
          export KUBECONFIG=$(pwd)/hack/auth/kubeconfig
          oc -n cloudnative-pg get secret cnpg-pull-secret \
            -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d > docker-config.json
          preflight check operator ${BUNDLE_IMG} \
            --docker-config docker-config.json --loglevel trace
      -
        name: Check preflight operator results
        run: |
          PASS=`jq -r .passed preflight_operator_results/results.json`
          if [[ "$PASS" == "false" ]]
          then
              exit 1
          fi
      -
        name: Run E2E tests
        timeout-minutes: 180
        if: (always() && !cancelled())
        run: |
          # Before running on OpenShift we make sure that the catalog is created
          # in the openshift-marketplace namespace
          sed -i -e 's/namespace: operators/namespace: openshift-marketplace/' cloudnative-pg-catalog.yaml
          find -type f -name "cloudnative-pg-catalog.yaml"
          cat cloudnative-pg-catalog.yaml
          KUBECONFIG=$(pwd)/hack/auth/kubeconfig bash -x hack/e2e/run-e2e-ocp.sh

      -
        # Summarize the failed E2E tests cases if there are any
        name: Report failed E2E tests
        if: failure()
        run: |
          set +x
          chmod +x .github/report-failed-test.sh
          ./.github/report-failed-test.sh
      -
        # Create an individual artifact for each E2E test, which will be used to
        # generate E2E test summary in the follow-up job 'summarize-e2e-tests'
        name: Create individual artifact for each E2E test
        if: (always() && !cancelled())
        env:
          RUNNER: "openshift"
          RUN_ID: ${{ github.run_id }}
          REPOSITORY: ${{ github.repository }}
        run: |
          set +x
          python .github/generate-test-artifacts.py \
            -o testartifacts-${{ env.MATRIX }} \
            -f tests/e2e/out/report.json \
            --environment=true
          if [ -f tests/e2e/out/upgrade_report.json ]; then
            python .github/generate-test-artifacts.py \
              -o testartifacts-${{ env.MATRIX }} \
              -f tests/e2e/out/upgrade_report.json \
              --environment=true
          fi
      -
        name: Archive test artifacts
        if: (always() && !cancelled())
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: testartifacts-${{ env.MATRIX }}
          path: testartifacts-${{ env.MATRIX }}/
          retention-days: 7
      -
        name: Cleanup test artifacts
        if: always()
        run:
          rm -rf testartifacts-${{ env.MATRIX }}/
      -
        name: Cleanup ginkgo JSON report
        # Delete report.json after the analysis. File should always exist.
        # Delete upgrade_report.json. It may not exist depending on test level.
        if: always()
        run: |
          if [ -f tests/e2e/out/upgrade_report.json ]; then
            rm tests/e2e/out/upgrade_report.json
          fi
          if [ -f tests/e2e/out/report.json ]; then
            rm tests/e2e/out/report.json
          fi
      -
        name: Archive e2e failure contexts
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: test-failure-contexts-${{ env.MATRIX }}
          path: |
            tests/*/out/
          retention-days: 7
          if-no-files-found: ignore
      -
        name: Archive e2e logs
        if: failure()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: cluster-logs-${{ env.MATRIX }}
          path: |
            tests/e2e/cluster_logs/**
          retention-days: 7
          if-no-files-found: ignore
      -
        name: Destroy OpenShift Cluster ${{ env.K8S_VERSION }}
        if: always()
        run: |
          openshift-install destroy cluster --dir hack/

  # Summarize E2E test results, display in the GitHub 'summary' view
  summarize-e2e-tests:
    name: E2E test suite
    needs:
      - evaluate_options
      - e2e-kind
      - e2e-k3d
      - e2e-eks
      - e2e-aks
      - e2e-gke
      - e2e-openshift
    if: |
      (always() && !cancelled()) &&
      ((
        needs.e2e-kind.result == 'success' ||
        needs.e2e-kind.result == 'failure'
      ) ||
      (
        needs.e2e-k3d.result == 'success' ||
        needs.e2e-k3d.result == 'failure'
      ) ||
      (
        needs.e2e-eks.result == 'success' ||
        needs.e2e-eks.result == 'failure'
      ) ||
      (
        needs.e2e-aks.result == 'success' ||
        needs.e2e-aks.result == 'failure'
      ) ||
      (
        needs.e2e-gke.result == 'success' ||
        needs.e2e-gke.result == 'failure'
      ) ||
      (
        needs.e2e-openshift.result == 'success' ||
        needs.e2e-openshift.result == 'failure'
      ))
    runs-on: ubuntu-24.04
    steps:
      - name: Create a directory for the artifacts
        run: mkdir test-artifacts

      - name: Download all artifacts to the directory
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          path: test-artifacts
          pattern: testartifacts-*

      - name: Flatten all artifacts onto directory
        # The download-artifact action, since we did not give it a name,
        # downloads all artifacts and creates a new folder for each.
        # In this step we bring all the JSONs to a single folder
        run: |
          mkdir test-artifacts/data
          mv test-artifacts/*/*.json test-artifacts/data || true

      - name: Display the structure of the artifact folder
        run: ls -R test-artifacts/data

      - name: Compute the E2E test summary
        id: generate-summary
        uses: cloudnative-pg/ciclops@fc1c72c2eb99e10aa437f14cd8563686b4b6e3b1 # v1.3.2
        with:
          artifact_directory: test-artifacts/data

      - name: If there is an overflow summary, archive it
        if: steps.generate-summary.outputs.Overflow
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: ${{ steps.generate-summary.outputs.Overflow }}
          path: ${{ steps.generate-summary.outputs.Overflow }}
          retention-days: 7

      - name: Send the Ciclops view over Slack
        # Send the Ciclops thermometer on every scheduled run on `main`.
        # or when there are systematic failures in release branches
        uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # v2
        if: |
          github.repository_owner == env.REPOSITORY_OWNER &&
          (
            github.event_name == 'schedule' ||
            (
              steps.generate-summary.outputs.alerts &&
              startsWith(needs.evaluate_options.outputs.git_ref, 'refs/heads/release-')
            )
          )
        env:
          # SLACK_COLOR is where we distinguish a run with/without alerts. It's where the
          # action has hooks for conditionality in the message body (yeah, weird)
          SLACK_COLOR: ${{ steps.generate-summary.outputs.alerts && 'failure' || 'success' }}
          SLACK_ICON: https://avatars.githubusercontent.com/u/85171364?size=48
          SLACK_USERNAME: ${{ env.SLACK_USERNAME }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_TITLE: CICLOPS view for ${{ github.repository }}
          SLACK_MESSAGE_ON_SUCCESS: |
            ${{ steps.generate-summary.outputs.thermometer }}
          SLACK_MESSAGE_ON_FAILURE: |
            ${{ steps.generate-summary.outputs.thermometer }}
            :warning: *Systematic failures!*
            ${{ steps.generate-summary.outputs.alerts }}
          SLACK_FOOTER: |
            <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|*See full CI run*>

      - name: Delete the downloaded files
        run: rm -rf test-artifacts

  # Adds the 'ok-to-merge' label to workflows that have run successfully and
  # have adequate test and matrix coverage.
  # This label is a prerequisite to be able to merge a PR.
  # Also see to 'require-labels.yml'
  ok-to-merge:
    name: Label the PR as "ok to merge :ok_hand:"
    needs:
      - evaluate_options
      - e2e-kind
    if: |
      always() &&
      needs.e2e-kind.result == 'success' &&
      github.event_name == 'issue_comment' &&
      needs.evaluate_options.outputs.test_level == '4'
    runs-on: ubuntu-24.04
    steps:
      - name: Check preconditions
        id: get_pr_number_and_labels
        env:
          GITHUB_TOKEN: ${{ secrets.REPO_GHA_PAT }}
          ISSUE_NUMBER: ${{ github.event.issue.number }}
        run: |
          ok_label=$(gh pr view "${ISSUE_NUMBER}" --json labels -q ".labels.[].name" 2>/dev/null | grep "ok to merge :ok_hand:" || :)
          echo "OK_LABEL=${ok_label}" >> $GITHUB_ENV

      - name: Label the PR as "ok to merge :ok_hand:"
        if: |
          env.OK_LABEL == ''
        uses: actions-ecosystem/action-add-labels@18f1af5e3544586314bbe15c0273249c770b2daf # v1.1.3
        with:
          github_token: ${{ secrets.REPO_GHA_PAT }}
          number: ${{ github.event.issue.number }}
          labels: "ok to merge :ok_hand:"

  # Remove the "ok to merge :ok_hand:" label if the E2E tests or previous steps failed
  unlabel-ok-to-merge:
    name: Remove the "ok to merge :ok_hand:" label from the PR
    needs:
      - evaluate_options
      - e2e-kind
    if: |
      always() &&
      needs.e2e-kind.result == 'failure' &&
      github.event_name == 'issue_comment'
    runs-on: ubuntu-24.04
    steps:
      - name: Check preconditions
        id: get_pr_number_and_labels
        env:
          GITHUB_TOKEN: ${{ secrets.REPO_GHA_PAT }}
          ISSUE_NUMBER: ${{ github.event.issue.number }}
        run: |
          ok_label=$(gh pr view "${ISSUE_NUMBER}" --json labels -q ".labels.[].name" 2>/dev/null | grep "ok to merge :ok_hand:" || :)
          echo "OK_LABEL=${ok_label}" >> $GITHUB_ENV

      - name: Remove "ok to merge :ok_hand:" label from PR
        if: |
          env.OK_LABEL != ''
        uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1.3.0
        with:
          github_token: ${{ secrets.REPO_GHA_PAT }}
          number: ${{ github.event.issue.number }}
          labels: "ok to merge :ok_hand:"