release_branches.yml

name: ReleaseBranchCI

env:
  # Force the stdout and stderr streams to be unbuffered
  PYTHONUNBUFFERED: 1
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  REGRESSION_RESULTS_URL: ${{github.event.number}}/${GITHUB_SHA}/testflows
  REGRESSION_COMMON_COMMIT: 1108c52f249af64885c255f39192ba3cc4c145ab
  REGRESSION_PARQUET_COMMIT: 63a15b5dfc55badefcf4b869296e3ec99ca08141
  REGRESSION_KEY_VALUE_COMMIT: e072060fba19d3f81a96f4c5cbe9c5d0b1dcfa9d


on: # yamllint disable-line rule:truthy
  pull_request:
    types:
      - synchronize
      - reopened
      - opened
    branches:
      # Anything/22.8 (e.g customizations/22.8)
      - '**/22.8*'
  release:
    types:
      - published
      - prereleased
  push:
    branches:
      - 'releases/22.8**'

jobs:
  # DockerHubPushAarch64:
  #   runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none-aarch64]
  #   steps:
  #     - name: Check out repository code
  #       uses: ClickHouse/checkout@v1
  #     - name: Images check
  #       run: |
  #         cd "$GITHUB_WORKSPACE/tests/ci"
  #         python3 docker_images_check.py --suffix aarch64
  #     - name: Upload images files to artifacts
  #       uses: actions/upload-artifact@v2
  #       with:
  #         name: changed_images_aarch64
  #         path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
  # Former DockerHubPushAmd64
  DockerHubPush:
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    steps:
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
      - name: Images check
        run: |
          cd "$GITHUB_WORKSPACE/tests/ci"
          python3 docker_images_check.py --suffix amd64
      # TODO(vnemkov): remove this step if you uncomment DockerHubPushAarch64 and DockerHubPush below.
      # The rest of the pipeline expects changed_images.json, which was generated by previous version of DockerHubPush.
      - name: Rename artifact
        run: |
          mv ${{ runner.temp }}/docker_images_check/changed_images_amd64.json ${{ runner.temp }}/docker_images_check/changed_images.json
      - name: Upload images files to artifacts
        uses: actions/upload-artifact@v3
        with:
          name: changed_images
          path: ${{ runner.temp }}/docker_images_check/changed_images.json
  # DockerHubPush:
  #   needs: [DockerHubPushAmd64, DockerHubPushAarch64]
  #   runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
  #   steps:
  #     - name: Check out repository code
  #       uses: ClickHouse/checkout@v1
  #     - name: Download changed aarch64 images
  #       uses: actions/download-artifact@v2
  #       with:
  #         name: changed_images_aarch64
  #         path: ${{ runner.temp }}
  #     - name: Download changed amd64 images
  #       uses: actions/download-artifact@v2
  #       with:
  #         name: changed_images_amd64
  #         path: ${{ runner.temp }}
  #     - name: Images check
  #       run: |
  #         cd "$GITHUB_WORKSPACE/tests/ci"
  #         python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64
  #     - name: Upload images files to artifacts
  #       uses: actions/upload-artifact@v2
  #       with:
  #         name: changed_images
  #         path: ${{ runner.temp }}/changed_images.json
  CompatibilityCheck:
    needs: [BuilderDebRelease]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    steps:
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          TEMP_PATH=${{runner.temp}}/compatibility_check
          REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
          REPORTS_PATH=${{runner.temp}}/reports_dir
          EOF
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: CompatibilityCheck
        run: |
          sudo rm -fr "$TEMP_PATH"
          mkdir -p "$TEMP_PATH"
          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py
      - name: Cleanup
        if: always()
        run: |
          # shellcheck disable=SC2046
          docker kill $(docker ps -q) ||:
          # shellcheck disable=SC2046
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr "$TEMP_PATH"
#########################################################################################
#################################### ORDINARY BUILDS ####################################
#########################################################################################
  BuilderDebRelease:
    needs: [DockerHubPush]
    runs-on: [self-hosted, altinity-type-ccx53, altinity-on-demand, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    steps:
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          TEMP_PATH=${{runner.temp}}/build_check
          IMAGES_PATH=${{runner.temp}}/images_path
          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
          CACHES_PATH=${{runner.temp}}/../ccaches
          BUILD_NAME=package_release
          CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable
          EOF
      - name: Download changed images
        uses: actions/download-artifact@v3
        with:
          name: changed_images
          path: ${{ env.IMAGES_PATH }}
      - name: Trust My Directory
        run: git config --global --add safe.directory * # https://stackoverflow.com/a/71940133
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
          submodules: true
          fetch-depth: 0 # otherwise we will have no info about contributors
      - name: Build
        run: |
          sudo rm -fr "$TEMP_PATH"
          mkdir -p "$TEMP_PATH/build_check/package_release"
          cd .. && tar czf $TEMP_PATH/build_source.src.tar.gz ClickHouse/
          cd $TEMP_PATH && tar xvzf $TEMP_PATH/build_source.src.tar.gz
          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
      - name: Upload build URLs to artifacts
        if: ${{ success() || failure() }}
        uses: actions/upload-artifact@v3
        with:
          name: ${{ env.BUILD_URLS }}
          path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
      - name: Cleanup
        if: always()
        run: |
          # shellcheck disable=SC2046
          docker kill $(docker ps -q) ||:
          # shellcheck disable=SC2046
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
  # BuilderDebAarch64:
  #   needs: [DockerHubPush]
  #   runs-on: [self-hosted, builder]
  #   steps:
  #     - name: Set envs
  #       run: |
  #         cat >> "$GITHUB_ENV" << 'EOF'
  #         TEMP_PATH=${{runner.temp}}/build_check
  #         IMAGES_PATH=${{runner.temp}}/images_path
  #         REPO_COPY=${{runner.temp}}/build_check/ClickHouse
  #         CACHES_PATH=${{runner.temp}}/../ccaches
  #         BUILD_NAME=package_aarch64
  #         EOF
  #     - name: Download changed images
  #       uses: actions/download-artifact@v2
  #       with:
  #         name: changed_images
  #         path: ${{ runner.temp }}/images_path
  #     - name: Check out repository code
  #       uses: ClickHouse/checkout@v1
  #       with:
  #         fetch-depth: 0 # otherwise we will have no info about contributors
  #     - name: Build
  #       run: |
  #         git -C "$GITHUB_WORKSPACE" submodule sync
  #         git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --recursive --init --jobs=10
  #         sudo rm -fr "$TEMP_PATH"
  #         mkdir -p "$TEMP_PATH"
  #         cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
  #         cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
  #     - name: Upload build URLs to artifacts
  #       uses: actions/upload-artifact@v2
  #       with:
  #         name: ${{ env.BUILD_URLS }}
  #         path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
  #     - name: Cleanup
  #       if: always()
  #       run: |
  #         # shellcheck disable=SC2046
  #         docker kill $(docker ps -q) ||:
  #         # shellcheck disable=SC2046
  #         docker rm -f $(docker ps -a -q) ||:
  #         sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
############################################################################################
##################################### Docker images  #######################################
############################################################################################
  DockerServerImages:
    needs:
      - BuilderDebRelease
      # - BuilderDebAarch64
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    steps:
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
          fetch-depth: 0  # It MUST BE THE SAME for all dependencies and the job itself
      - name: Check docker altinity/clickhouse-server building
        run: |
          cd "$GITHUB_WORKSPACE/tests/ci"
          python3 docker_server.py --release-type head --no-push \
            --image-repo altinity/clickhouse-server --image-path docker/server
          python3 docker_server.py --release-type head --no-push \
            --image-repo altinity/clickhouse-keeper --image-path docker/keeper
      - name: Cleanup
        if: always()
        run: |
          # shellcheck disable=SC2046
          docker kill $(docker ps -q) ||:
          # shellcheck disable=SC2046
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr "$TEMP_PATH"
############################################################################################
##################################### BUILD REPORTER #######################################
############################################################################################
  BuilderReport:
    needs:
      - BuilderDebRelease
      # - BuilderDebAarch64
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    if: ${{ success() || failure() }}
    steps:
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          CHECK_NAME=ClickHouse build check
          REPORTS_PATH=${{runner.temp}}/reports_dir
          REPORTS_PATH=${{runner.temp}}/reports_dir
          TEMP_PATH=${{runner.temp}}/report_check
          NEEDS_DATA_PATH=${{runner.temp}}/needs.json
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
      - name: Report Builder
        run: |
          sudo rm -fr "$TEMP_PATH"
          mkdir -p "$TEMP_PATH"
          cat > "$NEEDS_DATA_PATH" << 'EOF'
          ${{ toJSON(needs) }}
          EOF
          cd "$GITHUB_WORKSPACE/tests/ci"
          python3 build_report_check.py "$CHECK_NAME"
      - name: Cleanup
        if: always()
        run: |
          # shellcheck disable=SC2046
          docker kill $(docker ps -q) ||:
          # shellcheck disable=SC2046
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr "$TEMP_PATH"
  # BuilderSpecialReport:
  #   needs:
  #     # - BuilderBinDarwin
  #     - BuilderBinDarwinAarch64
  #   runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
  #   if: ${{ success() || failure() }}
  #   steps:
  #     - name: Set envs
  #       run: |
  #         cat >> "$GITHUB_ENV" << 'EOF'
  #         TEMP_PATH=${{runner.temp}}/report_check
  #         REPORTS_PATH=${{runner.temp}}/reports_dir
  #         CHECK_NAME=ClickHouse special build check
  #         NEEDS_DATA_PATH=${{runner.temp}}/needs.json
  #         EOF
  #     - name: Download json reports
  #       uses: actions/download-artifact@v3
  #       with:
  #         path: ${{ env.REPORTS_PATH }}
  #     - name: Check out repository code
  #       uses: ClickHouse/checkout@v1
  #       with:
  #         clear-repository: true
  #     - name: Report Builder
  #       run: |
  #         sudo rm -fr "$TEMP_PATH"
  #         mkdir -p "$TEMP_PATH"
  #         cat > "$NEEDS_DATA_PATH" << 'EOF'
  #         ${{ toJSON(needs) }}
  #         EOF
  #         cd "$GITHUB_WORKSPACE/tests/ci"
  #         python3 build_report_check.py "$CHECK_NAME"
  #     - name: Cleanup
  #       if: always()
  #       run: |
  #         # shellcheck disable=SC2046
  #         docker kill $(docker ps -q) ||:
  #         # shellcheck disable=SC2046
  #         docker rm -f $(docker ps -a -q) ||:
  #         sudo rm -fr "$TEMP_PATH"
  MarkReleaseReady:
    needs:
  #    - BuilderBinDarwin
  #    - BuilderBinDarwinAarch64
      - BuilderDebRelease
  #    - BuilderDebAarch64
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    steps:
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
      - name: Mark Commit Release Ready
        run: |
          cd "$GITHUB_WORKSPACE/tests/ci"
          python3 mark_release_ready.py
##############################################################################################
########################### FUNCTIONAl STATELESS TESTS #######################################
##############################################################################################
  FunctionalStatelessTestRelease:
    needs: [BuilderDebRelease]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-in-ash,altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    steps:
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          TEMP_PATH=${{runner.temp}}/stateless_debug
          REPORTS_PATH=${{runner.temp}}/reports_dir
          CHECK_NAME=Stateless tests (release)
          REPO_COPY=${{runner.temp}}/stateless_debug/ClickHouse
          KILL_TIMEOUT=10800
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
      - name: Functional test
        run: |
          sudo rm -fr "$TEMP_PATH"
          mkdir -p "$TEMP_PATH"
          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
          cd "$REPO_COPY/tests/ci"
          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
      - name: Cleanup
        if: always()
        run: |
          # shellcheck disable=SC2046
          docker kill $(docker ps -q) ||:
          # shellcheck disable=SC2046
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr "$TEMP_PATH"
  # FunctionalStatelessTestAarch64:
  #   needs: [BuilderDebAarch64]
  #   runs-on: [self-hosted, func-tester-aarch64]
  #   steps:
  #     - name: Set envs
  #       run: |
  #         cat >> "$GITHUB_ENV" << 'EOF'
  #         TEMP_PATH=${{runner.temp}}/stateless_release
  #         REPORTS_PATH=${{runner.temp}}/reports_dir
  #         CHECK_NAME=Stateless tests (aarch64)
  #         REPO_COPY=${{runner.temp}}/stateless_release/ClickHouse
  #         KILL_TIMEOUT=10800
  #         EOF
  #     - name: Download json reports
  #       uses: actions/download-artifact@v3
  #       with:
  #         path: ${{ env.REPORTS_PATH }}
  #     - name: Check out repository code
  #       uses: ClickHouse/checkout@v1
  #       with:
  #         clear-repository: true
  #     - name: Functional test
  #       run: |
  #         sudo rm -fr "$TEMP_PATH"
  #         mkdir -p "$TEMP_PATH"
  #         cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
  #         cd "$REPO_COPY/tests/ci"
  #         python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
  #     - name: Cleanup
  #       if: always()
  #       run: |
  #         # shellcheck disable=SC2046
  #         docker kill $(docker ps -q) ||:
  #         # shellcheck disable=SC2046
  #         docker rm -f $(docker ps -a -q) ||:
  #         sudo rm -fr "$TEMP_PATH"
##############################################################################################
############################ FUNCTIONAl STATEFUL TESTS #######################################
##############################################################################################
  FunctionalStatefulTestRelease:
    needs: [BuilderDebRelease]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-in-ash,altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    steps:
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          TEMP_PATH=${{runner.temp}}/stateful_debug
          REPORTS_PATH=${{runner.temp}}/reports_dir
          CHECK_NAME=Stateful tests (release)
          REPO_COPY=${{runner.temp}}/stateful_debug/ClickHouse
          KILL_TIMEOUT=3600
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
      - name: Functional test
        run: |
          sudo rm -fr "$TEMP_PATH"
          mkdir -p "$TEMP_PATH"
          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
          cd "$REPO_COPY/tests/ci"
          python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
      - name: Cleanup
        if: always()
        run: |
          # shellcheck disable=SC2046
          docker kill $(docker ps -q) ||:
          # shellcheck disable=SC2046
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr "$TEMP_PATH"
  # FunctionalStatefulTestAarch64:
  #   needs: [BuilderDebAarch64]
  #   runs-on: [self-hosted, func-tester-aarch64]
  #   steps:
  #     - name: Set envs
  #       run: |
  #         cat >> "$GITHUB_ENV" << 'EOF'
  #         TEMP_PATH=${{runner.temp}}/stateful_release
  #         REPORTS_PATH=${{runner.temp}}/reports_dir
  #         CHECK_NAME=Stateful tests (aarch64)
  #         REPO_COPY=${{runner.temp}}/stateful_release/ClickHouse
  #         KILL_TIMEOUT=3600
  #         EOF
  #     - name: Download json reports
  #       uses: actions/download-artifact@v3
  #       with:
  #         path: ${{ env.REPORTS_PATH }}
  #     - name: Check out repository code
  #       uses: ClickHouse/checkout@v1
  #       with:
  #         clear-repository: true
  #     - name: Functional test
  #       run: |
  #         sudo rm -fr "$TEMP_PATH"
  #         mkdir -p "$TEMP_PATH"
  #         cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
  #         cd "$REPO_COPY/tests/ci"
  #         python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
  #     - name: Cleanup
  #       if: always()
  #       run: |
  #         # shellcheck disable=SC2046
  #         docker kill $(docker ps -q) ||:
  #         # shellcheck disable=SC2046
  #         docker rm -f $(docker ps -a -q) ||:
  #         sudo rm -fr "$TEMP_PATH"
#############################################################################################
############################# INTEGRATION TESTS #############################################
#############################################################################################
  IntegrationTestsRelease0:
    needs: [BuilderDebRelease]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    steps:
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          TEMP_PATH=${{runner.temp}}/integration_tests_release
          REPORTS_PATH=${{runner.temp}}/reports_dir
          CHECK_NAME=Integration tests (release)
          REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
          RUN_BY_HASH_NUM=0
          RUN_BY_HASH_TOTAL=2
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
      - name: Integration test
        run: |
          sudo rm -fr "$TEMP_PATH"
          mkdir -p "$TEMP_PATH"
          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
          cd "$REPO_COPY/tests/ci"
          python3 integration_test_check.py "$CHECK_NAME"
      - name: Cleanup
        if: always()
        run: |
          # shellcheck disable=SC2046
          docker kill $(docker ps -q) ||:
          # shellcheck disable=SC2046
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr "$TEMP_PATH"
  IntegrationTestsRelease1:
    needs: [BuilderDebRelease]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    steps:
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          TEMP_PATH=${{runner.temp}}/integration_tests_release
          REPORTS_PATH=${{runner.temp}}/reports_dir
          CHECK_NAME=Integration tests (release)
          REPO_COPY=${{runner.temp}}/integration_tests_release/ClickHouse
          RUN_BY_HASH_NUM=1
          RUN_BY_HASH_TOTAL=2
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
      - name: Integration test
        run: |
          sudo rm -fr "$TEMP_PATH"
          mkdir -p "$TEMP_PATH"
          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
          cd "$REPO_COPY/tests/ci"
          python3 integration_test_check.py "$CHECK_NAME"
      - name: Cleanup
        if: always()
        run: |
          # shellcheck disable=SC2046
          docker kill $(docker ps -q) ||:
          # shellcheck disable=SC2046
          docker rm -f $(docker ps -a -q) ||:
          sudo rm -fr "$TEMP_PATH"
#############################################################################################
##################################### REGRESSION TESTS ######################################
#############################################################################################
  regression_start:
    ## Not depending on the tests above since they can fail at any given moment.
    needs: [BuilderDebRelease]
    runs-on: ubuntu-latest
    steps:
      - run: true

  regression_common:
    strategy:
      fail-fast: false
      matrix:
        SUITE: [aes_encryption, aggregate_functions, atomic_insert, base_58, clickhouse_keeper, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, kafka, kerberos, lightweight_delete, data_types, part_moves_between_shards, rbac, selects, session_timezone, ssl_server, tiered_storage, window_functions]
    needs: [regression_start]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
    steps:
      - name: Checkout regression repo
        uses: actions/checkout@v3
        with:
          repository: Altinity/clickhouse-regression
          ref: ${{ env.REGRESSION_COMMON_COMMIT }}
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          REPORTS_PATH=${{runner.temp}}/reports_dir
          SUITE=${{ matrix.SUITE }}
          artifacts=builds
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Setup
        run: .github/setup.sh
      - name: Get deb url
        run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
      - name: Run ${{ env.SUITE }} suite
        run: python3
              -u ${{ env.SUITE }}/regression.py
              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
              --test-to-end
              --local
              --collect-service-logs
              --output classic
              --parallel 1
              --attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="${GITHUB_ACTOR}" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="${GITHUB_RUN_ID}" job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)"
              --log raw.log
      - name: Create and upload logs
        if: always()
        run: .github/create_and_upload_logs.sh 1
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: ${{ env.SUITE }}-artifacts
          path: |
            ./report.html
            ./*.log.txt
            ./*.log
            ./*.html
            ./*/_instances/*.log
            ./*/_instances/*/logs/*.log
            ./*/*/_instances/*/logs/*.log
            ./*/*/_instances/*.log

  benchmark:
    strategy:
      fail-fast: false
      matrix:
        STORAGE: [minio, aws_s3, gcs]
    needs: [regression_start]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
    steps:
      - name: Checkout regression repo
        uses: actions/checkout@v3
        with:
          repository: Altinity/clickhouse-regression
          ref: ${{ env.REGRESSION_COMMON_COMMIT }}
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          REPORTS_PATH=${{runner.temp}}/reports_dir
          SUITE=ontime_benchmark
          STORAGE=/${{ matrix.STORAGE }}
          artifacts=builds
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Setup
        run: .github/setup.sh
      - name: Get deb url
        run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
      - name: Run ${{ env.SUITE }} suite
        run: python3
              -u ${{ env.SUITE }}/benchmark.py
              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
              --storage ${{ matrix.STORAGE }}
              --gcs-uri ${{ secrets.REGRESSION_GCS_URI }}
              --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }}
              --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }}
              --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }}
              --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }}
              --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }}
              --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }}
              --test-to-end
              --local
              --collect-service-logs
              --output classic
              --parallel 1
              --attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="${GITHUB_ACTOR}" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="${GITHUB_RUN_ID}" job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)"
              --log raw.log
      - name: Create and upload logs
        if: always()
        run: .github/create_and_upload_logs.sh 1
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: ${{ env.SUITE }}-minio-artifacts
          path: |
            ./report.html
            ./*.log.txt
            ./*.log
            ./*.html
            ./*/_instances/*.log
            ./*/_instances/*/logs/*.log
            ./*/*/_instances/*/logs/*.log
            ./*/*/_instances/*.log

  clickhouse_keeper_ssl:
    needs: [regression_start]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
    steps:
      - name: Checkout regression repo
        uses: actions/checkout@v3
        with:
          repository: Altinity/clickhouse-regression
          ref: ${{ env.REGRESSION_COMMON_COMMIT }}
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          REPORTS_PATH=${{runner.temp}}/reports_dir
          SUITE=clickhouse_keeper
          STORAGE=/ssl
          artifacts=builds
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Setup
        run: .github/setup.sh
      - name: Get deb url
        run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
      - name: Run ${{ env.SUITE }} suite
        run: python3
              -u ${{ env.SUITE }}/regression.py
              --ssl
              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
              --test-to-end
              --local
              --collect-service-logs
              --output classic
              --parallel 1
              --attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="${GITHUB_ACTOR}" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="${GITHUB_RUN_ID}" job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)"
              --log raw.log
      - name: Create and upload logs
        if: always()
        run: .github/create_and_upload_logs.sh 1
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: ${{ env.SUITE }}-artifacts
          path: |
            ./report.html
            ./*.log.txt
            ./*.log
            ./*.html
            ./*/_instances/*.log
            ./*/_instances/*/logs/*.log
            ./*/*/_instances/*/logs/*.log
            ./*/*/_instances/*.log
  key_value:
    needs: [regression_start]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
    steps:
      - name: Checkout regression repo
        uses: actions/checkout@v3
        with:
          repository: Altinity/clickhouse-regression
          ref: ${{ env.REGRESSION_KEY_VALUE_COMMIT }}
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          REPORTS_PATH=${{runner.temp}}/reports_dir
          SUITE=key_value
          artifacts=public
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Setup
        run: .github/setup.sh
      - name: Get deb url
        run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
      - name: Run ${{ env.SUITE }} suite
        run: python3
              -u ${{ env.SUITE }}/regression.py
              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
              --test-to-end
              --local
              --collect-service-logs
              --output classic
              --parallel 1
              --attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="${GITHUB_ACTOR}" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="${GITHUB_RUN_ID}" job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)"
              --log raw.log
      - name: Create and upload logs
        if: always()
        run: .github/create_and_upload_logs.sh 1
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: ${{ env.SUITE }}-artifacts
          path: |
            ./report.html
            ./*.log.txt
            ./*.log
            ./*.html
            ./*/_instances/*.log
            ./*/_instances/*/logs/*.log
            ./*/*/_instances/*/logs/*.log
            ./*/*/_instances/*.log

  ldap:
    strategy:
      fail-fast: false
      matrix:
        SUITE: [authentication, external_user_directory, role_mapping]
    needs: [regression_start]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
    steps:
      - name: Checkout regression repo
        uses: actions/checkout@v3
        with:
          repository: Altinity/clickhouse-regression
          ref: ${{ env.REGRESSION_COMMON_COMMIT }}
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          REPORTS_PATH=${{runner.temp}}/reports_dir
          SUITE=ldap/${{ matrix.SUITE }}
          artifacts=builds
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Setup
        run: .github/setup.sh
      - name: Get deb url
        run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
      - name: Run ${{ env.SUITE }} suite
        run: python3
              -u ${{ env.SUITE }}/regression.py
              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
              --test-to-end
              --local
              --collect-service-logs
              --output classic
              --parallel 1
              --attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="${GITHUB_ACTOR}" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="${GITHUB_RUN_ID}" job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)"
              --log raw.log
      - name: Create and upload logs
        if: always()
        run: .github/create_and_upload_logs.sh 1
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: ldap-authentication-artifacts
          path: |
            ./report.html
            ./*.log.txt
            ./*.log
            ./*.html
            ./*/_instances/*.log
            ./*/_instances/*/logs/*.log
            ./*/*/_instances/*/logs/*.log
            ./*/*/_instances/*.log

  parquet:
    needs: [regression_start]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
    steps:
      - name: Checkout regression repo
        uses: actions/checkout@v3
        with:
          repository: Altinity/clickhouse-regression
          ref: ${{ env.REGRESSION_PARQUET_COMMIT }}
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          REPORTS_PATH=${{runner.temp}}/reports_dir
          SUITE=parquet
          STORAGE=/no_s3
          artifacts=public
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Setup
        run: .github/setup.sh
      - name: Get deb url
        run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
      - name: Run ${{ env.SUITE }} suite
        run: python3
              -u ${{ env.SUITE }}/regression.py
              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
              --test-to-end
              --local
              --collect-service-logs
              --output classic
              --parallel 1
              --attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="${GITHUB_ACTOR}" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="${GITHUB_RUN_ID}" job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)"
              --log raw.log
      - name: Create and upload logs
        if: always()
        run: .github/create_and_upload_logs.sh 1
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: ${{ env.SUITE }}-artifacts
          path: |
            ./report.html
            ./*.log.txt
            ./*.log
            ./*.html
            ./*/_instances/*.log
            ./*/_instances/*/logs/*.log
            ./*/*/_instances/*/logs/*.log
            ./*/*/_instances/*.log

  parquet_minio:
    needs: [regression_start]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
    steps:
      - name: Checkout regression repo
        uses: actions/checkout@v3
        with:
          repository: Altinity/clickhouse-regression
          ref: ${{ env.REGRESSION_PARQUET_COMMIT }}
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          REPORTS_PATH=${{runner.temp}}/reports_dir
          SUITE=parquet
          STORAGE=/minio
          artifacts=public
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Setup
        run: .github/setup.sh
      - name: Get deb url
        run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
      - name: Run ${{ env.SUITE }} suite
        run: python3
              -u ${{ env.SUITE }}/regression.py
              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
              --test-to-end
              --local
              --collect-service-logs
              --output classic
              --parallel 1
              --attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="${GITHUB_ACTOR}" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="${GITHUB_RUN_ID}" job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)"
              --log raw.log
              --storage minio
      - name: Create and upload logs
        if: always()
        run: .github/create_and_upload_logs.sh 1
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: ${{ env.SUITE }}-minio-artifacts
          path: |
            ./report.html
            ./*.log.txt
            ./*.log
            ./*.html
            ./*/_instances/*.log
            ./*/_instances/*/logs/*.log
            ./*/*/_instances/*/logs/*.log
            ./*/*/_instances/*.log

  parquet_aws:
    needs: [regression_start]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
    steps:
      - name: Checkout regression repo
        uses: actions/checkout@v3
        with:
          repository: Altinity/clickhouse-regression
          ref: ${{ env.REGRESSION_PARQUET_COMMIT }}
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          REPORTS_PATH=${{runner.temp}}/reports_dir
          SUITE=parquet
          STORAGE=/aws_s3
          artifacts=public
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Setup
        run: .github/setup.sh
      - name: Get deb url
        run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
      - name: Run ${{ env.SUITE }} suite
        run: python3
              -u ${{ env.SUITE }}/regression.py
              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
              --test-to-end
              --local
              --collect-service-logs
              --output classic
              --parallel 1
              --attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="${GITHUB_ACTOR}" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="${GITHUB_RUN_ID}" job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)"
              --log raw.log
              --storage aws_s3
              --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }}
              --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }}
              --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }}
              --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }}
      - name: Create and upload logs
        if: always()
        run: .github/create_and_upload_logs.sh 1
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: ${{ env.SUITE }}-aws_s3-artifacts
          path: |
            ./report.html
            ./*.log.txt
            ./*.log
            ./*.html
            ./*/_instances/*.log
            ./*/_instances/*/logs/*.log
            ./*/*/_instances/*/logs/*.log
            ./*/*/_instances/*.log

  s3:
    strategy:
      fail-fast: false
      matrix:
        STORAGE: [minio, aws_s3, gcs]
    needs: [regression_start]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
    steps:
      - name: Checkout regression repo
        uses: actions/checkout@v3
        with:
          repository: Altinity/clickhouse-regression
          ref: ${{ env.REGRESSION_COMMON_COMMIT }}
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          REPORTS_PATH=${{runner.temp}}/reports_dir
          SUITE=s3
          STORAGE=/${{ matrix.STORAGE }}
          artifacts=builds
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Setup
        run: .github/setup.sh
      - name: Get deb url
        run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
      - name: Run ${{ env.SUITE }} suite
        run: python3
              -u ${{ env.SUITE }}/regression.py
              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
              --test-to-end
              --local
              --collect-service-logs
              --output classic
              --parallel 1
              --attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="${GITHUB_ACTOR}" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="${GITHUB_RUN_ID}" job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)"
              --log raw.log
              --storage ${{ matrix.STORAGE }}
              --gcs-uri ${{ secrets.REGRESSION_GCS_URI }}
              --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }}
              --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }}
              --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }}
              --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }}
              --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }}
              --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }}
      - name: Create and upload logs
        if: always()
        run: .github/create_and_upload_logs.sh 1
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-artifacts
          path: |
            ./report.html
            ./*.log.txt
            ./*.log
            ./*.html
            ./*/_instances/*.log
            ./*/_instances/*/logs/*.log
            ./*/*/_instances/*/logs/*.log
            ./*/*/_instances/*.log

  tiered_storage_s3:
    strategy:
      fail-fast: false
      matrix:
        STORAGE: [minio, s3amazon, s3gcs]
    needs: [regression_start]
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
    steps:
      - name: Checkout regression repo
        uses: actions/checkout@v3
        with:
          repository: Altinity/clickhouse-regression
          ref: ${{ env.REGRESSION_COMMON_COMMIT }}
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          REPORTS_PATH=${{runner.temp}}/reports_dir
          SUITE=tiered_storage
          STORAGE=/${{ matrix.STORAGE }}
          artifacts=builds
          EOF
      - name: Download json reports
        uses: actions/download-artifact@v3
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Setup
        run: .github/setup.sh
      - name: Get deb url
        run: python3 .github/get-deb-url.py --reports-path ${{ env.REPORTS_PATH }} --github-env $GITHUB_ENV
      - name: Run ${{ env.SUITE }} suite
        run: python3
              -u ${{ env.SUITE }}/regression.py
              --clickhouse-binary-path ${{ env.clickhouse_binary_path }}
              --test-to-end
              --local
              --collect-service-logs
              --output classic
              --parallel 1
              --attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" package="${{ env.clickhouse_binary_path }}" version="${{ env.version }}" user.name="${GITHUB_ACTOR}" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.id="${GITHUB_RUN_ID}" job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)"
              --log raw.log
              --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }}
              --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }}
              --aws-s3-uri https://s3.${{ secrets.REGRESSION_AWS_S3_REGION}}.amazonaws.com/${{ secrets.REGRESSION_AWS_S3_BUCKET }}/data/
              --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }}
              --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }}
              --gcs-uri ${{ secrets.REGRESSION_GCS_URI }}
              --with-${{ matrix.STORAGE }}
      - name: Create and upload logs
        if: always()
        run: .github/create_and_upload_logs.sh 1
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-artifacts
          path: |
            ./report.html
            ./*.log.txt
            ./*.log
            ./*.html
            ./*/_instances/*.log
            ./*/_instances/*/logs/*.log
            ./*/*/_instances/*/logs/*.log
            ./*/*/_instances/*.log

  SignRelease:
    needs: [BuilderDebRelease]
    runs-on: [ self-hosted ]
    steps:
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          TEMP_PATH=${{runner.temp}}/signed
          REPORTS_PATH=${{runner.temp}}/reports_dir
          EOF
      - name: Clear repository
        run: |
          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
      - name: Check out repository code
        uses: actions/checkout@v2
      - name: Download json reports
        uses: actions/download-artifact@v2
        with:
          path: ${{ env.REPORTS_PATH }}
      - name: Sign release
        env:
          GPG_BINARY_SIGNING_KEY: ${{ secrets.GPG_BINARY_SIGNING_KEY }}
          GPG_BINARY_SIGNING_PASSPHRASE: ${{ secrets.GPG_BINARY_SIGNING_PASSPHRASE }}
          REPORTS_PATH: ${{ env.REPORTS_PATH }}
        run: |
          cd "$GITHUB_WORKSPACE/tests/ci"
          python3 sign_release.py
      - name: Upload signed hashes
        uses: actions/upload-artifact@v2
        with:
          name: signed-hashes
          path: ${{ env.TEMP_PATH }}/*.gpg
      - name: Cleanup
        if: always()
        run: |
          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
          sudo rm -fr "$TEMP_PATH"
  ###########################################################################################
  ################################ FINISH CHECK #############################################
  ###########################################################################################
  FinishCheck:
    needs:
      - DockerHubPush
      - DockerServerImages
      - BuilderReport
      # - BuilderSpecialReport
      - MarkReleaseReady
      - FunctionalStatelessTestRelease
      # - FunctionalStatelessTestAarch64
      - FunctionalStatefulTestRelease
      # - FunctionalStatefulTestAarch64
      - IntegrationTestsRelease0
      - IntegrationTestsRelease1
      - CompatibilityCheck
      - SignRelease
      - regression_common
      - benchmark
      - clickhouse_keeper_ssl
      - key_value
      - ldap
      - parquet
      - parquet_minio
      - parquet_aws
      - s3
      - tiered_storage_s3
    runs-on: [self-hosted, altinity-on-demand, altinity-type-cpx51, altinity-image-x86-snapshot-22.04-amd, altinity-startup-snapshot, altinity-setup-none]
    steps:
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
      - name: Finish label
        run: |
          cd "$GITHUB_WORKSPACE/tests/ci"
          python3 finish_check.py