pulsar-ci.yaml

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: Pulsar CI
on:
  pull_request:
    branches:
      - master
      - branch-*
      - pulsar-*
  schedule:
    # scheduled job with JDK 21
    - cron: '0 12 * * *'
    # scheduled job with JDK 17
    # if cron expression is changed, make sure to update the expression in jdk_major_version step in preconditions job
    - cron: '0 6 * * *'
  workflow_dispatch:
    inputs:
      collect_coverage:
        description: 'Collect test coverage and upload to Codecov'
        required: true
        type: boolean
        default: true
      jdk_major_version:
        description: 'JDK major version to use for the build'
        required: true
        type: choice
        options:
          - '17'
          - '21'
          - '24'
          - '25'
        default: '21'
      trace_test_resource_cleanup:
        description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.'
        required: true
        type: choice
        options:
          - 'off'
          - 'on'
          - 'full'
        default: 'off'
      thread_leak_detector_wait_millis:
        description: 'Duration in ms to wait for threads to exit in thread leak detection between test classes. It is necessary to wait for threads to complete before they are determined to be leaked threads.'
        required: true
        type: number
        default: 10000
      netty_leak_detection:
        description: 'Controls Netty leak detection. When set to "report", Netty leak detection is enabled. When set to "fail_on_leak", Netty leak detection is enabled and a build job will fail if leaks are detected. When set to "off", Netty leak detection is disabled.'
        required: true
        type: choice
        options:
          - 'report'
          - 'fail_on_leak'
          - 'off'
        default: 'report'
      docker_image_type:
        description: 'Type of docker image to build. When set to "alpine", an Alpine based docker image will be built. When set to "wolfi", a Wolfi based docker image will be built.'
        required: true
        type: choice
        options:
          - 'alpine'
          - 'wolfi'
        default: 'alpine'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '' }}
  cancel-in-progress: true

env:
  MAVEN_OPTS: -Xss1500k -Xmx2048m -XX:+UnlockDiagnosticVMOptions -XX:+IgnoreUnrecognizedVMOptions -XX:GCLockerRetryAllocationCount=100 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
  # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
  # it's possible to rerun individual failed jobs when the build artifacts are available
  # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR
  ARTIFACT_RETENTION_DAYS: 3
  JDK_DISTRIBUTION: corretto

jobs:
  preconditions:
    name: Preconditions
    runs-on: ubuntu-24.04
    outputs:
      docs_only: ${{ steps.check_changes.outputs.docs_only }}
      changed_tests: ${{ steps.changes.outputs.tests_files }}
      need_owasp: ${{ steps.changes.outputs.need_owasp }}
      collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }}
      jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version }}
      java_non_tests: ${{ steps.changes.outputs.java_non_tests }}
      netty_leak_detection: ${{ steps.netty_leak_detection.outputs.netty_leak_detection }}
      java_test_image_base_matrix: ${{ steps.docker_images_matrix.outputs.java_test_image_base_matrix }}
      latest_version_image_base_matrix: ${{ steps.docker_images_matrix.outputs.latest_version_image_base_matrix }}
    steps:
      - name: Cancel scheduled jobs in forks by default
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'schedule' }}
        uses: actions/github-script@v6
        with:
          script: |
            await github.rest.actions.cancelWorkflowRun({owner: context.repo.owner, repo: context.repo.repo, run_id: context.runId});
            process.exit(1);

      - name: Select JDK major version
        id: jdk_major_version
        run: |
          # use JDK 17 for the scheduled build with cron expression '0 6 * * *'
          if [[ "${{ github.event_name == 'schedule' && github.event.schedule == '0 6 * * *' && 'true' || 'false' }}" == "true" ]]; then
            echo "jdk_major_version=17" >> $GITHUB_OUTPUT
            exit 0
          fi
          # use JDK 21 for build unless overridden with workflow_dispatch input
          echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '21'}}" >> $GITHUB_OUTPUT

      - name: checkout
        if: ${{ github.event_name == 'pull_request' }}
        uses: actions/checkout@v4

      - name: Detect changed files
        if: ${{ github.event_name == 'pull_request' }}
        id:   changes
        uses: apache/pulsar-test-infra/paths-filter@master
        with:
          filters: .github/changes-filter.yaml
          list-files: csv

      - name: Check changed files
        if: ${{ github.event_name == 'pull_request' }}
        id: check_changes
        run: |
          if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then
            echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
          else
            echo docs_only=false >> $GITHUB_OUTPUT
          fi

      - name: Choose docker images to build
        id: docker_images_matrix
        run: |
          # build JSON for java_test_image_base_matrix and latest_version_image_base_matrix
          # This JSON is used as input or the build matrixes in the docker image build jobs.
          # For workflow_dispatch, the docker_image_type input is used to determine the base image
          # For scheduled builds, the default base image is alpine
          # For pull requests, the default base image is alpine. If the changed files include Dockerfile.wolfi changes
          # the wolfi base image is built in addition to the alpine image.
          # create json for the default base image of the build
          main_type="alpine"
          if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then
            main_type="${{ github.event.inputs.docker_image_type }}"
          fi
          if [[ "$main_type" == "wolfi" ]]; then
            main_type_json='{"type": "'$main_type'", "save_artifact": true, "additional_build_args": "-Pdocker-wolfi"}'
          else
            main_type_json='{"type": "'$main_type'", "save_artifact": true}'
          fi
          # build JSON for java_test_image_base_matrix
          echo "java_test_image_base_matrix<<EOF" >> $GITHUB_OUTPUT
          echo "[" >> $GITHUB_OUTPUT
          echo "$main_type_json" >> $GITHUB_OUTPUT
          if [[ ${{ github.event_name == 'pull_request' && fromJSON(steps.changes.outputs.pulsar_dockerfile_wolfi_count) || 0 }} -gt 0 ]]; then
              echo ',{"type": "wolfi", "save_artifact": false, "additional_build_args": "-Pdocker-wolfi"}' >> $GITHUB_OUTPUT
          fi
          echo "]" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT
          # build JSON for latest_version_image_base_matrix
          echo "latest_version_image_base_matrix<<EOF" >> $GITHUB_OUTPUT
          echo "[" >> $GITHUB_OUTPUT
          echo "$main_type_json" >> $GITHUB_OUTPUT
          if [[ ${{ github.event_name == 'pull_request' && fromJSON(steps.changes.outputs.pulsar_all_dockerfile_wolfi_count) || 0 }} -gt 0 ]]; then
              echo ',{"type": "wolfi", "save_artifact": false, "additional_build_args": "-Pdocker-wolfi"}' >> $GITHUB_OUTPUT
          fi
          echo "]" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

      - name: Check if coverage should be collected
        id: check_coverage
        run: |
          echo "collect_coverage=${{
          (steps.check_changes.outputs.docs_only != 'true' && github.event_name != 'workflow_dispatch'
            && (github.base_ref == 'master' || github.ref_name == 'master'))
          || (github.event_name == 'workflow_dispatch' && github.event.inputs.collect_coverage == 'true')
          }}"  >> $GITHUB_OUTPUT

      - name: Set Netty leak detection mode
        id: netty_leak_detection
        run: |
          echo "netty_leak_detection=${{
          github.event_name == 'workflow_dispatch' && github.event.inputs.netty_leak_detection || 'report'
          }}" >> $GITHUB_OUTPUT

      - name: Check if the PR has been approved for testing
        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
        env:
          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          build/pulsar_ci_tool.sh check_ready_to_test

  build-and-license-check:
    needs: preconditions
    name: Build and License check
    env:
      JOB_NAME: Build and License check
      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Check source code license headers
        run: mvn -B -T 8 -ntp initialize apache-rat:check license:check

      - name: Check source code style
        run: mvn -B -T 8 -ntp initialize checkstyle:check

      - name: Build core-modules
        run: |
          mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true

      - name: Check binary licenses
        run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Save maven build results to Github artifact cache so that the results can be reused
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \
          tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \
              -I zstd -cf - .m2/repository/org/apache/pulsar
          cd $GITHUB_WORKSPACE
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \
          tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait


  unit-tests:
    name: CI - Unit - ${{ matrix.name }}
    env:
      JOB_NAME: CI - Unit - ${{ matrix.name }}
      COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }}
      TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
      THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }}
      THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
      NETTY_LEAK_DETECTION: "${{ needs.preconditions.outputs.netty_leak_detection }}"
      NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/target/netty-leak-dumps
    runs-on: ubuntu-24.04
    timeout-minutes: ${{ matrix.timeout || 60 }}
    needs: ['preconditions', 'build-and-license-check']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Other
            group: OTHER
            timeout: 75
          - name: Brokers - Broker Group 1
            group: BROKER_GROUP_1
          - name: Brokers - Broker Group 2
            group: BROKER_GROUP_2
          - name: Brokers - Broker Group 3
            group: BROKER_GROUP_3
          - name: Brokers - Broker Group 4
            group: BROKER_GROUP_4
          - name: Brokers - Broker Group 5
            group: BROKER_GROUP_5
          - name: Brokers - Client Api
            group: BROKER_CLIENT_API
          - name: Brokers - Client Impl
            group: BROKER_CLIENT_IMPL
          - name: Proxy
            group: PROXY
          - name: Pulsar IO
            group: PULSAR_IO
            timeout: 75
          - name: Pulsar IO - Elastic Search
            group: PULSAR_IO_ELASTIC
          - name: Pulsar IO - Kafka Connect Adaptor
            group: PULSAR_IO_KAFKA_CONNECT
          - name: Pulsar Client
            group: CLIENT
          - name: Pulsar Metadata
            group: METADATA
          - name: Protobuf v4
            group: PROTOBUFV4

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Clean Disk when tracing test resource cleanup
        if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
        uses: ./.github/actions/clean-disk

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Run setup commands
        run: |
          ${{ matrix.setup }}

      - name: Run unit test group '${{ matrix.group }}'
        run: |
          CHANGED_TESTS="${{ needs.preconditions.outputs.tests_files }}" ./build/run_unit_group.sh ${{ matrix.group }}

      - name: Upload coverage to build artifacts
        if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_unittest_coverage_files ${{ matrix.group }}

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'test-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Report detected thread leaks
        if: ${{ always() }}
        run: |
          if [ -d "$THREAD_LEAK_DETECTOR_DIR" ]; then
            cd "$THREAD_LEAK_DETECTOR_DIR"
            cat threadleak*.txt | awk '/^Summary:/ {print "::warning::" $0 "\n"; next} {print}'
          fi

      - name: Report detected Netty leaks
        if: ${{ always() && env.NETTY_LEAK_DETECTION != 'off' }}
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh report_netty_leaks

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v4
        if: ${{ !success() || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
        with:
          name: Unit-${{ matrix.group }}-surefire-reports
          path: surefire-reports
          retention-days: 7

      - name: Upload possible heap dump, core dump or crash files
        uses: actions/upload-artifact@v4
        if: ${{ always() }}
        with:
          name: Unit-${{ matrix.group }}-dumps
          path: |
            /tmp/*.hprof
            **/hs_err_*.log
            **/core.*
            ${{ env.NETTY_LEAK_DUMP_DIR }}/*
            ${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/*
            ${{ env.THREAD_LEAK_DETECTOR_DIR }}/*
          retention-days: 7
          if-no-files-found: ignore

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait


  unit-tests-upload-coverage:
    name: CI - Unit - Upload Coverage
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    needs: ['preconditions', 'unit-tests']
    env:
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Restore coverage files from build artifacts and create Jacoco reports
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_unittest_coverage_files
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
          cd $GITHUB_WORKSPACE/target
          zip -qr jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report || true

      - name: Upload Jacoco report files to build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: Jacoco-coverage-report-unittests
          path: target/jacoco_test_coverage_report_unittests.zip
          retention-days: 3

      - name: Upload to Codecov
        uses: ./.github/actions/upload-coverage
        with:
          flags: unittests
          token: ${{ secrets.CODECOV_TOKEN }}

      - name: Delete coverage files from build artifacts
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_unittest_coverage_files

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  pulsar-java-test-image:
    name: Build Pulsar java-test-image ${{ matrix.base.type }} docker image for ${{ matrix.platform }}
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    needs: ['preconditions', 'build-and-license-check']
    if: ${{ needs.preconditions.outputs.docs_only != 'true'}}
    strategy:
      fail-fast: false
      matrix:
        base: ${{ fromJSON(needs.preconditions.outputs.java_test_image_base_matrix) }}
        platform:
          - linux/amd64
          - linux/arm64
    env:
      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          platforms: arm64

      - name: Build java-test-image ${{ matrix.base.type }} docker image - ${{ matrix.platform }}
        run: |
          # build docker image
          DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
          -Ddocker.platforms=${{ matrix.platform }} ${{ matrix.base.additional_build_args }} \
          -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true \
          -Dspotbugs.skip=true  -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true

      - name: save docker image apachepulsar/java-test-image:latest to Github artifact cache
        if: ${{ matrix.base.save_artifact && matrix.platform == 'linux/amd64' }}
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  integration-tests:
    name: CI - Integration - ${{ matrix.name }}
    runs-on: ubuntu-24.04
    timeout-minutes: ${{ matrix.timeout || 60 }}
    needs: ['preconditions', 'pulsar-java-test-image']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      JOB_NAME: CI - Integration - ${{ matrix.name }}
      PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      NETTY_LEAK_DETECTION: "${{ needs.preconditions.outputs.netty_leak_detection }}"
      NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/target/netty-leak-dumps
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Backwards Compatibility
            group: BACKWARDS_COMPAT
            no_coverage: true

          - name: Cli
            group: CLI

          - name: Messaging
            group: MESSAGING

          - name: LoadBalance
            group: LOADBALANCE
            no_coverage: true

          - name: Shade on Java 17
            group: SHADE_RUN
            upload_name: SHADE_RUN_17
            runtime_jdk: 17
            setup: ./build/run_integration_group.sh SHADE_BUILD
            no_coverage: true

          - name: Shade on Java 21
            group: SHADE_RUN
            upload_name: SHADE_RUN_21
            runtime_jdk: 21
            setup: ./build/run_integration_group.sh SHADE_BUILD
            no_coverage: true

          - name: Shade on Java 24
            group: SHADE_RUN
            upload_name: SHADE_RUN_24
            runtime_jdk: 24
            setup: ./build/run_integration_group.sh SHADE_BUILD
            no_coverage: true

          - name: Standalone
            group: STANDALONE

          - name: Transaction
            group: TRANSACTION

          - name: Metrics
            group: METRICS

          - name: Upgrade
            group: UPGRADE

          - name: Kubernetes
            group: PULSAR_K8S

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image

      - name: Run setup commands
        if: ${{ matrix.setup }}
        run: |
          ${{ matrix.setup }}

      - name: Set up runtime JDK ${{ matrix.runtime_jdk }}
        uses: actions/setup-java@v4
        if: ${{ matrix.runtime_jdk }}
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ matrix.runtime_jdk }}

      - name: Run integration test group '${{ matrix.group }}'
        run: |
          if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then
            coverage_args="--coverage"
          fi
          ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args

      - name: Upload coverage to build artifacts
        if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }}
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_inttest_coverage_files ${{ matrix.group }}

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'test-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Report detected Netty leaks
        if: ${{ always() && env.NETTY_LEAK_DETECTION != 'off' }}
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh report_netty_leaks

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        with:
          name: Integration-${{ matrix.upload_name || matrix.group }}-surefire-reports
          path: surefire-reports
          retention-days: 7

      - name: Upload possible heap dump, core dump or crash files
        uses: actions/upload-artifact@v4
        if: ${{ always() }}
        with:
          name: Integration-${{ matrix.upload_name || matrix.group }}-dumps
          path: |
            /tmp/*.hprof
            **/hs_err_*.log
            **/core.*
            ${{ env.NETTY_LEAK_DUMP_DIR }}/*
          retention-days: 7
          if-no-files-found: ignore

      - name: Upload container logs
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        continue-on-error: true
        with:
          name: Integration-${{ matrix.upload_name || matrix.group }}-container-logs
          path: tests/integration/target/container-logs
          retention-days: 7

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  integration-tests-upload-coverage:
    name: CI - Integration - Upload Coverage
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    needs: ['preconditions', 'integration-tests']
    if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
    env:
      PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image

      - name: Restore coverage files from build artifacts and create Jacoco reports
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_inttest_coverage_files
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
          cd $GITHUB_WORKSPACE/target
          zip -qr jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true

      - name: Upload Jacoco report files to build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: Jacoco-coverage-report-inttests
          path: target/jacoco_test_coverage_report_inttests.zip
          retention-days: 3

      - name: Upload to Codecov
        uses: ./.github/actions/upload-coverage
        with:
          flags: inttests
          token: ${{ secrets.CODECOV_TOKEN }}

      - name: Delete coverage files from build artifacts
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_inttest_coverage_files

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  delete-integration-test-docker-image-artifact:
    name: "Delete integration test docker image artifact"
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    needs: [
      'preconditions',
      'integration-tests',
      'integration-tests-upload-coverage'
    ]
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Delete docker image from GitHub Actions Artifacts
        run: |
          gh-actions-artifact-client.js delete pulsar-java-test-image.zst

  pulsar-test-latest-version-image:
    name: Build Pulsar ${{ matrix.base.type }} docker image
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    needs: ['preconditions', 'build-and-license-check']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    strategy:
      fail-fast: false
      matrix:
        base: ${{ fromJSON(needs.preconditions.outputs.latest_version_image_base_matrix) }}
    env:
      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Clean Disk
        uses: ./.github/actions/clean-disk
        with:
          mode: full

      - name: Cache local Maven repository
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: Build latest-version-image docker ${{ matrix.base.type }} image
        run: |
          # build docker image
          # include building of Connectors, Offloaders and server distros
          DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
          -Pmain,docker -Dmaven.test.skip=true ${{ matrix.base.additional_build_args }} \
          -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true

      # check full build artifacts licenses
      - name: Check binary licenses
        run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz

      - name: Run Trivy container scan
        id: trivy_scan
        uses: aquasecurity/trivy-action@0.26.0
        if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }}
        continue-on-error: true
        with:
          image-ref: "apachepulsar/pulsar:latest"
          scanners: vuln
          severity: CRITICAL,HIGH,MEDIUM,LOW
          limit-severities-for-sarif: true
          format: 'sarif'
          output: 'trivy-results.sarif'

      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v3
        if: ${{ steps.trivy_scan.outcome == 'success' && github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }}
        continue-on-error: true
        with:
          sarif_file: 'trivy-results.sarif'

      - name: Clean up disk space
        if: ${{ matrix.base.save_artifact }}
        run: |
          # release disk space since saving docker image consumes local disk space
          #
          echo "::group::Available diskspace before cleaning"
          time df -BM / /mnt
          echo "::endgroup::"
          echo "::group::Clean build directory"
          # docker build changes some files to root ownership, fix this before deleting files
          sudo chown -R $USER:$GROUP .
          # clean build directories
          time git clean -fdx
          echo "::endgroup::"
          echo "::group::Available diskspace after cleaning build directory"
          time df -BM / /mnt
          echo "::endgroup::"
          echo "::group::Delete maven repository"
          # delete maven repository
          time rm -rf ~/.m2/repository
          echo "::endgroup::"
          echo "::group::Available diskspace after cleaning maven repository"
          time df -BM / /mnt
          echo "::endgroup::"

      - name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache
        if: ${{ matrix.base.save_artifact }}
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  system-tests:
    name: CI - System - ${{ matrix.name }}
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    needs: ['preconditions', 'pulsar-test-latest-version-image']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      JOB_NAME: CI - System - ${{ matrix.name }}
      PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      NETTY_LEAK_DETECTION: "${{ needs.preconditions.outputs.netty_leak_detection }}"
      NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/target/netty-leak-dumps
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Tiered FileSystem
            group: TIERED_FILESYSTEM

          - name: Tiered JCloud
            group: TIERED_JCLOUD

          - name: Function
            group: FUNCTION

          - name: Schema
            group: SCHEMA

          - name: Pulsar Connectors - Thread
            group: PULSAR_CONNECTORS_THREAD

          - name: Pulsar Connectors - Process
            group: PULSAR_CONNECTORS_PROCESS

          - name: Pulsar IO
            group: PULSAR_IO
            clean_disk: true

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Clean Disk when needed
        if: ${{ matrix.clean_disk }}
        uses: ./.github/actions/clean-disk

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image

      - name: Run setup commands
        if: ${{ matrix.setup }}
        run: |
          ${{ matrix.setup }}

      - name: Run system test group '${{ matrix.group }}'
        run: |
          if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then
            coverage_args="--coverage"
          fi
          ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args

      - name: Upload coverage to build artifacts
        if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }}
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_systest_coverage_files ${{ matrix.group }}

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'test-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Report detected Netty leaks
        if: ${{ always() && env.NETTY_LEAK_DETECTION != 'off' }}
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh report_netty_leaks

      - name: Upload container logs
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        continue-on-error: true
        with:
          name: System-${{ matrix.group }}-container-logs
          path: tests/integration/target/container-logs
          retention-days: 7

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        with:
          name: System-${{ matrix.name }}-surefire-reports
          path: surefire-reports
          retention-days: 7

      - name: Upload possible heap dump, core dump or crash files
        uses: actions/upload-artifact@v4
        if: ${{ always() }}
        with:
          name: System-${{ matrix.group }}-dumps
          path: |
            /tmp/*.hprof
            **/hs_err_*.log
            **/core.*
            ${{ env.NETTY_LEAK_DUMP_DIR }}/*
          retention-days: 7
          if-no-files-found: ignore

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  system-tests-upload-coverage:
    name: CI - System - Upload Coverage
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    needs: ['preconditions', 'system-tests']
    if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
    env:
      PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image

      - name: Restore coverage files from build artifacts and create Jacoco reports
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_systest_coverage_files
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
          cd $GITHUB_WORKSPACE/target
          zip -qr jacoco_test_coverage_report_systests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true

      - name: Upload Jacoco report files to build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: Jacoco-coverage-report-systests
          path: target/jacoco_test_coverage_report_systests.zip
          retention-days: 3

      - name: Upload to Codecov
        uses: ./.github/actions/upload-coverage
        with:
          flags: systests
          token: ${{ secrets.CODECOV_TOKEN }}

      - name: Delete coverage files from build artifacts
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_systest_coverage_files

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  flaky-system-tests:
    name: CI Flaky - System - ${{ matrix.name }}
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    needs: [ 'preconditions', 'pulsar-test-latest-version-image' ]
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      JOB_NAME: CI Flaky - System - ${{ matrix.name }}
      PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      NETTY_LEAK_DETECTION: "${{ needs.preconditions.outputs.netty_leak_detection }}"
      NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/target/netty-leak-dumps
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Plugin
            group: PLUGIN

          - name: Pulsar IO - Oracle
            group: PULSAR_IO_ORA
            clean_disk: true

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Clean Disk when needed
        if: ${{ matrix.clean_disk }}
        uses: ./.github/actions/clean-disk

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image

      - name: Run setup commands
        if: ${{ matrix.setup }}
        run: |
          ${{ matrix.setup }}

      - name: Run system test group '${{ matrix.group }}'
        run: |
          ./build/run_integration_group.sh ${{ matrix.group }}

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'test-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Report detected Netty leaks
        if: ${{ always() && env.NETTY_LEAK_DETECTION != 'off' }}
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh report_netty_leaks

      - name: Upload container logs
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        continue-on-error: true
        with:
          name: Flaky-System-${{ matrix.group }}-container-logs
          path: tests/integration/target/container-logs
          retention-days: 7

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        with:
          name: Flaky-System-${{ matrix.name }}-surefire-reports
          path: surefire-reports
          retention-days: 7

      - name: Upload possible heap dump, core dump or crash files
        uses: actions/upload-artifact@v4
        if: ${{ always() }}
        with:
          name: Flaky-System-${{ matrix.group }}-dumps
          path: |
            /tmp/*.hprof
            **/hs_err_*.log
            **/core.*
            ${{ env.NETTY_LEAK_DUMP_DIR }}/*
          retention-days: 7
          if-no-files-found: ignore

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  delete-system-test-docker-image-artifact:
    name: "Delete system test docker image artifact"
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    needs: [
      'preconditions',
      'system-tests',
      'system-tests-upload-coverage',
      'flaky-system-tests'
    ]
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Delete docker image from GitHub Actions Artifacts
        run: |
          gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst

  macos-build:
    name: Build Pulsar on MacOS
    runs-on: macos-latest
    timeout-minutes: 120
    needs: ['preconditions', 'integration-tests']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-all-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: build package
        run: mvn -B clean package -DskipTests -T 1C -ntp

  codeql:
    name: Run CodeQL Analysis
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    needs: ['preconditions', 'unit-tests']
    if: ${{ (needs.preconditions.outputs.java_non_tests == 'true' || github.event_name != 'pull_request') && ((github.event_name == 'pull_request' && github.base_ref == 'master') || (github.event_name != 'pull_request' && github.ref_name == 'master')) }}
    permissions:
      actions: read
      contents: read
      security-events: write
    env:
      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      CODEQL_LANGUAGE: java-kotlin
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Clean Disk when needed
        if: ${{ matrix.clean_disk }}
        uses: ./.github/actions/clean-disk

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Initialize CodeQL
        uses: github/codeql-action/init@v3
        with:
          languages: ${{ env.CODEQL_LANGUAGE }}

      - name: Build Java code
        run: |
          mvn -B -ntp -Pcore-modules,-main install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v3
        with:
          category: "/language:${{ env.CODEQL_LANGUAGE }}"

  owasp-dep-check:
    name: OWASP dependency check
    runs-on: ubuntu-24.04
    timeout-minutes: 120
    needs: [ 'preconditions', 'integration-tests' ]
    if: ${{ needs.preconditions.outputs.need_owasp == 'true' }}
    env:
      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Restore Maven repository cache
        uses: actions/cache/restore@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}

      - name: Clean Disk
        uses: ./.github/actions/clean-disk

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: OWASP cache key weeknum
        id: get-weeknum
        run: |
          echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT
        shell: bash

      - name: Restore OWASP Dependency Check data
        id: restore-owasp-dependency-check-data
        uses: actions/cache/restore@v4
        timeout-minutes: 5
        with:
          path: ~/.m2/repository/org/owasp/dependency-check-data
          key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }}
          enableCrossOsArchive: true
          restore-keys: |
            owasp-dependency-check-data-

      - name: Log warning when skipped
        if: ${{ !steps.restore-owasp-dependency-check-data.outputs.cache-matched-key }}
        run: |
          echo "::warning::OWASP Dependency Check was skipped since the OWASP Dependency check data wasn't found in the cache. Run ci-owasp-dependency-check.yaml workflow to update the cache."

      # Projects dependent on hdfs, and hbase currently excluded from the scan.
      - name: trigger dependency check
        if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-matched-key }}
        run: |
          mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \
            -pl '!distribution/server,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/hbase,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'

      - name: Upload report
        uses: actions/upload-artifact@v4
        if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-matched-key && (cancelled() || failure()) }}
        continue-on-error: true
        with:
          name: dependency report
          path: target/dependency-check-report.html

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  # This job is required for pulls to be merged. This job is referenced by name in .asf.yaml file in the
  # protected_branches section for master branch required_status_checks.
  # It depends on all other jobs in this workflow.
  # This job also cleans up the binaries at the end of the workflow.
  pulsar-ci-checks-completed:
    name: "Pulsar CI checks completed"
    # run always, but skip for other repositories than apache/pulsar when a scheduled workflow is cancelled
    # this is to allow the workflow scheduled jobs to show as cancelled instead of failed since scheduled
    # jobs are not enabled for other than apache/pulsar repository.
    if: ${{ always() && !(cancelled() && github.repository != 'apache/pulsar' && github.event_name == 'schedule') }}
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    needs: [
      'preconditions',
      'unit-tests',
      'integration-tests',
      'system-tests',
      'flaky-system-tests',
      'macos-build',
      'unit-tests-upload-coverage',
      'integration-tests-upload-coverage',
      'system-tests-upload-coverage',
      'owasp-dep-check',
      'codeql'
    ]
    steps:
      - name: Check that all required jobs were completed successfully
        if: ${{ needs.preconditions.result != 'success' || needs.preconditions.outputs.docs_only != 'true'  }}
        run: |
          if [[ ! ( \
                   "${{ needs.preconditions.result }}" == "success" \
                && "${{ needs.unit-tests.result }}" == "success" \
                && "${{ needs.integration-tests.result }}" == "success" \
                && "${{ needs.system-tests.result }}" == "success" \
                && "${{ needs.macos-build.result }}" == "success" \
                && ( "${{ needs.codeql.result }}" == "success" || "${{ needs.codeql.result }}" == "skipped" ) \
               ) ]]; then
            echo "Required jobs haven't been completed successfully."
            exit 1
          fi

      - name: checkout
        if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
        uses: actions/checkout@v4

      - name: Tune Runner VM
        if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
        uses: ./.github/actions/tune-runner-vm

      - name: Install gh-actions-artifact-client.js
        if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Delete maven repository binaries from GitHub Actions Artifacts
        if: ${{ needs.preconditions.outputs.docs_only != 'true' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
        run: |
          gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true
          gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true