backend.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the 'License'); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Backend

on:
  workflow_call:
    inputs:
      TEST_IN_PR:
        required: false
        type: string
        default: 'true'

concurrency:
  group: backend-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  TEST_IN_PR: ${{ inputs.TEST_IN_PR }}

jobs:
  license-header:
    name: License header
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true
      - name: Check license header
        uses: apache/skywalking-eyes@v0.5.0

  code-style:
    name: Code style
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true
      - name: Check code style
        run: ./mvnw --batch-mode --quiet --no-snapshot-updates clean spotless:check
      - name: Check code specification
        run: ./mvnw -B -T 1 clean test -D"license.skipAddThirdParty"=true -pl seatunnel-ci-tools -am --no-snapshot-updates
        env:
          MAVEN_OPTS: -Xmx512m
      - name: Check for .class files in git
        run: |
          echo "Checking for .class files tracked by git..."

          # Find all .class files tracked by git
          CLASS_FILES=$(git ls-files '*.class')

          if [ -n "$CLASS_FILES" ]; then
            echo "ERROR: The following .class files are tracked by git:"
            echo "$CLASS_FILES"
            echo ""
            echo "Please remove .class files from the repository."
            echo "These files should not be committed. You can remove them using:"
            echo "  git rm --cached <file>.class"
            echo "  git commit -m 'Remove .class files'"
            echo ""
            echo "Also, consider adding '*.class' to .gitignore if not already present."
            exit 1
          else
            echo "No .class files found in git repository."
          fi

  helm-chart-check:
    name: Check Helm Chart Syntax
    needs: [ license-header, code-style]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Setup Helm
        uses: azure/setup-helm@v4.3.0
        id: install
      - name: Lint Chart
        run: helm lint deploy/kubernetes/seatunnel

#  dead-link:
#    name: Dead links
#    runs-on: ubuntu-latest
#    timeout-minutes: 150
#    # Temporarily ignore this job to avoid blocking PRs
#    continue-on-error: true
#    steps:
#      - uses: actions/checkout@v2
#      - run: sudo npm install -g markdown-link-check@3.8.7
#      - run: |
#          for file in $(find . -name "*.md"); do
#            markdown-link-check -c .dlc.json -q "$file"
#          done

  sanity-check:
    name: Sanity check results
    needs: [ license-header, code-style ]
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Check results
        run: |
          [[ ${{ needs.license-header.result }} == 'success' ]] || exit 1;
          [[ ${{ needs.code-style.result }} == 'success' ]] || exit 1;

  changes:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    outputs:
      api: ${{ steps.filter.outputs.api }}
      engine: ${{ steps.filter.outputs.engine }}
      engine-e2e: ${{ steps.filter.outputs.engine-e2e }}
      docs: ${{ steps.filter.outputs.docs }}
      ut-modules: ${{ steps.ut-modules.outputs.modules }}
      it-modules: ${{ steps.it-modules.outputs.modules }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: '2000'
      - name: checkout apache seatunnel dev branch
        id: git_init
        run: |
          /usr/bin/git remote add apache https://github.com/apache/seatunnel
          /usr/bin/git -c protocol.version=2 fetch --no-tags --prune --no-recurse-submodules --depth=2000 apache +refs/heads/dev*:refs/remotes/apache/dev* +refs/tags/dev*:refs/tags/dev*
          /usr/bin/git checkout apache/dev
          /usr/bin/git checkout '${{ github.ref }}'
          echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
      - uses: actions/setup-python@v4
        with:
          python-version: '3.11.0'
      - name: Check for file changes by python
        id: filter
        run: |
          current_branch='${{ steps.git_init.outputs.branch }}'
          pip install GitPython
          workspace="${GITHUB_WORKSPACE}"
          repository_owner="${GITHUB_REPOSITORY_OWNER}"
          cv2_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-connectors-v2/**"`
          true_or_false=${cv2_files%%$'\n'*}
          file_list=${cv2_files#*$'\n'}
          echo "cv2=$true_or_false" >> $GITHUB_OUTPUT
          echo "cv2_files=$file_list" >> $GITHUB_OUTPUT
          
          cv2_e2e_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-e2e/seatunnel-connector-v2-e2e/**"`
          true_or_false=${cv2_e2e_files%%$'\n'*}
          file_list=${cv2_e2e_files#*$'\n'}
          echo "cv2-e2e=$true_or_false" >> $GITHUB_OUTPUT
          echo "cv2-e2e_files=$file_list" >> $GITHUB_OUTPUT
          
          engine_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-engine/**"`
          true_or_false=${engine_files%%$'\n'*}
          file_list=${engine_files#*$'\n'}
          echo "engine=$true_or_false" >> $GITHUB_OUTPUT
          echo "engine_files=$file_list" >> $GITHUB_OUTPUT
          
          deleted_poms_files=`python tools/update_modules_check/check_file_updates.py d $workspace apache/dev origin/$current_branch "**/pom.xml"`
          true_or_false=${deleted_poms_files%%$'\n'*}
          file_list=${deleted_poms_files#*$'\n'}
          echo "deleted-poms=$true_or_false" >> $GITHUB_OUTPUT
          echo "deleted-poms_files=$file_list" >> $GITHUB_OUTPUT
          
          doc_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "docs/**"`
          true_or_false=${doc_files%%$'\n'*}
          file_list=${doc_files#*$'\n'}
          echo "docs=$true_or_false" >> $GITHUB_OUTPUT
          echo "docs_files=$file_list" >> $GITHUB_OUTPUT
          
          engine_e2e_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-e2e/seatunnel-engine-e2e/**"`
          true_or_false=${engine_e2e_files%%$'\n'*}
          file_list=${engine_e2e_files#*$'\n'}
          echo "engine-e2e=$true_or_false" >> $GITHUB_OUTPUT
          echo "engine-e2e_files=$file_list" >> $GITHUB_OUTPUT
          
          api_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-api/**" "seatunnel-common/**" "seatunnel-config/**" "seatunnel-core/**" "seatunnel-e2e/seatunnel-e2e-common/**" "seatunnel-formats/**" "seatunnel-plugin-discovery/**" "seatunnel-transforms-v2/**" "seatunnel-translation/**" "seatunnel-e2e/seatunnel-transforms-v2-e2e/**" "pom.xml" "**/workflows/**" "tools/**" "seatunnel-dist/**"`
          true_or_false=${api_files%%$'\n'*}
          file_list=${api_files#*$'\n'}
          if [[ $repository_owner == 'apache' ]];then
            true_or_false='true'
          fi
          echo "api=$true_or_false" >> $GITHUB_OUTPUT
          echo "api_files=$file_list" >> $GITHUB_OUTPUT

      - name: Check Connector V2 Update
        id: cv2-modules
        if: ${{ steps.filter.outputs.cv2 == 'true' }}
        run: |
          update_files='${{ steps.filter.outputs.cv2_files }}'
          modules=`python tools/update_modules_check/update_modules_check.py cv2 "$update_files"`
          echo $modules
          echo "modules=$modules" >> $GITHUB_OUTPUT

      - name: Check Connector V2 E2E Update
        id: cv2-e2e-modules
        if: ${{ steps.filter.outputs.cv2-e2e == 'true' }}
        run: |
          update_files='${{ steps.filter.outputs.cv2-e2e_files }}'
          modules=`python tools/update_modules_check/update_modules_check.py cv2-e2e "$update_files"`
          echo $modules
          echo "modules=$modules" >> $GITHUB_OUTPUT

      - name: Check Engine Update
        id: engine-modules
        if: ${{ steps.filter.outputs.engine == 'true' }}
        run: |
          update_files='${{ steps.filter.outputs.engine_files }}'
          modules=`python tools/update_modules_check/update_modules_check.py engine "$update_files"`
          echo $modules
          echo "modules=$modules" >> $GITHUB_OUTPUT

      - name: Check Engine E2E Update
        id: engine-e2e-modules
        if: ${{ steps.filter.outputs.engine-e2e == 'true' }}
        run: |
          update_files='${{ steps.filter.outputs.engine-e2e_files }}'
          modules=`python tools/update_modules_check/update_modules_check.py engine-e2e "$update_files"`
          echo $modules
          echo "modules=$modules" >> $GITHUB_OUTPUT

      - name: Check Deleted Modules
        id: deleted-modules
        if: ${{ steps.filter.outputs.deleted-poms == 'true' }}
        run: |
          update_files='${{ steps.filter.outputs.deleted-poms_files }}'
          modules=`python tools/update_modules_check/update_modules_check.py delete "$update_files"`
          echo $modules
          echo "modules=$modules" >> $GITHUB_OUTPUT      

      - name: Make unit test modules
        id: ut-modules
        timeout-minutes: 60
        if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '') }}
        run: |
          modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
          modules=${modules: 1}
          pl_modules=`python tools/update_modules_check/update_modules_check.py replace "$modules"`
          # remove deleted modules
          delete_modules='${{ steps.deleted-modules.outputs.modules }}'
          if [[ "zz"$delete_modules != "zz" ]];then
            pl_modules=`python tools/update_modules_check/update_modules_check.py rm "$pl_modules" "$delete_modules"`
          fi
          
          if [[ "zz"$pl_modules == "zz" ]];then
            exit 0
          fi
          
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl $pl_modules > /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          tree_modules="$modules$sub_modules"
          includes=`python tools/update_modules_check/update_modules_check.py tree "$tree_modules"`
          ./mvnw -Pci -D"e2e.dependency.skip"=false  dependency:tree $includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt
          build_modules=`python tools/update_modules_check/update_modules_check.py final_ut /tmp/tree_out.txt`
          if [[ "zz"$build_modules == "zz" ]];then
            build_modules=$pl_modules
          fi 
          echo $build_modules
          echo "modules=$build_modules" >> $GITHUB_OUTPUT

      - name: Make integration test modules
        id: it-modules
        timeout-minutes: 60
        if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '' || steps.cv2-e2e-modules.outputs.modules != '' || steps.cv2-flink-e2e-modules.outputs.modules != '' || steps.cv2-spark-e2e-modules.outputs.modules != '') }}
        run: |
          modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.cv2-flink-e2e-modules.outputs.modules }}${{ steps.cv2-spark-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
          modules=${modules: 1}
          pl_modules=`python tools/update_modules_check/update_modules_check.py replace "$modules"`
          # remove deleted modules
          delete_modules='${{ steps.deleted-modules.outputs.modules }}'
          if [[ "zz"$delete_modules != "zz" ]];then
            pl_modules=`python tools/update_modules_check/update_modules_check.py rm "$pl_modules" "$delete_modules"`
          fi
          
          if [[ "zz"$pl_modules == "zz" ]];then
            exit 0
          fi
          
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl $pl_modules > /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          tree_modules="$modules$sub_modules"
          includes=`python tools/update_modules_check/update_modules_check.py tree "$tree_modules"`
          ./mvnw -Pci -D"e2e.dependency.skip"=false  dependency:tree $includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt
          build_modules=`python tools/update_modules_check/update_modules_check.py final_it /tmp/tree_out.txt`
          echo $build_modules
          echo "modules=$build_modules" >> $GITHUB_OUTPUT

  dependency-license:
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    name: Dependency licenses
    needs: [ changes, sanity-check ]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true
      - uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '8'
          cache: 'maven'
      - name: Install
        uses: nick-fields/retry@v2
        with:
          timeout_minutes: 40
          max_attempts: 3
          retry_on: error
          command: |
            ./mvnw -B install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"license.skipAddThirdParty" -D"skip.ui"=true
      - name: Check Dependencies Licenses
        run: tools/dependencies/checkLicense.sh

  document:
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.docs == 'true'
    needs: [ changes, sanity-check ]
    name: Build website
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout PR
        uses: actions/checkout@v3
        with:
          path: seatunnel-pr
      - name: Checkout website repo
        uses: actions/checkout@v3
        with:
          repository: apache/seatunnel-website
          path: seatunnel-website
      - name: Sync PR changes to website
        run: |
          bash seatunnel-pr/tools/documents/sync.sh seatunnel-pr seatunnel-website
      - uses: actions/setup-node@v2
        with:
          node-version: 18.20.7
      - name: Run docusaurus build
        run: |
          cd seatunnel-website
          npm set strict-ssl false
          npm install
          npm run build

  seatunnel-ui:
    if: needs.changes.outputs.api == 'true'
    needs: [ changes, sanity-check ]
    name: Build SeaTunnel UI
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout PR
        uses: actions/checkout@v3
      - uses: actions/setup-node@v2
        with:
          node-version: 20.x
      - name: Install Dependencies and Check Code Style
        run: |
          cd seatunnel-engine/seatunnel-engine-ui/
          npm install
          npm run lint
      - name: Run unit tests
        run: |
          cd seatunnel-engine/seatunnel-engine-ui/
          npm run test:unit
      - name: Build SeaTunnel UI
        run: |
          cd seatunnel-engine/seatunnel-engine-ui/
          npm run build

  unit-test:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api == 'false' && needs.changes.outputs.ut-modules != '')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest', 'windows-latest' ]
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run all modules unit test
        run: |
          ./mvnw -B -T 1 clean verify -DskipUT=false -DskipIT=true -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates
        env:
          MAVEN_OPTS: -Xmx4096m

  updated-modules-integration-test-part-1:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != ''
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run updated modules integration test (part-1)
        run: |
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 0`
          if [ ! -z $sub_modules ]; then
            echo $sub_modules
            ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci
          else
            echo "sub modules is empty, skipping"
          fi
        env:
          MAVEN_OPTS: -Xmx2048m

  updated-modules-integration-test-part-2:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != ''
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 150
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run updated modules integration test (part-2)
        run: |
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 1`
          if [ ! -z $sub_modules ]; then
            echo $sub_modules
            ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci
          else
            echo "sub modules is empty, skipping"
          fi
        env:
          MAVEN_OPTS: -Xmx4096m

  updated-modules-integration-test-part-3:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != ''
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run updated modules integration test (part-3)
        run: |
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 2`
          if [ ! -z $sub_modules ]; then
            echo $sub_modules
            ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci
          else
            echo "sub modules is empty, skipping"
          fi
        env:
          MAVEN_OPTS: -Xmx2048m

  updated-modules-integration-test-part-4:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != ''
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 200
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run updated modules integration test (part-4)
        run: |
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 3`
          if [ ! -z $sub_modules ]; then
            echo $sub_modules
            ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci
          else
            echo "sub modules is empty, skipping"
          fi
        env:
          MAVEN_OPTS: -Xmx4096m
  updated-modules-integration-test-part-5:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != ''
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run updated modules integration test (part-5)
        run: |
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 4`
          if [ ! -z $sub_modules ]; then
            echo $sub_modules
            ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci
          else
            echo "sub modules is empty, skipping"
          fi
        env:
          MAVEN_OPTS: -Xmx2048m
  updated-modules-integration-test-part-6:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != ''
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run updated modules integration test (part-6)
        run: |
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 5`
          if [ ! -z $sub_modules ]; then
            echo $sub_modules
            ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci
          else
            echo "sub modules is empty, skipping"
          fi
        env:
          MAVEN_OPTS: -Xmx2048m
  updated-modules-integration-test-part-7:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != ''
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run updated modules integration test (part-7)
        run: |
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 6`
          if [ ! -z $sub_modules ]; then
            echo $sub_modules
            ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci
          else
            echo "sub modules is empty, skipping"
          fi
        env:
          MAVEN_OPTS: -Xmx2048m

  updated-modules-integration-test-part-8:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine == 'false' && needs.changes.outputs.it-modules != ''
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run updated modules integration test (part-8)
        run: |
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 8 7`
          if [ ! -z $sub_modules ]; then
            echo $sub_modules
            ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $sub_modules -am -Pci
          else
            echo "sub modules is empty, skipping"
          fi
        env:
          MAVEN_OPTS: -Xmx2048m

  engine-v2-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || needs.changes.outputs.engine-e2e == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run seatunnel zeta integration test
        run: |
          ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-seatunnel-e2e-base,:connector-console-seatunnel-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  engine-k8s-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || contains(needs.changes.outputs.it-modules, 'seatunnel-engine-k8s-e2e')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 45
    steps:
      - name: install k8s
        run: |
          curl -sfL https://get.k3s.io | K3S_KUBECONFIG_MODE=777 sh -s - --docker
          cat /etc/rancher/k3s/k3s.yaml
          mkdir -p ~/.kube
          cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
        env:
          KUBECONFIG: /etc/rancher/k3s/k3s.yaml
      - uses: actions/checkout@v2
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: run seatunnel zeta on k8s test
        run: |
          ./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :seatunnel-engine-k8s-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m
          KUBECONFIG: /etc/rancher/k3s/k3s.yaml

  transform-v2-it-part-1:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run transform-v2 integration test (part-1)
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-1 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  transform-v2-it-part-2:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 150
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run transform-v2 integration test (part-2)
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-2 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-1:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 210
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run connector-v2 integration test (part-1)
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 0`
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-2:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 150
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run connector-v2 integration test (part-2)
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 1`
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-3:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 210
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run connector-v2 integration test (part-3)
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 2`
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-4:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 210
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run connector-v2 integration test (part-4)
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 3`
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-5:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run connector-v2 integration test (part-5)
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 4`
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-6:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 210
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run connector-v2 integration test (part-6)
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 5`
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  all-connectors-it-7:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 210
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run connector-v2 integration test (part-7)
        run: |
          ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
          sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
          run_it_modules=`python tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules" 7 6`
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl $run_it_modules -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  jdbc-connectors-it-part-1:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run jdbc connectors integration test (part-1)
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-1 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  jdbc-connectors-it-part-2:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run jdbc connectors integration test (part-2)
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-2 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  jdbc-connectors-it-part-3:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run jdbc connectors integration test (part-3)
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-3 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  jdbc-connectors-it-part-4:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run jdbc connectors integration test (part-4)
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-4 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  jdbc-connectors-it-part-5:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run jdbc connectors integration test (part-5)
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-5 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  jdbc-connectors-it-part-6:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run jdbc connectors integration test (part-6)
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-6 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  jdbc-connectors-it-part-7:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run jdbc connectors integration test (part-7)
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-part-7 -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  jdbc-connectors-it-ddl:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
    runs-on: ${{ matrix.os }}
    env:
      RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
      RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run jdbc connectors integration test (sink ddl)
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-jdbc-e2e-ddl -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  kudu-connector-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-kudu-e2e')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run kudu connector integration test
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-kudu-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  amazonSqs-connector-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-amazonsqs-e2e')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run amazonsqs connector integration test
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-amazonsqs-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  kafka-connector-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-kafka-e2e')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 210
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run kafka connector integration test
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-kafka-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  rocketmq-connector-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-rocketmq-e2e')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run rocket connector integration test
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-rocketmq-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m


  doris-connector-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-doris-e2e')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run doris connector integration test
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-doris-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  paimon-connector-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-paimon-e2e')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run paimon connector integration test
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-paimon-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  oracle-cdc-connector-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-cdc-oracle-e2e')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 210
    steps:
      - name: Checkout repository
        uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run oracle cdc connector integration test
        uses: nick-fields/retry@v3
        with:
          timeout_seconds: 9000
          max_attempts: 3
          retry_on: error
          command: |
            echo 'running oracle cdc connector integration test...' && \
            ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-cdc-oracle-e2e -am -Pci

  connector-file-local-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-file-local-e2e')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run file local connector integration test
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-file-local-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  connector-file-sftp-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-file-sftp-e2e')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run file sftp connector integration test
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-file-sftp-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  connector-redis-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(needs.changes.outputs.it-modules, 'connector-redis-e2e')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 210
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run redis connector integration test
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-redis-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m

  connector-sensorsdata-it:
    needs: [ changes, sanity-check ]
    if: needs.changes.outputs.api == 'true' || contains(needs.changes.outputs.it-modules, 'connector-sensorsdata-e2e')
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        java: [ '8', '11' ]
        os: [ 'ubuntu-latest' ]
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v3
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
          cache: 'maven'
      - name: free disk space
        run: tools/github/free_disk_space.sh
      - name: run sensorsdata connector integration test
        run: |
          ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :connector-sensorsdata-e2e -am -Pci
        env:
          MAVEN_OPTS: -Xmx4096m