Merge branch 'master' into enhance-medium-control

Author: Ryan19929

.asf.yaml

@@ -50,7 +50,7 @@ github:
           - Clang Formatter
           - CheckStyle
           - P0 Regression (Doris Regression)
-          - NonConcurrent Regression (Doris Regression)
+          - NonConcurrent Regression (nonConcurrent)
           - External Regression (Doris External Regression)
           - cloud_p0 (Doris Cloud Regression)
           - vault_p0 (Doris Cloud Regression)
@@ -77,13 +77,12 @@ github:
           - Clang Formatter
           - CheckStyle
           - Build Broker
-          - Build Third Party Libraries (Linux)
           - FE UT (Doris FE UT)
           - BE UT (Doris BE UT)
           - Cloud UT (Doris Cloud UT)
           - COMPILE (DORIS_COMPILE)
           - P0 Regression (Doris Regression)
-          - NonConcurrent Regression (Doris Regression)
+          - NonConcurrent Regression (nonConcurrent)
           - External Regression (Doris External Regression) 
           - cloud_p0 (Doris Cloud Regression)
           - vault_p0 (Doris Cloud Regression)
@@ -105,7 +104,7 @@ github:
           - Cloud UT (Doris Cloud UT)
           - COMPILE (DORIS_COMPILE)
           - P0 Regression (Doris Regression)
-          - NonConcurrent Regression (Doris Regression)
+          - NonConcurrent Regression (nonConcurrent)
           - External Regression (Doris External Regression) 
           - cloud_p0 (Doris Cloud Regression)
           - vault_p0 (Doris Cloud Regression)
@@ -127,7 +126,7 @@ github:
           - Cloud UT (Doris Cloud UT)
           - COMPILE (DORIS_COMPILE)
           - P0 Regression (Doris Regression)
-          - NonConcurrent Regression (Doris Regression)
+          - NonConcurrent Regression (nonConcurrent)
           - External Regression (Doris External Regression)
           - cloud_p0 (Doris Cloud Regression)
           - vault_p0 (Doris Cloud Regression)
@@ -140,7 +139,6 @@ github:
         strict: false
         contexts:
           - License Check
-          - Clang Formatter
           - CheckStyle
           - Build Broker
           - Build Third Party Libraries (Linux)
@@ -181,5 +179,7 @@ github:
     - yuanyuan8983
     - yz-jayhua
     - ixzc
+    - deardeng
+    - wyxxxcat
 notifications:
   pullrequests_status:  commits@doris.apache.org

.github/workflows/auto-cherry-pick.yml

@@ -31,7 +31,7 @@ permissions:
 jobs:
   auto_cherry_pick:
     runs-on: ubuntu-latest
-    if: ${{(contains(github.event.pull_request.labels.*.name, 'dev/4.0.x') || contains(github.event.pull_request.labels.*.name, 'dev/3.1.x') || github.event.label.name == 'dev/4.0.x' || github.event.label.name == 'dev/3.1.x') && github.event.pull_request.merged == true }}
+    if: ${{(contains(github.event.pull_request.labels.*.name, 'dev/4.0.x') || github.event.label.name == 'dev/4.0.x') && github.event.pull_request.merged == true }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -54,14 +54,7 @@ jobs:
             else
               echo "SHA matches: $calculated_sha"
             fi    
-      - name: Auto cherry-pick to branch-3.1
-        if: ${{ ((github.event.action == 'labeled' && github.event.label.name == 'dev/3.1.x'))|| ((github.event_name == 'pull_request_target' && github.event.action == 'closed') && contains(github.event.pull_request.labels.*.name, 'dev/3.1.x')) }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          REPO_NAME: ${{ github.repository }}
-          CONFLICT_LABEL: dev/3.1.x-conflict
-        run: |
-            python tools/auto-pick-script.py ${{ github.event.pull_request.number }} branch-3.1    
+
       - name: Auto cherry-pick to branch-4.0
         if: ${{ ((github.event.action == 'labeled' && github.event.label.name == 'dev/4.0.x'))|| ((github.event_name == 'pull_request_target' && github.event.action == 'closed') && contains(github.event.pull_request.labels.*.name, 'dev/4.0.x')) }}
         env:

.github/workflows/build-extension.yml

@@ -40,6 +40,7 @@ jobs:
     outputs:
       broker_changes: ${{ steps.filter.outputs.broker_changes }}
       docs_changes: ${{ steps.filter.outputs.docs_changes }}
+      cdc_client_changes: ${{ steps.filter.outputs.cdc_client_changes }}
     steps:
       - name: Checkout ${{ github.ref }}
         uses: actions/checkout@v3
@@ -53,9 +54,11 @@ jobs:
         with:
           filters: |
             broker_changes:
-              - 'fs_brokers/**'
+              - 'fs_brokers/apache_hdfs_broker/**'
             docs_changes:
               - 'docs/**'
+            cdc_client_changes:
+              - 'fs_brokers/cdc_client/**'
   build-broker:
     name: Build Broker
     needs: changes
@@ -92,6 +95,41 @@ jobs:
       - name: Build broker
         run: |
           cd fs_brokers/apache_hdfs_broker/ && /bin/bash build.sh
+  build-cdc-client:
+    name: Build Cdc Client
+    needs: changes
+    if: ${{ needs.changes.outputs.cdc_client_changes == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout ${{ github.ref }}
+        uses: actions/checkout@v3
+
+      - name: Setup java
+        uses: actions/setup-java@v2
+        with:
+          distribution: adopt
+          java-version: '17'
+
+      - name: Setup thrift
+        run: |
+          pushd thirdparty
+          branch="${{ github.base_ref }}"
+          if [[ -z "${branch}" ]] || [[ "${branch}" == 'master' || "${branch}" == 'branch-4.0' || "${branch}" == 'branch-3.0' || "${branch}" == 'branch-2.1' ]]; then
+            curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-prebuilt-linux-x86_64.tar.xz \
+              -o doris-thirdparty-prebuilt-linux-x86_64.tar.xz
+          else
+            curl -L "https://github.com/apache/doris-thirdparty/releases/download/automation-${branch/branch-/}/doris-thirdparty-prebuilt-linux-x86_64.tar.xz" \
+              -o doris-thirdparty-prebuilt-linux-x86_64.tar.xz
+          fi
+          tar -xvf doris-thirdparty-prebuilt-linux-x86_64.tar.xz
+          popd
+          export PATH="${DEFAULT_DIR}/ldb-toolchain/bin/:$(pwd)/thirdparty/installed/bin/:${PATH}"
+          
+          thrift --version
+
+      - name: Build cdc client
+        run: |
+          cd fs_brokers/cdc_client/ && /bin/bash build.sh
   # build-docs:
   #   name: Build Documents
   #   needs: changes

.github/workflows/build-thirdparty.yml

@@ -65,16 +65,17 @@ jobs:
         run: |
           git clone -b v7 https://github.com/easimon/maximize-build-space
 
-      - name: Maximize build space
-        uses: ./maximize-build-space
-        with:
-            root-reserve-mb: 4096
-            swap-size-mb: 8192
-            remove-dotnet: 'true'
-            remove-android: 'true'
-            remove-haskell: 'true'
-            remove-codeql: 'true'
-            remove-docker-images: 'true'
+      #- name: Maximize build space
+      #  uses: ./maximize-build-space
+      #  with:
+      #      root-reserve-mb: 4096
+      #      temp-reserve-mb: 4096
+      #      swap-size-mb: 8192
+      #      remove-dotnet: 'true'
+      #      remove-android: 'true'
+      #      remove-haskell: 'true'
+      #      remove-codeql: 'true'
+      #      remove-docker-images: 'true'
 
       - name: Checkout ${{ github.ref }}
         uses: actions/checkout@v4
@@ -135,8 +136,18 @@ jobs:
           export CMAKE_POLICY_VERSION_MINIMUM=3.10
           export CUSTOM_CMAKE="/usr/local/bin/cmake"
 
+          #export TMPDIR=/home/runner/work/doris/doris/.tmp
+          #export TMP=$TMPDIR
+          #export TEMP=$TMPDIR
+          #mkdir -p $TMPDIR
+
+          df -h
+          echo $TMPDIR
+          echo $RUNNER_TEMP
+
           cd thirdparty
-          ./build-thirdparty.sh -j "$(nproc)"
+          #./build-thirdparty.sh -j "$(nproc)"
+          ./build-thirdparty.sh -j 2
 
   build_macos:
     name: Build Third Party Libraries (macOS)
@@ -195,7 +206,12 @@ jobs:
           export CMAKE_POLICY_VERSION_MINIMUM=3.10
           export CUSTOM_CMAKE="/usr/local/bin/cmake"
 
+          df -h
+          echo $TMPDIR
+          echo $RUNNER_TEMP
+
           cd thirdparty
+          #./build-thirdparty.sh -j "$(nproc)"
           ./build-thirdparty.sh -j "$(nproc)"
 
   build_macos_arm64:
@@ -255,6 +271,11 @@ jobs:
           export CMAKE_POLICY_VERSION_MINIMUM=3.10
           export CUSTOM_CMAKE="/usr/local/bin/cmake"
 
+          df -h
+          echo $TMPDIR
+          echo $RUNNER_TEMP
+
           cd thirdparty
-          ./build-thirdparty.sh -j "$(nproc)"
+          #./build-thirdparty.sh -j "$(nproc)"
+          ./build-thirdparty.sh -j 2