Merge remote-tracking branch 'upstream/master'

Author: MarcBresson

.github/workflows/pipe-it-2cluster.yml

@@ -9,6 +9,7 @@ on:
     paths-ignore:
       - 'docs/**'
       - 'site/**'
+      - 'iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**' #queryengine
   pull_request:
     branches:
       - master
@@ -17,6 +18,7 @@ on:
     paths-ignore:
       - 'docs/**'
       - 'site/**'
+      - 'iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**' #queryengine
   # allow manually run the action:
   workflow_dispatch:
 
@@ -37,7 +39,7 @@ jobs:
       matrix:
         java: [17]
         # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
-        cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode]
+        cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
         os: [ ubuntu-latest ]
     runs-on: ${{ matrix.os }}
     steps:
@@ -47,6 +49,9 @@ jobs:
         with:
           distribution: liberica
           java-version: ${{ matrix.java }}
+      - name: Sleep for a random duration between 0 and 10000 milliseconds
+        run: |
+          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
       - name: IT Test
         shell: bash
         # we do not compile client-cpp for saving time, it is tested in client.yml
@@ -74,7 +79,7 @@ jobs:
       matrix:
         java: [17]
         # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
-        cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode]
+        cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
         cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode]
         os: [ ubuntu-latest ]
         exclude:
@@ -90,6 +95,14 @@ jobs:
             cluster2: LightWeightStandaloneMode
           - cluster1: HighPerformanceMode
             cluster2: HighPerformanceMode
+          - cluster1: PipeConsensusBatchMode
+            cluster2: LightWeightStandaloneMode
+          - cluster1: PipeConsensusBatchMode
+            cluster2: HighPerformanceMode
+          - cluster1: PipeConsensusStreamMode
+            cluster2: LightWeightStandaloneMode
+          - cluster1: PipeConsensusStreamMode
+            cluster2: HighPerformanceMode
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
@@ -98,6 +111,9 @@ jobs:
         with:
           distribution: liberica
           java-version: ${{ matrix.java }}
+      - name: Sleep for a random duration between 0 and 10000 milliseconds
+        run: |
+          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
       - name: IT Test
         shell: bash
         # we do not compile client-cpp for saving time, it is tested in client.yml
@@ -125,7 +141,7 @@ jobs:
       matrix:
         java: [ 17 ]
         # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
-        cluster1: [ LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode ]
+        cluster1: [ ScalableSingleNodeMode ]
         cluster2: [ ScalableSingleNodeMode ]
         os: [ ubuntu-latest ]
     runs-on: ${{ matrix.os }}
@@ -136,6 +152,9 @@ jobs:
         with:
           distribution: liberica
           java-version: ${{ matrix.java }}
+      - name: Sleep for a random duration between 0 and 10000 milliseconds
+        run: |
+          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
       - name: IT Test
         shell: bash
         # we do not compile client-cpp for saving time, it is tested in client.yml
@@ -174,6 +193,9 @@ jobs:
         with:
           distribution: liberica
           java-version: ${{ matrix.java }}
+      - name: Sleep for a random duration between 0 and 10000 milliseconds
+        run: |
+          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
       - name: IT Test
         shell: bash
         # we do not compile client-cpp for saving time, it is tested in client.yml
@@ -212,6 +234,9 @@ jobs:
         with:
           distribution: liberica
           java-version: ${{ matrix.java }}
+      - name: Sleep for a random duration between 0 and 10000 milliseconds
+        run: |
+          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
       - name: IT Test
         shell: bash
         # we do not compile client-cpp for saving time, it is tested in client.yml
@@ -232,3 +257,43 @@ jobs:
           name: cluster-log-subscription-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
           path: integration-test/target/cluster-logs
           retention-days: 30
+  table-model:
+    strategy:
+      fail-fast: false
+      max-parallel: 15
+      matrix:
+        java: [17]
+        # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
+        cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode]
+        os: [ ubuntu-latest ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: liberica
+          java-version: ${{ matrix.java }}
+      - name: Sleep for a random duration between 0 and 10000 milliseconds
+        run: |
+          sleep  $(( $(( RANDOM % 10000 + 1 )) / 1000))
+      - name: IT Test
+        shell: bash
+        # we do not compile client-cpp for saving time, it is tested in client.yml
+        # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
+        run: |
+          mvn clean verify \
+          -P with-integration-tests \
+          -DskipUTs \
+          -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
+          -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
+          -pl integration-test \
+          -am -PMultiClusterIT2TableModel \
+          -ntp
+      - name: Upload Artifact
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cluster-log-table-model-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
+          path: integration-test/target/cluster-logs
+          retention-days: 30

.github/workflows/vulnerability-check.yml

@@ -0,0 +1,49 @@
+name: vulnerability-check
+on:
+  schedule:
+    # Run at UTC 00:00 every week (CST 03:00 AM)
+    - cron: '0 0 * * 3'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_ARGS: --batch-mode --no-transfer-progress
+  DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+
+jobs:
+  dependency-check:
+    strategy:
+      fail-fast: false
+      max-parallel: 15
+      matrix:
+        java: [ 17 ]
+        os: [ ubuntu-latest ]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: corretto
+          java-version: ${{ matrix.java }}
+      - name: Cache Maven packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2-
+      - name: Do the dependency-check:check
+        shell: bash
+        run: mvn org.owasp:dependency-check-maven:check
+      - name: Do the dependency-check:aggregate
+        shell: bash
+        run: mvn org.owasp:dependency-check-maven:aggregate
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: vulnerability-check-result-${{ runner.os }}
+          path: target/dependency-check-report.html
+          retention-days: 15

LICENSE

@@ -204,23 +204,16 @@
                            APACHE IOTDB SUBCOMPONENTS
 --------------------------------------------------------------------------------
 
-The following class is copied from maven-wrapper (https://github.com/takari/maven-wrapper),
-which is under Apache License 2.0:
-
-mvnw files from https://github.com/apache/maven-wrapper Apache 2.0
-
---------------------------------------------------------------------------------
-
 The following files include code modified from Apache HBase project.
 
-./iotdb-core/confignode/src/main/java/org/apache/iotdb/procedure/Procedure.java
-./iotdb-core/confignode/src/main/java/org/apache/iotdb/procedure/ProcedureExecutor.java
-./iotdb-core/confignode/src/main/java/org/apache/iotdb/procedure/StateMachineProcedure.java
-./iotdb-core/confignode/src/main/java/org/apache/iotdb/procedure/TimeoutExecutorThread.java
-./iotdb-core/confignode/src/main/java/org/apache/iotdb/procedure/StoppableThread.java
+./iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java
+./iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
+./iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/StateMachineProcedure.java
+./iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
+./iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/StoppableThread.java
 
 Project page: https://github.com/apache/hbase
-License: http://www.apache.org/licenses/LICENSE-2.0
+License: https://github.com/apache/hbase/blob/master/LICENSE.txt
 
 --------------------------------------------------------------------------------
 
@@ -242,30 +235,32 @@ License: https://github.com/ggalmazor/lt_downsampling_java8/blob/main/LICENSE
 
 The following files include code modified from Micrometer project.
 
-./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/predefined/jvm/JvmClassLoaderMetrics
-./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/predefined/jvm/JvmCompileMetrics
-./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/predefined/jvm/JvmGcMetrics
-./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/predefined/jvm/JvmMemoryMetrics
-./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/predefined/jvm/JvmThreadMetrics
-./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/predefined/logback/LogbackMetrics
-./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/JvmUtils
+./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmClassLoaderMetrics.java
+./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmCompileMetrics.java
+./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmGcMetrics.java
+./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmMemoryMetrics.java
+./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/jvm/JvmThreadMetrics.java
+./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/JvmUtils.java
+./iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/logback/logback/LogbackMetrics.java
 
 Copyright: 2017 VMware
 Project page: https://github.com/micrometer-metrics/micrometer
 License: https://github.com/micrometer-metrics/micrometer/blob/main/LICENSE
 
 --------------------------------------------------------------------------------
 
-The following files include code modified from Trino project(https://github.com/trinodb/trino),
-which is under Apache License 2.0:
+The following files include code modified from Trino project.
 
 ./iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/QueryState.java
 ./iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/StateMachine.java
 ./iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/Driver.java
 ./iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceState.java
 ./iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceStateMachine.java
 ./iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
-./iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/*
+
+Trino is open source software licensed under the Apache License 2.0 and supported by the Trino Software Foundation.
+Project page: https://github.com/trinodb/trino
+License: https://github.com/trinodb/trino/blob/master/LICENSE
 
 --------------------------------------------------------------------------------
 
@@ -274,4 +269,4 @@ The following files include code modified from Apache Kafka project.
 ./iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util/PollTimer.java
 
 Project page: https://github.com/apache/kafka
-License: http://www.apache.org/licenses/LICENSE-2.0
+License: https://github.com/apache/kafka/blob/trunk/LICENSE