Daily IT #540
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Daily IT | |
| on: | |
| schedule: | |
| # Run at UTC 19:00 every day (CST 03:00 AM) | |
| - cron: "0 19 * * *" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 | |
| MAVEN_ARGS: --batch-mode --no-transfer-progress | |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} | |
| jobs: | |
| Simple: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 20 | |
| matrix: | |
| java: [8, 17] | |
| runs-on: [self-hosted, iotdb] | |
| # group: self-hosted | |
| # labels: iotdb | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: IT/UT Test | |
| shell: bash | |
| run: | | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.forkCount=6 -DConfigNodeMaxHeapSize=1024 -DDataNodeMaxHeapSize=1024 \ | |
| -pl integration-test \ | |
| -am -PDailyIT | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cluster-log-java${{ matrix.java }}-${{ runner.os }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 3 | |
| SingleRegionTableModel: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [8, 17] | |
| runs-on: [self-hosted, iotdb] | |
| # group: self-hosted | |
| # labels: iotdb | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: IT/UT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 -DintegrationTest.dataRegionPerDataNode=1 \ | |
| -pl integration-test \ | |
| -am -PTableSimpleIT | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 3 | |
| PipeSingle: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster1: | |
| [ | |
| LightWeightStandaloneMode, | |
| ScalableSingleNodeMode, | |
| HighPerformanceMode, | |
| PipeConsensusBatchMode, | |
| PipeConsensusStreamMode, | |
| ] | |
| cluster2: | |
| [ | |
| LightWeightStandaloneMode, | |
| ScalableSingleNodeMode, | |
| HighPerformanceMode, | |
| ] | |
| os: [ubuntu-latest] | |
| exclude: | |
| - cluster1: LightWeightStandaloneMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: LightWeightStandaloneMode | |
| cluster2: ScalableSingleNodeMode | |
| - cluster1: ScalableSingleNodeMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: ScalableSingleNodeMode | |
| cluster2: HighPerformanceMode | |
| - cluster1: HighPerformanceMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: HighPerformanceMode | |
| cluster2: HighPerformanceMode | |
| - cluster1: PipeConsensusBatchMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: PipeConsensusBatchMode | |
| cluster2: HighPerformanceMode | |
| - cluster1: PipeConsensusStreamMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: PipeConsensusStreamMode | |
| cluster2: HighPerformanceMode | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT1 \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| PipeDualTreeAutoBasic: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster: | |
| [ | |
| LightWeightStandaloneMode, | |
| ScalableSingleNodeMode, | |
| HighPerformanceMode, | |
| PipeConsensusBatchMode, | |
| PipeConsensusStreamMode, | |
| ] | |
| os: [ubuntu-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2DualTreeAutoBasic \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| PipeDualTreeAutoEnhanced: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster1: | |
| [ | |
| LightWeightStandaloneMode, | |
| ScalableSingleNodeMode, | |
| HighPerformanceMode, | |
| PipeConsensusBatchMode, | |
| PipeConsensusStreamMode, | |
| ] | |
| cluster2: | |
| [ | |
| LightWeightStandaloneMode, | |
| ScalableSingleNodeMode, | |
| HighPerformanceMode, | |
| ] | |
| os: [ubuntu-latest] | |
| exclude: | |
| - cluster1: LightWeightStandaloneMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: LightWeightStandaloneMode | |
| cluster2: ScalableSingleNodeMode | |
| - cluster1: ScalableSingleNodeMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: ScalableSingleNodeMode | |
| cluster2: HighPerformanceMode | |
| - cluster1: HighPerformanceMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: HighPerformanceMode | |
| cluster2: HighPerformanceMode | |
| - cluster1: PipeConsensusBatchMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: PipeConsensusBatchMode | |
| cluster2: HighPerformanceMode | |
| - cluster1: PipeConsensusStreamMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: PipeConsensusStreamMode | |
| cluster2: HighPerformanceMode | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2DualTreeAutoEnhanced \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| PipeDualTreeManual: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster1: | |
| [ | |
| LightWeightStandaloneMode, | |
| ScalableSingleNodeMode, | |
| HighPerformanceMode, | |
| PipeConsensusBatchMode, | |
| PipeConsensusStreamMode, | |
| ] | |
| cluster2: | |
| [ | |
| LightWeightStandaloneMode, | |
| ScalableSingleNodeMode, | |
| HighPerformanceMode, | |
| ] | |
| os: [ubuntu-latest] | |
| exclude: | |
| - cluster1: LightWeightStandaloneMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: LightWeightStandaloneMode | |
| cluster2: ScalableSingleNodeMode | |
| - cluster1: ScalableSingleNodeMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: ScalableSingleNodeMode | |
| cluster2: HighPerformanceMode | |
| - cluster1: HighPerformanceMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: HighPerformanceMode | |
| cluster2: HighPerformanceMode | |
| - cluster1: PipeConsensusBatchMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: PipeConsensusBatchMode | |
| cluster2: HighPerformanceMode | |
| - cluster1: PipeConsensusStreamMode | |
| cluster2: LightWeightStandaloneMode | |
| - cluster1: PipeConsensusStreamMode | |
| cluster2: HighPerformanceMode | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2DualTreeManual \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| SubscriptionTreeArchVerification: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster1: | |
| [ | |
| ScalableSingleNodeMode, | |
| PipeConsensusBatchMode, | |
| PipeConsensusStreamMode, | |
| ] | |
| cluster2: [ScalableSingleNodeMode] | |
| os: [ubuntu-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2SubscriptionTreeArchVerification \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cluster-log-subscription-tree-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| SubscriptionTableArchVerification: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster1: [ScalableSingleNodeMode] | |
| cluster2: [ScalableSingleNodeMode] | |
| os: [ubuntu-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2SubscriptionTableArchVerification \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cluster-log-subscription-table-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| SubscriptionTreeRegressionConsumer: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal | |
| cluster1: | |
| [ | |
| ScalableSingleNodeMode, | |
| PipeConsensusBatchMode, | |
| PipeConsensusStreamMode, | |
| ] | |
| cluster2: [ScalableSingleNodeMode] | |
| os: [ubuntu-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2SubscriptionTreeRegressionConsumer \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cluster-log-subscription-tree-regression-consumer-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| SubscriptionTreeRegressionMisc: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal | |
| cluster1: | |
| [ | |
| ScalableSingleNodeMode, | |
| PipeConsensusBatchMode, | |
| PipeConsensusStreamMode, | |
| ] | |
| cluster2: [ScalableSingleNodeMode] | |
| os: [ubuntu-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2SubscriptionTreeRegressionMisc \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cluster-log-subscription-tree-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| PipeDualTableManualBasic: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster: | |
| [ | |
| LightWeightStandaloneMode, | |
| ScalableSingleNodeMode, | |
| HighPerformanceMode, | |
| PipeConsensusBatchMode, | |
| PipeConsensusStreamMode, | |
| ] | |
| os: [ubuntu-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2DualTableManualBasic \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 | |
| PipeDualTableManualEnhanced: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| java: [17] | |
| # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. | |
| cluster: | |
| [ | |
| LightWeightStandaloneMode, | |
| ScalableSingleNodeMode, | |
| HighPerformanceMode, | |
| PipeConsensusBatchMode, | |
| PipeConsensusStreamMode, | |
| ] | |
| os: [ubuntu-latest] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK ${{ matrix.java }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: corretto | |
| java-version: ${{ matrix.java }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cache Maven packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2- | |
| - name: Sleep for a random duration between 0 and 10000 milliseconds | |
| run: | | |
| sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) | |
| - name: IT Test | |
| shell: bash | |
| # we do not compile client-cpp for saving time, it is tested in client.yml | |
| # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml | |
| run: | | |
| retry() { | |
| local -i max_attempts=3 | |
| local -i attempt=1 | |
| local -i retry_sleep=5 | |
| local test_output | |
| while [ $attempt -le $max_attempts ]; do | |
| mvn clean verify \ | |
| -P with-integration-tests \ | |
| -DskipUTs \ | |
| -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ | |
| -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ | |
| -pl integration-test \ | |
| -am -PMultiClusterIT2DualTableManualEnhanced \ | |
| -ntp >> ~/run-tests-$attempt.log && return 0 | |
| test_output=$(cat ~/run-tests-$attempt.log) | |
| echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" | |
| echo "$test_output" | |
| echo "==================== END: ~/run-tests-$attempt.log ======================" | |
| if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then | |
| echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..." | |
| fi | |
| if echo "$test_output" | grep -q "Could not transfer artifact"; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." | |
| sleep $retry_sleep | |
| attempt=$((attempt + 1)) | |
| else | |
| echo "Test failed after $max_attempts attempts due to artifact transfer issue." | |
| echo "Treating this as a success because the issue is likely transient." | |
| return 0 | |
| fi | |
| elif [ $? -ne 0 ]; then | |
| echo "Test failed with a different error." | |
| return 1 | |
| else | |
| echo "Tests passed" | |
| return 0 | |
| fi | |
| done | |
| } | |
| retry | |
| - name: Upload Artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} | |
| path: integration-test/target/cluster-logs | |
| retention-days: 30 |