ci: add stateful test for large data

Author: everpcpc

.github/actions/test_stateful_cluster_linux/action.yml

@@ -39,10 +39,10 @@ runs:
         aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data/ontime_200.parquet s3://testbucket/admin/data/ontime_200_v1.parquet
         aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data/iceberg s3://testbucket/iceberg_data --recursive
 
-    - name: Run Stateful Tests with Cluster mode (ubuntu-latest only)
+    - name: Run Stateful Tests with Cluster mode
       shell: bash
       run: |
-        ./scripts/ci/ci-run-stateful-tests-cluster-s3.sh
+        ./scripts/ci/ci-run-stateful-tests-cluster-minio.sh
 
     - name: Upload failure
       if: failure() || cancelled()

.github/actions/test_stateful_large_data/action.yml

@@ -0,0 +1,33 @@
+name: "Test Stateful Large Data"
+description: "Running stateful with large data"
+inputs:
+  profile:
+    description: "The profile for this test"
+    required: true
+    default: "debug"
+  target:
+    description: ""
+    required: true
+    default: "x86_64-unknown-linux-gnu"
+runs:
+  using: "composite"
+  steps:
+    - name: Download artifact
+      uses: ./.github/actions/artifact_download
+      with:
+        profile: ${{ inputs.profile }}
+        sha: ${{ github.sha }}
+        target: ${{ inputs.target }}
+
+    - name: Run Stateful Tests with Standalone mode
+      shell: bash
+      env:
+        TEST_ID: github-${{ github.run_id }}
+      run: |
+        ./scripts/ci/ci-run-stateful-tests-large-data-s3.sh
+
+    - name: Upload failure
+      if: failure() || cancelled()
+      uses: ./.github/actions/artifact_failure
+      with:
+        name: test-stateful-large-data

.github/actions/test_stateful_standalone_linux/action.yml

@@ -39,10 +39,10 @@ runs:
         aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data/ontime_200.parquet s3://testbucket/admin/data/ontime_200_v1.parquet
         aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data/iceberg s3://testbucket/iceberg_data --recursive
 
-    - name: Run Stateful Tests with Standalone mode (ubuntu-latest only)
+    - name: Run Stateful Tests with Standalone mode
       shell: bash
       run: |
-        ./scripts/ci/ci-run-stateful-tests-standalone-s3.sh
+        ./scripts/ci/ci-run-stateful-tests-standalone-minio.sh
 
     - name: Upload failure
       if: failure() || cancelled()

.github/workflows/dev-linux.yml

@@ -176,11 +176,10 @@ jobs:
         with:
           name: test-sqllogic-standalone-${{ matrix.handlers }}-${{ matrix.dirs }}
 
-
   test_sqllogic_management_mode_linux:
     timeout-minutes: 30
     name: test_sqllogic_${{ matrix.dirs }}_linux
-    runs-on: [ self-hosted, X64, Linux, dev ]
+    runs-on: [self-hosted, X64, Linux, dev]
     needs: build_gnu
     strategy:
       matrix:
@@ -223,7 +222,6 @@ jobs:
         with:
           name: test-sqllogic-cluster-${{ matrix.handlers }}-${{ matrix.dirs }}
 
-
   test_stateless_standalone_linux:
     timeout-minutes: 20
     runs-on: [self-hosted, X64, Linux, dev]
@@ -256,6 +254,15 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_stateful_cluster_linux
 
+  test_stateful_large_data:
+    timeout-minutes: 60
+    if: contains(github.event.pull_request.labels.*.name, 'ci-largedata')
+    runs-on: [self-hosted, X64, Linux, dev]
+    needs: build_gnu
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/test_stateful_large_data
+
   test_fuzz_standalone_linux:
     timeout-minutes: 10
     runs-on: [self-hosted, X64, Linux, dev]

.github/workflows/release.yml

@@ -516,7 +516,7 @@ jobs:
           asset_content_type: text/plain
 
   run_benchmark:
-    name: run benchmark
+    name: benchmark
     needs: [create_release, publish_linux]
     uses: ./.github/workflows/trusted-benchmark.yml
     secrets: inherit

…s/ci/ci-run-stateful-tests-cluster-s3.sh …i/ci-run-stateful-tests-cluster-minio.shscripts/ci/ci-run-stateful-tests-cluster-s3.sh renamed to scripts/ci/ci-run-stateful-tests-cluster-minio.sh scripts/ci/ci-run-stateful-tests-cluster-s3.sh renamed to scripts/ci/ci-run-stateful-tests-cluster-minio.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Copyright 2020-2021 The Databend Authors.
+# SPDX-License-Identifier: Apache-2.0.
+
+set -e
+
+TEST_ID=${TEST_ID:-$(date +%s)}
+
+echo "*************************************"
+echo "* Setting STORAGE_TYPE to S3.       *"
+echo "*                                   *"
+echo "* Please make sure that S3 backend  *"
+echo "* is ready, and configured properly.*"
+echo "*************************************"
+export STORAGE_TYPE=s3
+export STORAGE_S3_BUCKET=databend-ci
+export STORAGE_S3_REGION=us-east-2
+export STORAGE_S3_ROOT="stateful/${TEST_ID}"
+export STORAGE_ALLOW_INSECURE=true
+
+echo "Install dependencies"
+python3 -m pip install --quiet mysql-connector-python
+
+echo "calling test suite"
+echo "Starting standalone DatabendQuery(debug)"
+./scripts/ci/deploy/databend-query-standalone.sh
+
+SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
+cd "$SCRIPT_PATH/../../tests" || exit
+
+echo "Starting databend-test"
+./databend-test "$1" --mode 'standalone' --run-dir 4_stateful_large_data

scripts/ci/ci-run-stateful-tests-large-data-s3.sh

@@ -0,0 +1 @@
+10000

…i/ci-run-stateful-tests-standalone-s3.sh …i-run-stateful-tests-standalone-minio.shscripts/ci/ci-run-stateful-tests-standalone-s3.sh renamed to scripts/ci/ci-run-stateful-tests-standalone-minio.sh scripts/ci/ci-run-stateful-tests-standalone-s3.sh renamed to scripts/ci/ci-run-stateful-tests-standalone-minio.sh

@@ -0,0 +1,6 @@
+create table oracle (a int, b timestamp, c String) Engine = Random;
+create stage oracle;
+copy into @oracle from (select * from oracle limit 10000) file_format=(type=parquet) max_file_size=8;
+create table target (a int, b timestamp, c String);
+copy into target from @oracle file_format=(type=parquet);
+select count(*) from target;