Merge pull request ClickHouse#75379 from ClickHouse/add-async-inserts-job

Tests: add test job with enabled async inserts

Author: fm4v

.github/workflows/master.yml

@@ -1611,6 +1611,44 @@ jobs:
             python3 -m praktika run 'Stateless tests (release, ParallelReplicas, s3 storage)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
           fi
 
+  stateless_tests_debug_asyncinsert_s3_storage:
+    runs-on: [self-hosted, func-tester]
+    needs: [config_workflow, dockers_build_amd_and_merge, build_amd_debug]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChkZWJ1ZywgQXN5bmNJbnNlcnQsIHMzIHN0b3JhZ2Up') }}
+    name: "Stateless tests (debug, AsyncInsert, s3 storage)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
+          mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_config_masterci.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run 'Stateless tests (debug, AsyncInsert, s3 storage)' --workflow "MasterCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
+          else
+            python3 -m praktika run 'Stateless tests (debug, AsyncInsert, s3 storage)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
+          fi
+
   stateless_tests_debug:
     runs-on: [self-hosted, func-tester]
     needs: [config_workflow, dockers_build_amd_and_merge, build_amd_debug]

.github/workflows/pull_request.yml

@@ -1765,6 +1765,44 @@ jobs:
             python3 -m praktika run 'Stateless tests (release, ParallelReplicas, s3 storage)' --workflow "PR" --ci |& tee ./ci/tmp/job.log
           fi
 
+  stateless_tests_debug_asyncinsert_s3_storage:
+    runs-on: [self-hosted, func-tester]
+    needs: [config_workflow, dockers_build_amd_and_merge, build_amd_debug]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChkZWJ1ZywgQXN5bmNJbnNlcnQsIHMzIHN0b3JhZ2Up') }}
+    name: "Stateless tests (debug, AsyncInsert, s3 storage)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
+          mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_config_pr.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run 'Stateless tests (debug, AsyncInsert, s3 storage)' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
+          else
+            python3 -m praktika run 'Stateless tests (debug, AsyncInsert, s3 storage)' --workflow "PR" --ci |& tee ./ci/tmp/job.log
+          fi
+
   stateless_tests_debug:
     runs-on: [self-hosted, func-tester]
     needs: [config_workflow, dockers_build_amd_and_merge, build_amd_debug]
@@ -4351,7 +4389,7 @@ jobs:
 
   finish_workflow:
     runs-on: [self-hosted, style-checker-aarch64]
-    needs: [config_workflow, dockers_build_arm, dockers_build_amd_and_merge, style_check, docs_check, fast_test, build_amd_tidy, build_arm_tidy, build_amd_debug, build_amd_release, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_release, build_arm_asan, build_arm_coverage, build_arm_binary, build_amd_darwin, build_arm_darwin, build_arm_v80compat, build_amd_freebsd, build_ppc64le, build_amd_compat, build_amd_musl, build_riscv64, build_s390x, build_loongarch64, build_fuzzers, unit_tests_asan, unit_tests_tsan, unit_tests_msan, unit_tests_ubsan, docker_server_image, docker_keeper_image, install_packages_release, install_packages_aarch64, compatibility_check_release, compatibility_check_aarch64, stateless_tests_asan_distributed_plan_1_2, stateless_tests_asan_distributed_plan_2_2, stateless_tests_release, stateless_tests_release_old_analyzer_s3_databasereplicated_1_2, stateless_tests_release_old_analyzer_s3_databasereplicated_2_2, stateless_tests_release_parallelreplicas_s3_storage, stateless_tests_debug, stateless_tests_tsan_1_3, stateless_tests_tsan_2_3, stateless_tests_tsan_3_3, stateless_tests_msan_1_4, stateless_tests_msan_2_4, stateless_tests_msan_3_4, stateless_tests_msan_4_4, stateless_tests_ubsan, stateless_tests_debug_distributed_plan_s3_storage, stateless_tests_tsan_s3_storage_1_3, stateless_tests_tsan_s3_storage_2_3, stateless_tests_tsan_s3_storage_3_3, stateless_tests_aarch64, stateless_tests_coverage_1_6, stateless_tests_coverage_2_6, stateless_tests_coverage_3_6, stateless_tests_coverage_4_6, stateless_tests_coverage_5_6, stateless_tests_coverage_6_6, bugfix_validation, stateless_tests_asan_flaky_check, integration_tests_asan_old_analyzer_1_6, integration_tests_asan_old_analyzer_2_6, integration_tests_asan_old_analyzer_3_6, integration_tests_asan_old_analyzer_4_6, integration_tests_asan_old_analyzer_5_6, integration_tests_asan_old_analyzer_6_6, integration_tests_release_1_4, integration_tests_release_2_4, integration_tests_release_3_4, integration_tests_release_4_4, integration_tests_aarch64_distributed_plan_1_4, integration_tests_aarch64_distributed_plan_2_4, integration_tests_aarch64_distributed_plan_3_4, integration_tests_aarch64_distributed_plan_4_4, integration_tests_tsan_1_6, integration_tests_tsan_2_6, integration_tests_tsan_3_6, integration_tests_tsan_4_6, integration_tests_tsan_5_6, integration_tests_tsan_6_6, integration_tests_asan_flaky_check, stress_test_amd_debug, stress_test_amd_tsan, stress_test_arm_asan, stress_test_amd_ubsan, stress_test_amd_msan, upgrade_check_arm_asan, upgrade_check_amd_tsan, upgrade_check_amd_msan, upgrade_check_amd_debug, ast_fuzzer_amd_debug, ast_fuzzer_arm_asan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_msan, ast_fuzzer_amd_ubsan, buzzhouse_amd_debug, buzzhouse_arm_asan, buzzhouse_amd_tsan, buzzhouse_amd_msan, buzzhouse_amd_ubsan, performance_comparison_amd_release_master_head_1_3, performance_comparison_amd_release_master_head_2_3, performance_comparison_amd_release_master_head_3_3, performance_comparison_arm_release_master_head_1_3, performance_comparison_arm_release_master_head_2_3, performance_comparison_arm_release_master_head_3_3]
+    needs: [config_workflow, dockers_build_arm, dockers_build_amd_and_merge, style_check, docs_check, fast_test, build_amd_tidy, build_arm_tidy, build_amd_debug, build_amd_release, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_release, build_arm_asan, build_arm_coverage, build_arm_binary, build_amd_darwin, build_arm_darwin, build_arm_v80compat, build_amd_freebsd, build_ppc64le, build_amd_compat, build_amd_musl, build_riscv64, build_s390x, build_loongarch64, build_fuzzers, unit_tests_asan, unit_tests_tsan, unit_tests_msan, unit_tests_ubsan, docker_server_image, docker_keeper_image, install_packages_release, install_packages_aarch64, compatibility_check_release, compatibility_check_aarch64, stateless_tests_asan_distributed_plan_1_2, stateless_tests_asan_distributed_plan_2_2, stateless_tests_release, stateless_tests_release_old_analyzer_s3_databasereplicated_1_2, stateless_tests_release_old_analyzer_s3_databasereplicated_2_2, stateless_tests_release_parallelreplicas_s3_storage, stateless_tests_debug_asyncinsert_s3_storage, stateless_tests_debug, stateless_tests_tsan_1_3, stateless_tests_tsan_2_3, stateless_tests_tsan_3_3, stateless_tests_msan_1_4, stateless_tests_msan_2_4, stateless_tests_msan_3_4, stateless_tests_msan_4_4, stateless_tests_ubsan, stateless_tests_debug_distributed_plan_s3_storage, stateless_tests_tsan_s3_storage_1_3, stateless_tests_tsan_s3_storage_2_3, stateless_tests_tsan_s3_storage_3_3, stateless_tests_aarch64, stateless_tests_coverage_1_6, stateless_tests_coverage_2_6, stateless_tests_coverage_3_6, stateless_tests_coverage_4_6, stateless_tests_coverage_5_6, stateless_tests_coverage_6_6, bugfix_validation, stateless_tests_asan_flaky_check, integration_tests_asan_old_analyzer_1_6, integration_tests_asan_old_analyzer_2_6, integration_tests_asan_old_analyzer_3_6, integration_tests_asan_old_analyzer_4_6, integration_tests_asan_old_analyzer_5_6, integration_tests_asan_old_analyzer_6_6, integration_tests_release_1_4, integration_tests_release_2_4, integration_tests_release_3_4, integration_tests_release_4_4, integration_tests_aarch64_distributed_plan_1_4, integration_tests_aarch64_distributed_plan_2_4, integration_tests_aarch64_distributed_plan_3_4, integration_tests_aarch64_distributed_plan_4_4, integration_tests_tsan_1_6, integration_tests_tsan_2_6, integration_tests_tsan_3_6, integration_tests_tsan_4_6, integration_tests_tsan_5_6, integration_tests_tsan_6_6, integration_tests_asan_flaky_check, stress_test_amd_debug, stress_test_amd_tsan, stress_test_arm_asan, stress_test_amd_ubsan, stress_test_amd_msan, upgrade_check_arm_asan, upgrade_check_amd_tsan, upgrade_check_amd_msan, upgrade_check_amd_debug, ast_fuzzer_amd_debug, ast_fuzzer_arm_asan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_msan, ast_fuzzer_amd_ubsan, buzzhouse_amd_debug, buzzhouse_arm_asan, buzzhouse_amd_tsan, buzzhouse_amd_msan, buzzhouse_amd_ubsan, performance_comparison_amd_release_master_head_1_3, performance_comparison_amd_release_master_head_2_3, performance_comparison_amd_release_master_head_3_3, performance_comparison_arm_release_master_head_1_3, performance_comparison_arm_release_master_head_2_3, performance_comparison_arm_release_master_head_3_3]
     if: ${{ !cancelled() }}
     name: "Finish Workflow"
     outputs:

ci/defs/job_configs.py

@@ -336,6 +336,7 @@ class JobConfigs:
             "release, old analyzer, s3, DatabaseReplicated, 1/2",
             "release, old analyzer, s3, DatabaseReplicated, 2/2",
             "release, ParallelReplicas, s3 storage",
+            "debug, AsyncInsert, s3 storage",
         ],
         runs_on=[
             RunnerLabels.FUNC_TESTER_AMD,
@@ -344,6 +345,7 @@ class JobConfigs:
             RunnerLabels.FUNC_TESTER_AMD,
             RunnerLabels.FUNC_TESTER_AMD,
             RunnerLabels.FUNC_TESTER_AMD,
+            RunnerLabels.FUNC_TESTER_AMD,
         ],
         requires=[
             ["Build (amd_asan)"],
@@ -352,6 +354,7 @@ class JobConfigs:
             ["Build (amd_release)"],
             ["Build (amd_release)"],
             ["Build (amd_release)"],
+            ["Build (amd_debug)"],
         ],
     )
     functional_tests_jobs_coverage = Job.Config(

tests/async_insert_blacklist.txt

@@ -0,0 +1,49 @@
+    Deduplication in dependent materialized view cannot work together with async inserts. Please disable either `deduplicate_blocks_in_dependent_materialized_views` or `async_insert` setting
+00633_materialized_view_and_too_many_parts_zookeeper
+02912_ingestion_mv_deduplication
+03008_deduplication_cases_from_docs
+03008_deduplication_insert_into_partitioned_table
+01927_query_views_log_matview_exceptions
+
+    !!!insert deduplication doesn't work
+02124_insert_deduplication_token_multiple_blocks
+02124_insert_deduplication_token_multiple_blocks_replica
+03008_deduplication_random_setttings
+
+    Cannot insert column value, because it is MATERIALIZED column
+00952_insert_into_distributed_with_materialized_column
+
+    !!!performance degradation (timeout)
+02046_low_cardinality_parallel_group_by
+03221_insert_timeout_overflow_mode
+
+    Not bugs
+
+    Transactions
+    Async inserts inside transactions are not supported. (NOT_IMPLEMENTED)
+01133_begin_commit_race
+01169_old_alter_partition_isolation_stress
+01170_alter_partition_isolation
+01171_mv_select_insert_isolation_long
+01172_transaction_counters
+01173_transaction_control_queries
+01174_select_insert_isolation
+02421_truncate_isolation_no_merges
+02421_truncate_isolation_with_mutations
+03167_transactions_are_really_disabled
+01167_isolation_hermitage
+
+    Async inserts with 'implicit_transaction' are not supported. (NOT_IMPLEMENTED)
+01168_mutations_isolation
+    Cannot execute query because current transaction failed. Expecting ROLLBACK statement. (INVALID_TRANSACTION
+01169_alter_partition_isolation_stress
+
+    Processing async inserts with both inlined and external data (from stdin or infile) is not supported. (NOT_IMPLEMENTED)
+02270_stdin_with_query_or_infile_data
+02435_rollback_cancelled_queries
+
+    https://github.com/ClickHouse/ClickHouse/issues/57768
+02423_insert_summary_behaviour
+
+    https://github.com/ClickHouse/ClickHouse/issues/79790
+02046_remote_table_function_named_collections

tests/ci/ci_cache.py

@@ -947,6 +947,7 @@ def apply(
         "Stateless tests (aarch64)": "deb6778b88",
         "Stateless tests (release, old analyzer, s3, DatabaseReplicated)": "deb6778b88",
         "Stateless tests (release, ParallelReplicas, s3 storage)": "deb6778b88",
+        "Stateless tests (debug, AsyncInsert, s3 storage)": "deb6778b88",
         "Stateless tests (debug, s3 storage)": "deb6778b88",
         "Stateless tests (tsan, s3 storage)": "deb6778b88",
         "Stress test (debug)": "aa298abf10",

tests/ci/ci_config.py

@@ -322,6 +322,9 @@ class CI:
         JobNames.STATELESS_TEST_PARALLEL_REPLICAS_REPLICATED_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=1
         ),
+        JobNames.STATELESS_TEST_ASYNC_INSERT_DEBUG: CommonJobConfigs.STATELESS_TEST.with_properties(
+            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=1
+        ),
         JobNames.STATELESS_TEST_S3_DEBUG_DISTRIBUTED_PLAN: CommonJobConfigs.STATELESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=1
         ),

tests/ci/ci_definitions.py

@@ -154,6 +154,9 @@ class JobNames(metaclass=WithIter):
     STATELESS_TEST_PARALLEL_REPLICAS_REPLICATED_RELEASE = (
         "Stateless tests (release, ParallelReplicas, s3 storage)"
     )
+    STATELESS_TEST_ASYNC_INSERT_DEBUG = (
+        "Stateless tests (debug, AsyncInsert, s3 storage)"
+    )
     STATELESS_TEST_S3_DEBUG_DISTRIBUTED_PLAN = (
         "Stateless tests (debug, distributed plan, s3 storage)"
     )

tests/ci/functional_test_check.py

@@ -62,6 +62,8 @@ def get_additional_envs(
         result.append("USE_POLYMORPHIC_PARTS=1")
     if "ParallelReplicas" in check_name:
         result.append("USE_PARALLEL_REPLICAS=1")
+    if "AsyncInsert" in check_name:
+        result.append("USE_ASYNC_INSERT=1")
     if "s3 storage" in check_name:
         result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1")
         result.append("RANDOMIZE_OBJECT_KEY_TYPE=1")

tests/ci/test_ci_options.py

@@ -78,6 +78,7 @@
     "Stateless tests (ubsan)",
     "Stateless tests (release, old analyzer, s3, DatabaseReplicated)",
     "Stateless tests (release, ParallelReplicas, s3 storage)"
+    "Stateless tests (debug, AsyncInsert, s3 storage)"
     "Stateless tests (debug, s3 storage)",
     "Stateless tests (tsan, s3 storage)",
     "Stateless tests (asan, flaky check)",

tests/clickhouse-test

@@ -910,6 +910,7 @@ class FailureReason(enum.Enum):
     AZURE_BLOB_STORAGE = "azure-blob-storage"
     BUILD = "not running for current build"
     NO_PARALLEL_REPLICAS = "smth in not supported with parallel replicas"
+    NO_ASYNC_INSERT = "test is not supported with async inserts enabled"
     SHARED_MERGE_TREE = "no-shared-merge-tree"
     DISTRIBUTED_CACHE = "distributed-cache"
     NO_STATELESS = "no-stateless"
@@ -1453,6 +1454,9 @@ class TestCase:
         if "no-parallel-replicas" in tags and args.no_parallel_replicas:
             return FailureReason.NO_PARALLEL_REPLICAS
 
+        if "no-async-insert" in tags and args.no_async_insert:
+            return FailureReason.NO_ASYNC_INSERT
+
         if args.skip and any(s in self.name for s in args.skip):
             return FailureReason.SKIP
 
@@ -1873,7 +1877,10 @@ class TestCase:
                 + database
                 + "_1"
             )
-            for env_to_param in ["CLICKHOUSE_USER_FILES", "CLICKHOUSE_USER_FILES_UNIQUE"]:
+            for env_to_param in [
+                "CLICKHOUSE_USER_FILES",
+                "CLICKHOUSE_USER_FILES_UNIQUE",
+            ]:
                 value = os.environ.get(env_to_param, None)
                 if value:
                     query_params += f" --param_{env_to_param}={value}"
@@ -3327,6 +3334,12 @@ def main(args):
         blacklist = try_get_skip_list(base_dir, "../parallel_replicas_blacklist.txt")
         blacklist_check.extend(blacklist)
 
+    if args.no_async_insert is True:
+        skip_list_async_inserts = try_get_skip_list(
+            base_dir, "../async_insert_blacklist.txt"
+        )
+        skip_list.extend(skip_list_async_inserts)
+
     for suite in sorted(os.listdir(base_dir), key=suite_key_func):
         if server_died.is_set():
             break
@@ -3804,6 +3817,12 @@ def parse_args():
         default=False,
         help="Do not include tests that are not supported with parallel replicas feature",
     )
+    parser.add_argument(
+        "--no-async-insert",
+        action="store_true",
+        default=False,
+        help="Do not include tests that are not supported with async insert feature",
+    )
     parser.add_argument(
         "--replace-replicated-with-shared",
         action="store_true",