Merge branch 'master' into fix_03355_join_to_in_optimization

Author: shankar-iyer

.github/workflows/create_release.yml

@@ -149,7 +149,7 @@ jobs:
           git diff HEAD
       - name: Create ChangeLog PR
         if: ${{ inputs.type == 'patch' && ! inputs.dry-run && ! inputs.only-repo && ! inputs.only-docker }}
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6
         with:
           author: "robot-clickhouse <[email protected]>"
           token: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}

.github/workflows/master.yml

@@ -1611,6 +1611,44 @@ jobs:
             python3 -m praktika run 'Stateless tests (release, ParallelReplicas, s3 storage)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
           fi
 
+  stateless_tests_debug_asyncinsert_s3_storage:
+    runs-on: [self-hosted, func-tester]
+    needs: [config_workflow, dockers_build_amd_and_merge, build_amd_debug]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChkZWJ1ZywgQXN5bmNJbnNlcnQsIHMzIHN0b3JhZ2Up') }}
+    name: "Stateless tests (debug, AsyncInsert, s3 storage)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
+          mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_config_masterci.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run 'Stateless tests (debug, AsyncInsert, s3 storage)' --workflow "MasterCI" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
+          else
+            python3 -m praktika run 'Stateless tests (debug, AsyncInsert, s3 storage)' --workflow "MasterCI" --ci |& tee ./ci/tmp/job.log
+          fi
+
   stateless_tests_debug:
     runs-on: [self-hosted, func-tester]
     needs: [config_workflow, dockers_build_amd_and_merge, build_amd_debug]

.github/workflows/pull_request.yml

@@ -1765,6 +1765,44 @@ jobs:
             python3 -m praktika run 'Stateless tests (release, ParallelReplicas, s3 storage)' --workflow "PR" --ci |& tee ./ci/tmp/job.log
           fi
 
+  stateless_tests_debug_asyncinsert_s3_storage:
+    runs-on: [self-hosted, func-tester]
+    needs: [config_workflow, dockers_build_amd_and_merge, build_amd_debug]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChkZWJ1ZywgQXN5bmNJbnNlcnQsIHMzIHN0b3JhZ2Up') }}
+    name: "Stateless tests (debug, AsyncInsert, s3 storage)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
+          mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_config_pr.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run 'Stateless tests (debug, AsyncInsert, s3 storage)' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
+          else
+            python3 -m praktika run 'Stateless tests (debug, AsyncInsert, s3 storage)' --workflow "PR" --ci |& tee ./ci/tmp/job.log
+          fi
+
   stateless_tests_debug:
     runs-on: [self-hosted, func-tester]
     needs: [config_workflow, dockers_build_amd_and_merge, build_amd_debug]
@@ -4351,7 +4389,7 @@ jobs:
 
   finish_workflow:
     runs-on: [self-hosted, style-checker-aarch64]
-    needs: [config_workflow, dockers_build_arm, dockers_build_amd_and_merge, style_check, docs_check, fast_test, build_amd_tidy, build_arm_tidy, build_amd_debug, build_amd_release, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_release, build_arm_asan, build_arm_coverage, build_arm_binary, build_amd_darwin, build_arm_darwin, build_arm_v80compat, build_amd_freebsd, build_ppc64le, build_amd_compat, build_amd_musl, build_riscv64, build_s390x, build_loongarch64, build_fuzzers, unit_tests_asan, unit_tests_tsan, unit_tests_msan, unit_tests_ubsan, docker_server_image, docker_keeper_image, install_packages_release, install_packages_aarch64, compatibility_check_release, compatibility_check_aarch64, stateless_tests_asan_distributed_plan_1_2, stateless_tests_asan_distributed_plan_2_2, stateless_tests_release, stateless_tests_release_old_analyzer_s3_databasereplicated_1_2, stateless_tests_release_old_analyzer_s3_databasereplicated_2_2, stateless_tests_release_parallelreplicas_s3_storage, stateless_tests_debug, stateless_tests_tsan_1_3, stateless_tests_tsan_2_3, stateless_tests_tsan_3_3, stateless_tests_msan_1_4, stateless_tests_msan_2_4, stateless_tests_msan_3_4, stateless_tests_msan_4_4, stateless_tests_ubsan, stateless_tests_debug_distributed_plan_s3_storage, stateless_tests_tsan_s3_storage_1_3, stateless_tests_tsan_s3_storage_2_3, stateless_tests_tsan_s3_storage_3_3, stateless_tests_aarch64, stateless_tests_coverage_1_6, stateless_tests_coverage_2_6, stateless_tests_coverage_3_6, stateless_tests_coverage_4_6, stateless_tests_coverage_5_6, stateless_tests_coverage_6_6, bugfix_validation, stateless_tests_asan_flaky_check, integration_tests_asan_old_analyzer_1_6, integration_tests_asan_old_analyzer_2_6, integration_tests_asan_old_analyzer_3_6, integration_tests_asan_old_analyzer_4_6, integration_tests_asan_old_analyzer_5_6, integration_tests_asan_old_analyzer_6_6, integration_tests_release_1_4, integration_tests_release_2_4, integration_tests_release_3_4, integration_tests_release_4_4, integration_tests_aarch64_distributed_plan_1_4, integration_tests_aarch64_distributed_plan_2_4, integration_tests_aarch64_distributed_plan_3_4, integration_tests_aarch64_distributed_plan_4_4, integration_tests_tsan_1_6, integration_tests_tsan_2_6, integration_tests_tsan_3_6, integration_tests_tsan_4_6, integration_tests_tsan_5_6, integration_tests_tsan_6_6, integration_tests_asan_flaky_check, stress_test_amd_debug, stress_test_amd_tsan, stress_test_arm_asan, stress_test_amd_ubsan, stress_test_amd_msan, upgrade_check_arm_asan, upgrade_check_amd_tsan, upgrade_check_amd_msan, upgrade_check_amd_debug, ast_fuzzer_amd_debug, ast_fuzzer_arm_asan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_msan, ast_fuzzer_amd_ubsan, buzzhouse_amd_debug, buzzhouse_arm_asan, buzzhouse_amd_tsan, buzzhouse_amd_msan, buzzhouse_amd_ubsan, performance_comparison_amd_release_master_head_1_3, performance_comparison_amd_release_master_head_2_3, performance_comparison_amd_release_master_head_3_3, performance_comparison_arm_release_master_head_1_3, performance_comparison_arm_release_master_head_2_3, performance_comparison_arm_release_master_head_3_3]
+    needs: [config_workflow, dockers_build_arm, dockers_build_amd_and_merge, style_check, docs_check, fast_test, build_amd_tidy, build_arm_tidy, build_amd_debug, build_amd_release, build_amd_asan, build_amd_tsan, build_amd_msan, build_amd_ubsan, build_amd_binary, build_arm_release, build_arm_asan, build_arm_coverage, build_arm_binary, build_amd_darwin, build_arm_darwin, build_arm_v80compat, build_amd_freebsd, build_ppc64le, build_amd_compat, build_amd_musl, build_riscv64, build_s390x, build_loongarch64, build_fuzzers, unit_tests_asan, unit_tests_tsan, unit_tests_msan, unit_tests_ubsan, docker_server_image, docker_keeper_image, install_packages_release, install_packages_aarch64, compatibility_check_release, compatibility_check_aarch64, stateless_tests_asan_distributed_plan_1_2, stateless_tests_asan_distributed_plan_2_2, stateless_tests_release, stateless_tests_release_old_analyzer_s3_databasereplicated_1_2, stateless_tests_release_old_analyzer_s3_databasereplicated_2_2, stateless_tests_release_parallelreplicas_s3_storage, stateless_tests_debug_asyncinsert_s3_storage, stateless_tests_debug, stateless_tests_tsan_1_3, stateless_tests_tsan_2_3, stateless_tests_tsan_3_3, stateless_tests_msan_1_4, stateless_tests_msan_2_4, stateless_tests_msan_3_4, stateless_tests_msan_4_4, stateless_tests_ubsan, stateless_tests_debug_distributed_plan_s3_storage, stateless_tests_tsan_s3_storage_1_3, stateless_tests_tsan_s3_storage_2_3, stateless_tests_tsan_s3_storage_3_3, stateless_tests_aarch64, stateless_tests_coverage_1_6, stateless_tests_coverage_2_6, stateless_tests_coverage_3_6, stateless_tests_coverage_4_6, stateless_tests_coverage_5_6, stateless_tests_coverage_6_6, bugfix_validation, stateless_tests_asan_flaky_check, integration_tests_asan_old_analyzer_1_6, integration_tests_asan_old_analyzer_2_6, integration_tests_asan_old_analyzer_3_6, integration_tests_asan_old_analyzer_4_6, integration_tests_asan_old_analyzer_5_6, integration_tests_asan_old_analyzer_6_6, integration_tests_release_1_4, integration_tests_release_2_4, integration_tests_release_3_4, integration_tests_release_4_4, integration_tests_aarch64_distributed_plan_1_4, integration_tests_aarch64_distributed_plan_2_4, integration_tests_aarch64_distributed_plan_3_4, integration_tests_aarch64_distributed_plan_4_4, integration_tests_tsan_1_6, integration_tests_tsan_2_6, integration_tests_tsan_3_6, integration_tests_tsan_4_6, integration_tests_tsan_5_6, integration_tests_tsan_6_6, integration_tests_asan_flaky_check, stress_test_amd_debug, stress_test_amd_tsan, stress_test_arm_asan, stress_test_amd_ubsan, stress_test_amd_msan, upgrade_check_arm_asan, upgrade_check_amd_tsan, upgrade_check_amd_msan, upgrade_check_amd_debug, ast_fuzzer_amd_debug, ast_fuzzer_arm_asan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_msan, ast_fuzzer_amd_ubsan, buzzhouse_amd_debug, buzzhouse_arm_asan, buzzhouse_amd_tsan, buzzhouse_amd_msan, buzzhouse_amd_ubsan, performance_comparison_amd_release_master_head_1_3, performance_comparison_amd_release_master_head_2_3, performance_comparison_amd_release_master_head_3_3, performance_comparison_arm_release_master_head_1_3, performance_comparison_arm_release_master_head_2_3, performance_comparison_arm_release_master_head_3_3]
     if: ${{ !cancelled() }}
     name: "Finish Workflow"
     outputs:

CMakeLists.txt

@@ -613,9 +613,11 @@ endif()
 
 include (cmake/sanitize_targets.cmake)
 
-if (COMPILER_CACHE STREQUAL "chcache")
+if (COMPILER_CACHE STREQUAL "chcache" AND CHCACHE_EXECUTABLE_PATH STREQUAL "")
+    message(STATUS "Adding chcache as dependency to all other targets")
     get_all_targets(all_targets)
-    set (chcache_targets _cargo-build_chcache cargo-build_chcache cargo-prebuild_chcache)
+    set(chcache_targets _cargo-build_chcache cargo-build_chcache cargo-prebuild_chcache)
+
     foreach(target ${all_targets})
         if (target IN_LIST chcache_targets)
             continue()

README.md

@@ -52,6 +52,7 @@ Upcoming meetups
 * [Istanbul Meetup](https://www.meetup.com/clickhouse-turkiye-meetup-group/events/306978337/) - May 15, 2025
 * [Shenzhen Meetup](https://www.huodongxing.com/event/7803892350511) - May 17, 2025
 * [Seattle Meetup](https://www.meetup.com/clickhouse-seattle-user-group/events/307622716/) - May 20, 2025
+* [Tokyo Meetup](https://www.meetup.com/clickhouse-tokyo-user-group/events/307689645/) - June 5, 2025
 * [Washington DC Meetup](https://www.meetup.com/clickhouse-dc-user-group/events/307622954/) - June 12, 2025
 * [Atlanta Meetup](https://www.meetup.com/clickhouse-atlanta-meetup-group/events/307627590/) - July 8, 2025
 * [New York Meetup](https://www.meetup.com/clickhouse-new-york-user-group/events/307627675/) - July 15, 2025

ci/defs/job_configs.py

@@ -336,6 +336,7 @@ class JobConfigs:
             "release, old analyzer, s3, DatabaseReplicated, 1/2",
             "release, old analyzer, s3, DatabaseReplicated, 2/2",
             "release, ParallelReplicas, s3 storage",
+            "debug, AsyncInsert, s3 storage",
         ],
         runs_on=[
             RunnerLabels.FUNC_TESTER_AMD,
@@ -344,6 +345,7 @@ class JobConfigs:
             RunnerLabels.FUNC_TESTER_AMD,
             RunnerLabels.FUNC_TESTER_AMD,
             RunnerLabels.FUNC_TESTER_AMD,
+            RunnerLabels.FUNC_TESTER_AMD,
         ],
         requires=[
             ["Build (amd_asan)"],
@@ -352,6 +354,7 @@ class JobConfigs:
             ["Build (amd_release)"],
             ["Build (amd_release)"],
             ["Build (amd_release)"],
+            ["Build (amd_debug)"],
         ],
     )
     functional_tests_jobs_coverage = Job.Config(

ci/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt

@@ -167,6 +167,7 @@ CityHash
 Clangd
 ClickBench
 ClickCat
+ClickHaskell
 ClickHouse
 ClickHouse's
 ClickHouseClient

cmake/ccache.cmake

@@ -2,6 +2,8 @@
 
 include(cmake/utils.cmake)
 
+set(CHCACHE_EXECUTABLE_PATH "" CACHE STRING "Path to chcache executable to use. If the compiler cache is set to use chcache, chcache will be used from here instead of building it.")
+
 # Defensive programming: early return to avoid configuring any cache after we've set dummy launchers.
 # If something includes this file by mistake after the first setup, it'd override the dummy launchers.
 if(USING_DUMMY_LAUNCHERS)
@@ -22,12 +24,18 @@ set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the cachin
 
 if(COMPILER_CACHE STREQUAL "auto")
     find_program (CCACHE_EXECUTABLE NAMES sccache ccache)
-elseif (COMPILER_CACHE STREQUAL "ccache")
+elseif(COMPILER_CACHE STREQUAL "ccache")
     find_program (CCACHE_EXECUTABLE ccache)
 elseif(COMPILER_CACHE STREQUAL "sccache")
     find_program (CCACHE_EXECUTABLE sccache)
 elseif(COMPILER_CACHE STREQUAL "chcache")
-    set(CCACHE_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/rust/chcache/chcache)
+    if(CHCACHE_EXECUTABLE_PATH STREQUAL "")
+        message(STATUS "Using self-built chcache")
+        set(CCACHE_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/rust/chcache/chcache)
+    else()
+        message(STATUS "Using already built chcache from ${CHCACHE_EXECUTABLE_PATH}")
+        set(CCACHE_EXECUTABLE ${CHCACHE_EXECUTABLE_PATH})
+    endif()
 elseif(COMPILER_CACHE STREQUAL "disabled")
     message(STATUS "Using *ccache: no (disabled via configuration)")
     return()

docker/keeper/Dockerfile

@@ -38,7 +38,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="25.4.2.31"
+ARG VERSION="25.4.3.22"
 ARG PACKAGES="clickhouse-keeper"
 ARG DIRECT_DOWNLOAD_URLS=""
 

docker/server/Dockerfile.alpine

@@ -35,7 +35,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="25.4.2.31"
+ARG VERSION="25.4.3.22"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 ARG DIRECT_DOWNLOAD_URLS=""