Merge branch 'master' into analyzer-rewrite-view-query

Author: yakov-olkhovskiy

CMakeLists.txt

@@ -204,11 +204,6 @@ option(OMIT_HEAVY_DEBUG_SYMBOLS
     ${OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT})
 
 option(BUILD_STANDALONE_KEEPER "Build keeper as small standalone binary" OFF)
-if (NOT BUILD_STANDALONE_KEEPER)
-    option(CREATE_KEEPER_SYMLINK "Create symlink for clickhouse-keeper to main server binary" ON)
-else ()
-    option(CREATE_KEEPER_SYMLINK "Create symlink for clickhouse-keeper to main server binary" OFF)
-endif ()
 
 # Create BuildID when using lld. For other linkers it is created by default.
 # (NOTE: LINKER_NAME can be either path or name, and in different variants)

README.md

@@ -47,15 +47,16 @@ Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com.
 You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
 
 Upcoming meetups
-* [Tokyo Meetup with Confluent](https://www.meetup.com/clickhouse-tokyo-user-group/events/306832118/) - April 15, 2025
-* [Kuala Lumper Meetup with CNCF](https://www.meetup.com/clickhouse-malaysia-meetup-group/events/306697678/) - April 16, 2025
 * [Jakarta Meetup with AWS](https://www.meetup.com/clickhouse-indonesia-user-group/events/306973747/) - April 22, 2025
 * [Denver Meetup](https://www.meetup.com/clickhouse-denver-user-group/events/306934991/) - April 23, 2025
 * [Austin Meetup](https://www.meetup.com/clickhouse-austin-user-group/events/307289908) - May 13, 2025
 * [London Meetup](https://www.meetup.com/clickhouse-london-user-group/events/306047172/) - May 14, 2025
 * [Istanbul Meetup](https://www.meetup.com/clickhouse-turkiye-meetup-group/events/306978337/) - May 15, 2025
+* [Shenzhen Meetup](hhttps://www.huodongxing.com/event/7803892350511) - May 17, 2025
 
 Recent meetups
+* [Kuala Lumper Meetup with CNCF](https://www.meetup.com/clickhouse-malaysia-meetup-group/events/306697678/) - April 16, 2025
+* [Tokyo Meetup with Confluent](https://www.meetup.com/clickhouse-tokyo-user-group/events/306832118/) - April 15, 2025
 * [Ho Chi Minh Meetup with AWS & Infinite Lambda](https://www.meetup.com/clickhouse-vietnam-meetup-group/events/306810105/) - April 12, 2025
 * [Oslo Meetup](https://www.meetup.com/open-source-real-time-data-warehouse-real-time-analytics/events/306414327/) - April 8, 2025
 * [Sydney Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/306549810/) - April 1, 2025

ci/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt

@@ -1887,8 +1887,10 @@ geospatial
 getClientHTTPHeader
 getMacro
 getMaxTableNameLengthForDatabase
+getMergeTreeSetting
 getOSKernelVersion
 getServerPort
+getServerSetting
 getSetting
 getSettingOrDefault
 getSizeOfEnumType
@@ -2233,6 +2235,7 @@ mebibytes
 memtable
 memtables
 mergeTreeIndex
+mergeTreeProjection
 mergeable
 mergetree
 messageID

ci/jobs/scripts/workflow_hooks/filter_job.py

@@ -83,17 +83,18 @@ def should_skip_job(job_name):
             f"Skipped, labeled with '{Labels.CI_FUNCTIONAL_FLAKY}' - run stateless test jobs only",
         )
 
-    if Labels.CI_INTEGRATION in _info_cache.pr_labels and (
+    if Labels.CI_INTEGRATION in _info_cache.pr_labels and not (
         job_name.startswith(JobNames.INTEGRATION) or job_name in JobConfigs.builds_for_tests
     ):
         return (
             True,
             f"Skipped, labeled with '{Labels.CI_INTEGRATION}' - run integration test jobs only",
         )
 
-    if Labels.CI_FUNCTIONAL in _info_cache.pr_labels and (
+    if Labels.CI_FUNCTIONAL in _info_cache.pr_labels and not (
         job_name.startswith(JobNames.STATELESS)
-        or job_name.startswith(JobNames.STATEFUL or job_name in JobConfigs.builds_for_tests)
+        or job_name.startswith(JobNames.STATEFUL)
+        or job_name in JobConfigs.builds_for_tests
     ):
         return (
             True,

cmake/darwin/default_libs.cmake

@@ -15,5 +15,18 @@ set(CMAKE_OSX_DEPLOYMENT_TARGET 10.15)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
+if(NOT CMAKE_CROSSCOMPILING)
+    execute_process(
+        COMMAND xcrun --sdk macosx --show-sdk-version
+        OUTPUT_VARIABLE OS_DARWIN_SDK_VERSION
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    if(${OS_DARWIN_SDK_VERSION} MATCHES "^[0-9]+\\.[0-9]+")
+        message(STATUS "Detected OSX SDK Version: ${OS_DARWIN_SDK_VERSION}")
+    else ()
+        message(WARNING "Unexpected OSX SDK Version: ${OS_DARWIN_SDK_VERSION}")
+    endif()
+endif()
+
 include (cmake/unwind.cmake)
-include (cmake/cxx.cmake)
+include (cmake/cxx.cmake)

contrib/arrow

@@ -79,6 +79,13 @@ if (OS_DARWIN OR OS_FREEBSD OR USE_MUSL)
     target_compile_definitions(_libpq PRIVATE -DSTRERROR_R_INT=1)
 endif()
 
+# Mac 15.4 / Xcode 16.3 started defining srtchrnul
+# Could be cleaned up if properly resolved in upstream
+# https://www.postgresql.org/message-id/flat/385134.1743523038%40sss.pgh.pa.us
+if (OS_DARWIN AND OS_DARWIN_SDK_VERSION AND OS_DARWIN_SDK_VERSION VERSION_GREATER_EQUAL 15.4)
+    target_compile_definitions(_libpq PRIVATE -DHAVE_STRCHRNUL)
+endif()
+
 target_link_libraries (_libpq PRIVATE OpenSSL::SSL)
 
 add_library(ch_contrib::libpq ALIAS _libpq)

contrib/postgres-cmake/CMakeLists.txt

@@ -63,18 +63,17 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
 COPY requirements.txt /
 RUN python3 -m pip install --no-cache-dir -r requirements.txt
 
-# Hudi supports only spark 3.3.*, not 3.4
-RUN curl -fsSL -O https://archive.apache.org/dist/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz \
-    && tar xzvf spark-3.3.2-bin-hadoop3.tgz -C / \
-    && rm spark-3.3.2-bin-hadoop3.tgz
+RUN curl -fsSL -O https://archive.apache.org/dist/spark/spark-3.5.5/spark-3.5.5-bin-hadoop3.tgz \
+    && tar xzvf spark-3.5.5-bin-hadoop3.tgz -C / \
+    && rm spark-3.5.5-bin-hadoop3.tgz
 
 # download spark and packages
 # if you change packages, don't forget to update them in tests/integration/helpers/cluster.py
-RUN packages="org.apache.hudi:hudi-spark3.3-bundle_2.12:0.13.0,\
-io.delta:delta-core_2.12:2.3.0,\
-org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.1.0" \
-    && /spark-3.3.2-bin-hadoop3/bin/spark-shell --packages "$packages" > /dev/null \
-    && find /root/.ivy2/ -name '*.jar' -exec ln -sf {} /spark-3.3.2-bin-hadoop3/jars/ \;
+RUN packages="io.delta:delta-spark_2.12:3.1.0,\
+org.apache.hudi:hudi-spark3.5-bundle_2.12:1.0.1,\
+org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.4.3" \
+    && /spark-3.5.5-bin-hadoop3/bin/spark-shell --packages "$packages" > /dev/null \
+    && find /root/.ivy2/ -name '*.jar' -exec ln -sf {} /spark-3.5.5-bin-hadoop3/jars/ \;
 
 RUN set -x \
   && addgroup --system dockremap \

contrib/zstd

@@ -54,7 +54,7 @@ set -e
 
 java_path="$(update-alternatives --config java | sed -n 's/.*(providing \/usr\/bin\/java): //p')"
 export JAVA_PATH=$java_path
-export SPARK_HOME="/spark-3.3.2-bin-hadoop3"
+export SPARK_HOME="/spark-3.5.5-bin-hadoop3"
 export PATH=$SPARK_HOME/bin:$PATH
 export JAVA_TOOL_OPTIONS="-Djdk.attach.allowAttachSelf=true"