Merge remote-tracking branch 'ClickHouse/master' into add_bech32_enc_dec

Author: rschu1ze

.gitmodules

@@ -296,9 +296,6 @@
 [submodule "contrib/aws-c-compression"]
 	path = contrib/aws-c-compression
 	url = https://github.com/awslabs/aws-c-compression
-[submodule "contrib/aws-s2n-tls"]
-	path = contrib/aws-s2n-tls
-	url = https://github.com/ClickHouse/s2n-tls
 [submodule "contrib/crc32-vpmsum"]
 	path = contrib/crc32-vpmsum
 	url = https://github.com/antonblanchard/crc32-vpmsum.git

CHANGELOG.md

@@ -41,7 +41,7 @@
 * Gives the possibility to truncate specific tables from a database, filtered with the `LIKE` keyword. [#78597](https://github.com/ClickHouse/ClickHouse/pull/78597) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
 * Support `_part_starting_offset` virtual column in `MergeTree`-family tables. This column represents the cumulative row count of all preceding parts, calculated at query time based on the current part list. The cumulative values are retained throughout query execution and remain effective even after part pruning. Related internal logic has been refactored to support this behavior. [#79417](https://github.com/ClickHouse/ClickHouse/pull/79417) ([Amos Bird](https://github.com/amosbird)).
 * Add functions `divideOrNull`,`moduloOrNull`, `intDivOrNull`,`positiveModuloOrNull` to return NULL when right argument is zero. [#78276](https://github.com/ClickHouse/ClickHouse/pull/78276) ([kevinyhzou](https://github.com/KevinyhZou)).
-* TODO: WTF is that? Explain it in a way your gradma will understand. Support for Iceberg partition pruning bucket transform. [#79262](https://github.com/ClickHouse/ClickHouse/pull/79262) ([Daniil Ivanik](https://github.com/divanik)).
+* Add [`icebergHash`](https://iceberg.apache.org/spec/#appendix-b-32-bit-hash-requirements) and [`icebergBucketTransform`](https://iceberg.apache.org/spec/#bucket-transform-details) functions. Support data files pruning in `Iceberg` tables partitioned with [`bucket transfom`](https://iceberg.apache.org/spec/#partitioning). [#79262](https://github.com/ClickHouse/ClickHouse/pull/79262) ([Daniil Ivanik](https://github.com/divanik)).
 
 #### Experimental Feature
 * Hive metastore catalog for Iceberg datalake. [#77677](https://github.com/ClickHouse/ClickHouse/pull/77677) ([scanhex12](https://github.com/scanhex12)).

base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h

@@ -236,21 +236,16 @@ namespace Net
         /// to be able to re-use it again.
 
     private:
-        using MutexT = Poco::FastMutex;
-        using LockT = MutexT::ScopedLock;
-        using UnLockT = Poco::ScopedLockWithUnlock<MutexT>;
-
         SecureSocketImpl(const SecureSocketImpl &);
         SecureSocketImpl & operator=(const SecureSocketImpl &);
 
         mutable std::recursive_mutex _mutex;
-        std::atomic<SSL *> _pSSL;
+        SSL * _pSSL; // GUARDED_BY _mutex
         Poco::AutoPtr<SocketImpl> _pSocket;
         Context::Ptr _pContext;
         bool _needHandshake;
         std::string _peerHostName;
         Session::Ptr _pSession;
-        mutable MutexT _ssl_mutex;
 
         friend class SecureStreamSocketImpl;
 

base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp

@@ -103,8 +103,6 @@ void SecureSocketImpl::acceptSSL()
 	std::lock_guard<std::recursive_mutex> lock(_mutex);
 	poco_assert (!_pSSL);
 
-	LockT l(_ssl_mutex);
-
 	BIO* pBIO = BIO_new(BIO_s_socket());
 	if (!pBIO) throw SSLException("Cannot create BIO object");
 	BIO_set_fd(pBIO, static_cast<int>(_pSocket->sockfd()), BIO_NOCLOSE);
@@ -171,8 +169,6 @@ void SecureSocketImpl::connectSSL(bool performHandshake)
 	poco_assert (!_pSSL);
 	poco_assert (_pSocket->initialized());
 
-	LockT l(_ssl_mutex);
-
 	BIO* pBIO = BIO_new(BIO_s_socket());
 	if (!pBIO) throw SSLException("Cannot create SSL BIO object");
 	BIO_set_fd(pBIO, static_cast<int>(_pSocket->sockfd()), BIO_NOCLOSE);
@@ -250,8 +246,6 @@ void SecureSocketImpl::shutdown()
 	std::lock_guard<std::recursive_mutex> lock(_mutex);
 	if (_pSSL)
 	{
-        UnLockT l(_ssl_mutex);
-
         // Don't shut down the socket more than once.
         int shutdownState = SSL_get_shutdown(_pSSL);
         bool shutdownSent = (shutdownState & SSL_SENT_SHUTDOWN) == SSL_SENT_SHUTDOWN;
@@ -266,7 +260,6 @@ void SecureSocketImpl::shutdown()
 			// done with it.
 			int rc = SSL_shutdown(_pSSL);
 			if (rc < 0) handleError(rc);
-			l.unlock();
 			if (_pSocket->getBlocking())
 			{
 				_pSocket->shutdown();
@@ -297,9 +290,6 @@ int SecureSocketImpl::sendBytes(const void* buffer, int length, int flags)
 	poco_check_ptr (_pSSL);
 
 	int rc;
-
-	LockT l(_ssl_mutex);
-
 	if (_needHandshake)
 	{
 		rc = completeHandshake();
@@ -341,8 +331,6 @@ int SecureSocketImpl::receiveBytes(void* buffer, int length, int flags)
 	poco_assert (_pSocket->initialized());
 	poco_check_ptr (_pSSL);
 
-	LockT l(_ssl_mutex);
-
 	/// Special case: just check that we can read from socket
 	if ((flags & MSG_DONTWAIT) && (flags & MSG_PEEK))
 		return _pSocket->receiveBytes(buffer, length, flags);
@@ -380,8 +368,6 @@ int SecureSocketImpl::available() const
 	std::lock_guard<std::recursive_mutex> lock(_mutex);
 	poco_check_ptr (_pSSL);
 
-	LockT l(_ssl_mutex);
-
 	return SSL_pending(_pSSL);
 }
 
@@ -478,20 +464,10 @@ bool SecureSocketImpl::isLocalHost(const std::string& hostName)
 X509* SecureSocketImpl::peerCertificate() const
 {
 	std::lock_guard<std::recursive_mutex> lock(_mutex);
-    LockT l(_ssl_mutex);
-
-	X509* pCert = nullptr;
-
 	if (_pSSL)
-	{
-		pCert = ::SSL_get_peer_certificate(_pSSL);
-
-		if (X509_V_OK != SSL_get_verify_result(_pSSL))
-			throw CertificateValidationException("SecureSocketImpl::peerCertificate(): "
-				"Certificate verification error " + Utility::getLastError());
-	}
-
-	return pCert;
+		return SSL_get1_peer_certificate(_pSSL);
+	else
+		return 0;
 }
 
 Poco::Timespan SecureSocketImpl::getMaxTimeoutOrLimit()
@@ -632,8 +608,6 @@ void SecureSocketImpl::reset()
 	close();
 	if (_pSSL)
 	{
-		LockT l(_ssl_mutex);
-
 		SSL_free(_pSSL);
 		_pSSL = nullptr;
 	}
@@ -678,12 +652,9 @@ bool SecureSocketImpl::sessionWasReused()
 {
 	std::lock_guard<std::recursive_mutex> lock(_mutex);
 	if (_pSSL)
-	{
-		LockT l(_ssl_mutex);
-		return ::SSL_session_reused(_pSSL) != 0;
-	}
-
-	return false;
+		return SSL_session_reused(_pSSL) != 0;
+	else
+		return false;
 }
 
 void SecureSocketImpl::setBlocking(bool flag)

contrib/CMakeLists.txt

@@ -122,7 +122,6 @@ add_contrib (aws-cmake
     aws-c-mqtt
     aws-c-s3
     aws-c-sdkutils
-    aws-s2n-tls
     aws-checksums
     aws-crt-cpp
     aws-cmake

contrib/aws-cmake/CMakeLists.txt

@@ -27,6 +27,8 @@ include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsThreadName.cmake")
 include("${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/AwsSIMD.cmake")
 include("${ClickHouse_SOURCE_DIR}/contrib/aws-crt-cpp/cmake/AwsGetVersion.cmake")
 
+set (AWS_STUBS "${ClickHouse_SOURCE_DIR}/contrib/aws-cmake/aws_stubs.cpp")
+
 
 # Gather sources and options.
 set(AWS_SOURCES)
@@ -47,11 +49,6 @@ if (ENABLE_OPENSSL_ENCRYPTION)
     list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DENABLE_OPENSSL_ENCRYPTION")
 endif()
 
-set(USE_S2N ON)
-if (USE_S2N)
-    list(APPEND AWS_PRIVATE_COMPILE_DEFS "-DUSE_S2N")
-endif()
-
 
 # Directories.
 SET(AWS_SDK_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws")
@@ -70,7 +67,6 @@ SET(AWS_EVENT_STREAM_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-event-stream")
 SET(AWS_HTTP_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-http")
 SET(AWS_IO_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-io")
 SET(AWS_MQTT_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-mqtt")
-SET(AWS_S2N_TLS_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-s2n-tls")
 SET(AWS_S3_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-s3")
 SET(AWS_SDKUTILS_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-c-sdkutils")
 
@@ -287,39 +283,10 @@ elseif (OS_DARWIN)
     )
 endif()
 
-set(AWS_IO_TLS_SRC)
-if (USE_S2N)
-    file(GLOB AWS_IO_TLS_SRC
-            "${AWS_IO_DIR}/source/s2n/*.c"
-    )
-endif()
-
-list(APPEND AWS_SOURCES ${AWS_IO_SRC} ${AWS_IO_OS_SRC} ${AWS_IO_TLS_SRC})
+list(APPEND AWS_SOURCES ${AWS_IO_SRC} ${AWS_IO_OS_SRC})
 list(APPEND AWS_PUBLIC_INCLUDES "${AWS_IO_DIR}/include/")
 
 
-# aws-s2n-tls
-if (USE_S2N)
-    file(GLOB AWS_S2N_TLS_SRC
-        "${AWS_S2N_TLS_DIR}/crypto/*.c"
-        "${AWS_S2N_TLS_DIR}/error/*.c"
-        "${AWS_S2N_TLS_DIR}/stuffer/*.c"
-        "${AWS_S2N_TLS_DIR}/pq-crypto/*.c"
-        "${AWS_S2N_TLS_DIR}/pq-crypto/kyber_r3/*.c"
-        "${AWS_S2N_TLS_DIR}/tls/*.c"
-        "${AWS_S2N_TLS_DIR}/tls/extensions/*.c"
-        "${AWS_S2N_TLS_DIR}/utils/*.c"
-    )
-
-    list(APPEND AWS_SOURCES ${AWS_S2N_TLS_SRC})
-
-    list(APPEND AWS_PRIVATE_INCLUDES
-        "${AWS_S2N_TLS_DIR}/"
-        "${AWS_S2N_TLS_DIR}/api/"
-    )
-endif()
-
-
 # aws-crt-cpp
 file(GLOB AWS_CRT_SRC
     "${AWS_CRT_DIR}/source/*.cpp"
@@ -336,11 +303,6 @@ list(APPEND AWS_PUBLIC_INCLUDES "${AWS_CRT_DIR}/include/")
 
 
 # aws-c-mqtt
-file(GLOB AWS_MQTT_SRC
-    "${AWS_MQTT_DIR}/source/*.c"
-)
-
-list(APPEND AWS_SOURCES ${AWS_MQTT_SRC})
 list(APPEND AWS_PUBLIC_INCLUDES "${AWS_MQTT_DIR}/include/")
 
 
@@ -388,6 +350,8 @@ file(GLOB AWS_SDK_GLUE_SRC
 list(APPEND AWS_SOURCES ${AWS_SDK_GLUE_SRC})
 list(APPEND AWS_PUBLIC_INCLUDES "${AWS_SDK_GLUE_DIR}/include/")
 
+list(APPEND AWS_SOURCES ${AWS_STUBS})
+
 # Add library.
 add_library(_aws ${AWS_SOURCES})
 

contrib/aws-cmake/aws_stubs.cpp

@@ -0,0 +1,25 @@
+extern "C" {
+
+/// Symbols for aws-c-mqtt and aws-s2n-tls
+/// which are not used anywhere except those stubs.
+
+// TLS context creation (returns null)
+__attribute__((weak)) void *aws_tls_client_ctx_new() { return nullptr; }
+__attribute__((weak)) void *aws_tls_server_ctx_new() { return nullptr; }
+
+// TLS static init/cleanup (do nothing)
+__attribute__((weak)) void aws_tls_init_static_state() {}
+__attribute__((weak)) void aws_tls_clean_up_static_state() {}
+
+// MQTT init/cleanup (do nothing)
+__attribute__((weak)) void aws_mqtt_library_init() {}
+__attribute__((weak)) void aws_mqtt_library_clean_up() {}
+
+// Darwin-specific TLS handlers (return null or 0)
+__attribute__((weak)) void *aws_tls_client_handler_new() { return nullptr; }
+__attribute__((weak)) void *aws_tls_server_handler_new() { return nullptr; }
+__attribute__((weak)) int aws_tls_client_handler_start_negotiation() { return 0; }
+__attribute__((weak)) void *aws_tls_handler_protocol() { return nullptr; }
+__attribute__((weak)) int aws_tls_is_alpn_available() { return 0; }
+
+} // extern "C"

contrib/aws-s2n-tls

@@ -38,7 +38,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="25.4.4.25"
+ARG VERSION="25.4.5.24"
 ARG PACKAGES="clickhouse-keeper"
 ARG DIRECT_DOWNLOAD_URLS=""
 

docker/keeper/Dockerfile

@@ -35,7 +35,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="25.4.4.25"
+ARG VERSION="25.4.5.24"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 ARG DIRECT_DOWNLOAD_URLS=""