Merge 'master' into ncb/splitbychar-test

Author: bharatnc

src/Common/ProfileEvents.cpp

@@ -1135,6 +1135,8 @@ Event end() { return END; }
 
 bool checkCPUOverload(Int64 os_cpu_busy_time_threshold, double min_ratio, double max_ratio, bool should_throw)
 {
+    if ((max_ratio <= 0.0) || (max_ratio <= min_ratio))
+        return false;
     double cpu_load = global_counters.getCPUOverload(os_cpu_busy_time_threshold);
 
     if (cpu_load > DBL_EPSILON)

src/Compression/CompressedReadBufferBase.cpp

@@ -1,4 +1,4 @@
-#include "CompressedReadBufferBase.h"
+#include <Compression/CompressedReadBufferBase.h>
 
 #include <bit>
 #include <cstring>
@@ -179,8 +179,8 @@ size_t CompressedReadBufferBase::readCompressedData(size_t & size_decompressed,
 
     UInt8 header_size = ICompressionCodec::getHeaderSize();
     own_compressed_buffer.resize(header_size + sizeof(Checksum));
-
     compressed_in->readStrict(own_compressed_buffer.data(), sizeof(Checksum) + header_size);
+    own_compressed_buffer_header_init = true;
 
     readHeaderAndGetCodecAndSize(
         own_compressed_buffer.data() + sizeof(Checksum),
@@ -233,6 +233,7 @@ size_t CompressedReadBufferBase::readCompressedDataBlockForAsynchronous(size_t &
 
     own_compressed_buffer.resize(header_size + sizeof(Checksum));
     compressed_in->readStrict(own_compressed_buffer.data(), sizeof(Checksum) + header_size);
+    own_compressed_buffer_header_init = true;
 
     readHeaderAndGetCodecAndSize(
         own_compressed_buffer.data() + sizeof(Checksum),
@@ -337,7 +338,9 @@ void CompressedReadBufferBase::addDiagnostics(Exception & e) const
     if (auto * seekable_in = dynamic_cast<SeekableReadBuffer *>(compressed_in))
         current_pos = seekable_in->tryGetPosition();
     UInt8 header_size = ICompressionCodec::getHeaderSize();
-    String header_hex = hexString(own_compressed_buffer.data(), std::min(own_compressed_buffer.size(), sizeof(Checksum) + header_size));
+    String header_hex = own_compressed_buffer_header_init ?
+        hexString(own_compressed_buffer.data(), std::min(own_compressed_buffer.size(), sizeof(Checksum) + header_size)) :
+        String("<uninitialized>"); // We do not print uninitialized memory because it's a security vulnerability and triggers msan
 
     e.addMessage("While reading or decompressing {} (position: {}, typename: {}, compressed data header: {})",
                  getFileNameFromReadBuffer(*compressed_in),

src/Compression/CompressedReadBufferBase.h

@@ -22,6 +22,7 @@ class CompressedReadBufferBase
 
     /// If 'compressed_in' buffer has whole compressed block - then use it. Otherwise copy parts of data to 'own_compressed_buffer'.
     PODArray<char> own_compressed_buffer;
+    bool own_compressed_buffer_header_init = false; // true if own_compressed_buffer header was initialized
     /// Points to memory, holding compressed block.
     char * compressed_buffer = nullptr;
 

src/Core/Settings.cpp

@@ -6510,8 +6510,8 @@ File/S3 engines/table function will parse paths with '::' as `<archive> :: <file
     DECLARE(Milliseconds, low_priority_query_wait_time_ms, 1000, R"(
 When the query prioritization mechanism is employed (see setting `priority`), low-priority queries wait for higher-priority queries to finish. This setting specifies the duration of waiting.
 )", BETA) \
-    DECLARE(Float, min_os_cpu_wait_time_ratio_to_throw, 2.0, "Min ratio between OS CPU wait (OSCPUWaitMicroseconds metric) and busy (OSCPUVirtualTimeMicroseconds metric) times to consider rejecting queries. Linear interpolation between min and max ratio is used to calculate the probability, the probability is 0 at this point.", 0) \
-    DECLARE(Float, max_os_cpu_wait_time_ratio_to_throw, 6.0, "Max ratio between OS CPU wait (OSCPUWaitMicroseconds metric) and busy (OSCPUVirtualTimeMicroseconds metric) times to consider rejecting queries. Linear interpolation between min and max ratio is used to calculate the probability, the probability is 1 at this point.", 0) \
+    DECLARE(Float, min_os_cpu_wait_time_ratio_to_throw, 0.0, "Min ratio between OS CPU wait (OSCPUWaitMicroseconds metric) and busy (OSCPUVirtualTimeMicroseconds metric) times to consider rejecting queries. Linear interpolation between min and max ratio is used to calculate the probability, the probability is 0 at this point.", 0) \
+    DECLARE(Float, max_os_cpu_wait_time_ratio_to_throw, 0.0, "Max ratio between OS CPU wait (OSCPUWaitMicroseconds metric) and busy (OSCPUVirtualTimeMicroseconds metric) times to consider rejecting queries. Linear interpolation between min and max ratio is used to calculate the probability, the probability is 1 at this point.", 0) \
     \
     /* ####################################################### */ \
     /* ########### START OF EXPERIMENTAL FEATURES ############ */ \

src/Core/SettingsChangesHistory.cpp

@@ -78,6 +78,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
             {"page_cache_lookahead_blocks", 16, 16, "Made this setting adjustable on a per-query level."},
             {"output_format_pretty_glue_chunks", "0", "auto", "A new setting to make Pretty formats prettier."},
             {"parallel_hash_join_threshold", 0, 100'000, "New setting"},
+            {"min_os_cpu_wait_time_ratio_to_throw", 0, 0, "Setting values were changed and backported to 25.4"},
+            {"max_os_cpu_wait_time_ratio_to_throw", 0, 0, "Setting values were changed and backported to 25.4"},
             {"make_distributed_plan", 0, 0, "New experimental setting."},
             {"execute_distributed_plan_locally", 0, 0, "New experimental setting."},
             {"default_shuffle_join_bucket_count", 8, 8, "New experimental setting."},
@@ -105,8 +107,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
             {"cast_string_to_variant_use_inference", true, true, "New setting to enable/disable types inference during CAST from String to Variant"},
             {"distributed_cache_read_request_max_tries", 20, 20, "New setting"},
             {"query_condition_cache_store_conditions_as_plaintext", false, false, "New setting"},
-            {"min_os_cpu_wait_time_ratio_to_throw", 0, 2, "New setting"},
-            {"max_os_cpu_wait_time_ratio_to_throw", 0, 6, "New setting"},
+            {"min_os_cpu_wait_time_ratio_to_throw", 0, 0, "New setting"},
+            {"max_os_cpu_wait_time_ratio_to_throw", 0, 0, "New setting"},
             {"query_plan_merge_filter_into_join_condition", false, true, "Added new setting to merge filter into join condition"},
             {"use_local_cache_for_remote_storage", true, false, "Obsolete setting."},
             {"iceberg_timestamp_ms", 0, 0, "New setting."},

src/DataTypes/DataTypesDecimal.h

@@ -130,11 +130,11 @@ inline UInt32 getDecimalScale(const DataTypeDecimal<T> & data_type)
 
 template <typename FromDataType, typename ToDataType, typename ReturnType = void>
 requires (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
-ReturnType convertDecimalsImpl(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result);
+ReturnType ALWAYS_INLINE convertDecimalsImpl(const typename FromDataType::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename ToDataType::FieldType & result);
 
 #define DISPATCH(FROM_DATA_TYPE, TO_DATA_TYPE) \
-    extern template void convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, void>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result); \
-    extern template bool convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, bool>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
+    extern template ALWAYS_INLINE void convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, void>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result); \
+    extern template ALWAYS_INLINE bool convertDecimalsImpl<FROM_DATA_TYPE, TO_DATA_TYPE, bool>(const typename FROM_DATA_TYPE::FieldType & value, UInt32 scale_from, UInt32 scale_to, typename TO_DATA_TYPE::FieldType & result);
 #define INVOKE(X) FOR_EACH_DECIMAL_TYPE_PASS(DISPATCH, X)
 FOR_EACH_DECIMAL_TYPE(INVOKE);
 #undef INVOKE

tests/clickhouse-test

@@ -1179,7 +1179,10 @@ class MergeTreeSettingsRandomizer:
 
 
 def replace_in_file(filename, what, with_what):
-    os.system(f"LC_ALL=C sed -i -e 's|{what}|{with_what}|g' {filename}")
+    with open(filename, "rb") as f:
+        data = f.read()
+    with open(filename, "wb") as f:
+        f.write(data.replace(what.encode(), with_what.encode()))
 
 
 class TestResult:

tests/queries/0_stateless/01945_show_debug_warning.expect

@@ -44,7 +44,7 @@ expect eof
 
 if { $Debug_type > 0} {
 
-spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_EXPECT_OPT --history_file=$history_file"
+spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \${CLICKHOUSE_CLIENT_EXPECT_OPT/--no-warnings} --history_file=$history_file"
 expect "Warnings:"
 expect " * Server was built in debug mode. It will work slowly."
 expect ":) "
@@ -58,7 +58,7 @@ send -- "q\r"
 expect eof
 }
 
-spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_EXPECT_OPT --max_memory_usage_for_all_queries=123 --history_file=$history_file"
+spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \${CLICKHOUSE_CLIENT_EXPECT_OPT/--no-warnings} --max_memory_usage_for_all_queries=123 --history_file=$history_file"
 expect "Warnings:"
 expect " * Obsolete setting"
 expect ":) "