Merge branch 'apache:master' into s4

Author: Tech-Circle-48

be/src/clucene

@@ -23,19 +23,20 @@
 
 #include "common/exception.h"
 #include "common/status.h"
+#include "vec/core/extended_types.h"
 
 namespace doris {
 
 template <typename T, typename U>
 void check_cast_value(U b) {
-    if constexpr (std::is_unsigned_v<U>) {
+    if constexpr (IsUnsignedV<U>) {
         if (b > std::numeric_limits<T>::max()) {
             throw doris::Exception(ErrorCode::INVALID_ARGUMENT,
                                    "value {} cast  to type {} out of range [{},{}]", b,
                                    typeid(T).name(), std::numeric_limits<T>::min(),
                                    std::numeric_limits<T>::max());
         }
-    } else if constexpr (std::is_unsigned_v<T>) {
+    } else if constexpr (IsUnsignedV<T>) {
         if (b < 0 || b > std::numeric_limits<T>::max()) {
             throw doris::Exception(ErrorCode::INVALID_ARGUMENT,
                                    "value {} cast  to type {} out of range [{},{}]", b,
@@ -53,7 +54,7 @@ void check_cast_value(U b) {
 }
 
 template <typename T, typename U, bool need_check_value = true>
-    requires std::is_integral_v<T> && std::is_integral_v<U>
+    requires IsIntegralV<T> && IsIntegralV<U>
 void cast_set(T& a, U b) {
     if constexpr (need_check_value) {
         check_cast_value<T>(b);
@@ -68,7 +69,7 @@ void cast_set(T& a, U b) {
 }
 
 template <typename T, typename U, bool need_check_value = true>
-    requires std::is_integral_v<T> && std::is_integral_v<U>
+    requires IsIntegralV<T> && IsIntegralV<U>
 T cast_set(U b) {
     if constexpr (need_check_value) {
         check_cast_value<T>(b);

be/src/common/cast_set.h

@@ -1047,12 +1047,11 @@ DEFINE_Int64(segcompaction_task_max_bytes, "157286400");
 // Global segcompaction thread pool size.
 DEFINE_mInt32(segcompaction_num_threads, "5");
 
+DEFINE_mInt32(segcompaction_wait_for_dbm_task_timeout_s, "3600"); // 1h
+
 // enable java udf and jdbc scannode
 DEFINE_Bool(enable_java_support, "true");
 
-// enable prefetch tablets before opening
-DEFINE_mBool(enable_prefetch_tablet, "true");
-
 // Set config randomly to check more issues in github workflow
 DEFINE_Bool(enable_fuzzy_mode, "false");
 

be/src/common/config.cpp

@@ -1087,15 +1087,14 @@ DECLARE_Int32(segcompaction_task_max_rows);
 // Max total file size allowed in a single segcompaction task.
 DECLARE_Int64(segcompaction_task_max_bytes);
 
+DECLARE_Int32(segcompaction_wait_for_dbm_task_timeout_s);
+
 // Global segcompaction thread pool size.
 DECLARE_mInt32(segcompaction_num_threads);
 
 // enable java udf and jdbc scannode
 DECLARE_Bool(enable_java_support);
 
-// enable prefetch tablets before opening
-DECLARE_mBool(enable_prefetch_tablet);
-
 // Set config randomly to check more issues in github workflow
 DECLARE_Bool(enable_fuzzy_mode);
 

be/src/common/config.h

@@ -34,9 +34,13 @@ Exception::Exception(int code, const std::string_view& msg) {
         _err_msg->_stack = get_stack_trace(0, "DISABLED");
     }
     // BE UT TEST exceptions thrown during execution cannot be caught
-    // and the error `C++ exception with description "argument not found" thrown in the test body` will be printed.
-    std::cout << "Exception: " << code << ", " << msg << ", " << get_stack_trace(0, "DISABLED")
-              << std::endl;
+    // and the error `C++ exception with description "argument not found" thrown in the test body` will be printed,
+    // which is of not much help.
+    // Disabled by default because now there are a lot of UT case that throw exceptions
+    // and it may slow down the test execution.
+    // Open this code to print the stack trace if you need to debug the UT case.
+    // std::cout << "Exception: " << code << ", " << msg << ", " << get_stack_trace(0, "DISABLED")
+    //           << std::endl;
 #endif
     if (config::exit_on_exception) {
         LOG(FATAL) << "[ExitOnException] error code: " << code << ", message: " << msg;

be/src/common/exception.cpp

@@ -439,6 +439,16 @@ inline void set_signal_task_id(TUniqueId tid) {
     query_id_lo = tid.lo;
 }
 
+// this struct is used to set signal task id in the constructor and reset it in the destructor
+// thread pool will reuse pthread, so we need to clean thread local data
+struct SignalTaskIdKeeper {
+    template <typename T>
+    SignalTaskIdKeeper(const T& id) {
+        set_signal_task_id(id);
+    }
+    ~SignalTaskIdKeeper() { set_signal_task_id(PUniqueId {}); }
+};
+
 inline void set_signal_is_nereids(bool is_nereids_arg) {
     is_nereids = is_nereids_arg;
 }

be/src/common/signal_handler.h

@@ -135,6 +135,7 @@ namespace ErrorCode {
     E(QUERY_MEMORY_EXCEEDED, -257, false);                   \
     E(WORKLOAD_GROUP_MEMORY_EXCEEDED, -258, false);          \
     E(PROCESS_MEMORY_EXCEEDED, -259, false);                 \
+    E(INVALID_INPUT_SYNTAX, -260, false);                    \
     E(CE_CMD_PARAMS_ERROR, -300, true);                      \
     E(CE_BUFFER_TOO_SMALL, -301, true);                      \
     E(CE_CMD_NOT_VALID, -302, true);                         \

be/src/common/status.h

@@ -33,7 +33,7 @@
 #include <assert.h>
 
 #include "olap/uint24.h"
-#include "vec/core/wide_integer.h"
+#include "vec/core/extended_types.h"
 
 inline uint16_t UNALIGNED_LOAD16(const void* p) {
     uint16_t t;

be/src/gutil/endian.h

@@ -139,14 +139,57 @@ int FloatToBuffer(float value, int width, char* buffer) {
     return snprintf_result;
 }
 
+// refer to: https://en.cppreference.com/w/cpp/types/numeric_limits/max_digits10.html
 int FastDoubleToBuffer(double value, char* buffer) {
-    auto end = fmt::format_to(buffer, FMT_COMPILE("{}"), value);
+    char* end = nullptr;
+    // output NaN and Infinity to be compatible with most of the implementations
+    if (std::isnan(value)) {
+        static constexpr char nan_str[] = "NaN";
+        static constexpr int nan_str_len = sizeof(nan_str) - 1;
+        memcpy(buffer, nan_str, nan_str_len);
+        end = buffer + nan_str_len;
+    } else if (std::isinf(value)) {
+        static constexpr char inf_str[] = "Infinity";
+        static constexpr int inf_str_len = sizeof(inf_str) - 1;
+        static constexpr char neg_inf_str[] = "-Infinity";
+        static constexpr int neg_inf_str_len = sizeof(neg_inf_str) - 1;
+        if (value > 0) {
+            memcpy(buffer, inf_str, inf_str_len);
+            end = buffer + inf_str_len;
+        } else {
+            memcpy(buffer, neg_inf_str, neg_inf_str_len);
+            end = buffer + neg_inf_str_len;
+        }
+    } else {
+        end = fmt::format_to(buffer, FMT_COMPILE("{}"), value);
+    }
     *end = '\0';
     return end - buffer;
 }
 
 int FastFloatToBuffer(float value, char* buffer) {
-    auto* end = fmt::format_to(buffer, FMT_COMPILE("{}"), value);
+    char* end = nullptr;
+    // output NaN and Infinity to be compatible with most of the implementations
+    if (std::isnan(value)) {
+        static constexpr char nan_str[] = "NaN";
+        static constexpr int nan_str_len = sizeof(nan_str) - 1;
+        memcpy(buffer, nan_str, nan_str_len);
+        end = buffer + nan_str_len;
+    } else if (std::isinf(value)) {
+        static constexpr char inf_str[] = "Infinity";
+        static constexpr int inf_str_len = sizeof(inf_str) - 1;
+        static constexpr char neg_inf_str[] = "-Infinity";
+        static constexpr int neg_inf_str_len = sizeof(neg_inf_str) - 1;
+        if (value > 0) {
+            memcpy(buffer, inf_str, inf_str_len);
+            end = buffer + inf_str_len;
+        } else {
+            memcpy(buffer, neg_inf_str, neg_inf_str_len);
+            end = buffer + neg_inf_str_len;
+        }
+    } else {
+        end = fmt::format_to(buffer, FMT_COMPILE("{}"), value);
+    }
     *end = '\0';
     return end - buffer;
 }

be/src/gutil/strings/numbers.cc

@@ -54,7 +54,6 @@ void LRUQueueRecorder::replay_queue_event(FileCacheType type) {
                 break;
             }
             case CacheLRULogType::MOVETOBACK: {
-                LOG(INFO) << "MOVETOBACK" << log->hash.to_string() << " " << log->offset;
                 auto it = shadow_queue.get(log->hash, log->offset, lru_log_lock);
                 if (it != std::list<LRUQueue::FileKeyAndOffset>::iterator()) {
                     shadow_queue.move_to_end(it, lru_log_lock);