[branch-4.0]Pick predicates-related PR

be/src/exec/olap_utils.h

@@ -67,60 +67,31 @@ enum SQLFilterOp {
     FILTER_LESS = 2,
     FILTER_LESS_OR_EQUAL = 3,
     FILTER_IN = 4,
-    FILTER_NOT_IN = 5
+    FILTER_NOT_IN = 5,
+    FILTER_EQ = 6,
+    FILTER_NE = 7
 };
 
 template <PrimitiveType>
 constexpr bool always_false_v = false;
 
-inline SQLFilterOp to_olap_filter_type(TExprOpcode::type type, bool opposite) {
-    switch (type) {
-    case TExprOpcode::LT:
-        return opposite ? FILTER_LARGER : FILTER_LESS;
-
-    case TExprOpcode::LE:
-        return opposite ? FILTER_LARGER_OR_EQUAL : FILTER_LESS_OR_EQUAL;
-
-    case TExprOpcode::GT:
-        return opposite ? FILTER_LESS : FILTER_LARGER;
-
-    case TExprOpcode::GE:
-        return opposite ? FILTER_LESS_OR_EQUAL : FILTER_LARGER_OR_EQUAL;
-
-    case TExprOpcode::EQ:
-        return opposite ? FILTER_NOT_IN : FILTER_IN;
-
-    case TExprOpcode::NE:
-        return opposite ? FILTER_IN : FILTER_NOT_IN;
-
-    case TExprOpcode::EQ_FOR_NULL:
-        return FILTER_IN;
-
-    default:
-        VLOG_CRITICAL << "TExprOpcode: " << type;
-        DCHECK(false);
-    }
-
-    return FILTER_IN;
-}
-
-inline SQLFilterOp to_olap_filter_type(const std::string& function_name, bool opposite) {
+inline SQLFilterOp to_olap_filter_type(const std::string& function_name) {
     if (function_name == "lt") {
-        return opposite ? FILTER_LARGER : FILTER_LESS;
+        return FILTER_LESS;
     } else if (function_name == "gt") {
-        return opposite ? FILTER_LESS : FILTER_LARGER;
+        return FILTER_LARGER;
     } else if (function_name == "le") {
-        return opposite ? FILTER_LARGER_OR_EQUAL : FILTER_LESS_OR_EQUAL;
+        return FILTER_LESS_OR_EQUAL;
     } else if (function_name == "ge") {
-        return opposite ? FILTER_LESS_OR_EQUAL : FILTER_LARGER_OR_EQUAL;
+        return FILTER_LARGER_OR_EQUAL;
     } else if (function_name == "eq") {
-        return opposite ? FILTER_NOT_IN : FILTER_IN;
+        return FILTER_EQ;
     } else if (function_name == "ne") {
-        return opposite ? FILTER_IN : FILTER_NOT_IN;
+        return FILTER_NE;
     } else if (function_name == "in") {
-        return opposite ? FILTER_NOT_IN : FILTER_IN;
+        return FILTER_IN;
     } else if (function_name == "not_in") {
-        return opposite ? FILTER_IN : FILTER_NOT_IN;
+        return FILTER_NOT_IN;
     } else {
         DCHECK(false) << "Function Name: " << function_name;
         return FILTER_IN;

be/src/exprs/bitmapfilter_predicate.h

@@ -19,6 +19,7 @@
 
 #include <algorithm>
 
+#include "common/cast_set.h"
 #include "runtime/define_primitive_type.h"
 #include "runtime/primitive_type.h"
 #include "runtime_filter/runtime_filter_definitions.h"
@@ -67,7 +68,8 @@ class BitmapFilterFunc : public BitmapFilterFuncBase {
         if (right < 0) {
             return false;
         }
-        return _bitmap_value->contains_any(std::max(left, (CppType)0), right);
+        return _bitmap_value->contains_any(cast_set<uint64_t>(std::max(left, (CppType)0)),
+                                           cast_set<uint64_t>(right));
     }
 
 private:

be/src/exprs/create_predicate_function.h

@@ -232,55 +232,58 @@ inline auto create_bitmap_filter(PrimitiveType type) {
 }
 
 template <PrimitiveType PT>
-ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
-                                              const std::shared_ptr<BloomFilterFuncBase>& filter,
-                                              const TabletColumn*, bool null_aware) {
+std::shared_ptr<const ColumnPredicate> create_olap_column_predicate(
+        uint32_t column_id, const std::shared_ptr<BloomFilterFuncBase>& filter, const TabletColumn*,
+        bool null_aware) {
     std::shared_ptr<BloomFilterFuncBase> filter_olap;
     filter_olap.reset(create_bloom_filter(PT, null_aware));
     filter_olap->light_copy(filter.get());
     // create a new filter to match the input filter and PT. For example, filter may be varchar, but PT is char
-    return new BloomFilterColumnPredicate<PT>(column_id, filter_olap);
+    return BloomFilterColumnPredicate<PT>::create_shared(column_id, filter_olap);
 }
 
 template <PrimitiveType PT>
-ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
-                                              const std::shared_ptr<BitmapFilterFuncBase>& filter,
-                                              const TabletColumn*, bool) {
+std::shared_ptr<const ColumnPredicate> create_olap_column_predicate(
+        uint32_t column_id, const std::shared_ptr<BitmapFilterFuncBase>& filter,
+        const TabletColumn*, bool) {
     if constexpr (PT == TYPE_TINYINT || PT == TYPE_SMALLINT || PT == TYPE_INT ||
                   PT == TYPE_BIGINT) {
-        return new BitmapFilterColumnPredicate<PT>(column_id, filter);
+        return BitmapFilterColumnPredicate<PT>::create_shared(column_id, filter);
     } else {
         throw Exception(ErrorCode::INTERNAL_ERROR, "bitmap filter do not support type {}", PT);
     }
 }
 
 template <PrimitiveType PT>
-ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
-                                              const std::shared_ptr<HybridSetBase>& filter,
-                                              const TabletColumn* column, bool) {
+std::shared_ptr<const ColumnPredicate> create_olap_column_predicate(
+        uint32_t column_id, const std::shared_ptr<HybridSetBase>& filter,
+        const TabletColumn* column, bool) {
     return create_in_list_predicate<PT, PredicateType::IN_LIST>(column_id, filter,
                                                                 column->length());
 }
 
 template <PrimitiveType PT>
-ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
-                                              const std::shared_ptr<FunctionFilter>& filter,
-                                              const TabletColumn* column, bool) {
+std::shared_ptr<ColumnPredicate> create_olap_column_predicate(
+        uint32_t column_id, const std::shared_ptr<FunctionFilter>& filter,
+        const TabletColumn* column, bool) {
     // currently only support like predicate
     if constexpr (PT == TYPE_CHAR) {
-        return new LikeColumnPredicate<TYPE_CHAR>(filter->_opposite, column_id, filter->_fn_ctx,
-                                                  filter->_string_param);
+        return LikeColumnPredicate<TYPE_CHAR>::create_shared(filter->_opposite, column_id,
+                                                             column->name(), filter->_fn_ctx,
+                                                             filter->_string_param);
     } else if constexpr (PT == TYPE_VARCHAR || PT == TYPE_STRING) {
-        return new LikeColumnPredicate<TYPE_STRING>(filter->_opposite, column_id, filter->_fn_ctx,
-                                                    filter->_string_param);
+        return LikeColumnPredicate<TYPE_STRING>::create_shared(filter->_opposite, column_id,
+                                                               column->name(), filter->_fn_ctx,
+                                                               filter->_string_param);
     }
     throw Exception(ErrorCode::INTERNAL_ERROR, "function filter do not support type {}", PT);
 }
 
 template <typename T>
-ColumnPredicate* create_column_predicate(uint32_t column_id, const std::shared_ptr<T>& filter,
-                                         FieldType type, const TabletColumn* column,
-                                         bool null_aware = false) {
+std::shared_ptr<ColumnPredicate> create_column_predicate(uint32_t column_id,
+                                                         const std::shared_ptr<T>& filter,
+                                                         FieldType type, const TabletColumn* column,
+                                                         bool null_aware = false) {
     switch (type) {
 #define M(NAME)                                                                           \
     case FieldType::OLAP_FIELD_##NAME: {                                                  \

be/src/olap/accept_null_predicate.h

@@ -40,8 +40,28 @@ class AcceptNullPredicate : public ColumnPredicate {
     ENABLE_FACTORY_CREATOR(AcceptNullPredicate);
 
 public:
-    AcceptNullPredicate(ColumnPredicate* nested)
-            : ColumnPredicate(nested->column_id(), nested->opposite()), _nested {nested} {}
+    AcceptNullPredicate(const std::shared_ptr<ColumnPredicate>& nested)
+            : ColumnPredicate(nested->column_id(), nested->col_name(), nested->primitive_type(),
+                              nested->opposite()),
+              _nested {nested} {}
+    AcceptNullPredicate(const AcceptNullPredicate& other, uint32_t col_id)
+            : ColumnPredicate(other, col_id),
+              _nested(assert_cast<const AcceptNullPredicate&>(other)._nested
+                              ? assert_cast<const AcceptNullPredicate&>(other)._nested->clone(
+                                        col_id)
+                              : nullptr) {}
+    AcceptNullPredicate(const AcceptNullPredicate& other) = delete;
+    ~AcceptNullPredicate() override = default;
+    std::shared_ptr<ColumnPredicate> clone(uint32_t col_id) const override {
+        return AcceptNullPredicate::create_shared(*this, col_id);
+    }
+    std::string debug_string() const override {
+        auto n = _nested;
+        fmt::memory_buffer debug_string_buffer;
+        fmt::format_to(debug_string_buffer, "AcceptNullPredicate({}, nested={})",
+                       ColumnPredicate::debug_string(), n ? n->debug_string() : "null");
+        return fmt::to_string(debug_string_buffer);
+    }
 
     PredicateType type() const override { return _nested->type(); }
 
@@ -173,11 +193,7 @@ class AcceptNullPredicate : public ColumnPredicate {
         return _nested->evaluate(column, sel, size);
     }
 
-    std::string _debug_string() const override {
-        return "passnull predicate for " + _nested->debug_string();
-    }
-
-    std::unique_ptr<ColumnPredicate> _nested;
+    std::shared_ptr<ColumnPredicate> _nested;
 };
 
 } //namespace doris

be/src/olap/bitmap_filter_predicate.h

@@ -27,17 +27,32 @@
 
 namespace doris {
 template <PrimitiveType T>
-class BitmapFilterColumnPredicate : public ColumnPredicate {
+class BitmapFilterColumnPredicate final : public ColumnPredicate {
 public:
+    ENABLE_FACTORY_CREATOR(BitmapFilterColumnPredicate);
     using CppType = typename PrimitiveTypeTraits<T>::CppType;
     using SpecificFilter = BitmapFilterFunc<T>;
 
-    BitmapFilterColumnPredicate(uint32_t column_id,
+    BitmapFilterColumnPredicate(uint32_t column_id, std::string col_name,
                                 const std::shared_ptr<BitmapFilterFuncBase>& filter)
-            : ColumnPredicate(column_id),
+            : ColumnPredicate(column_id, col_name, T),
               _filter(filter),
               _specific_filter(assert_cast<SpecificFilter*>(_filter.get())) {}
     ~BitmapFilterColumnPredicate() override = default;
+    BitmapFilterColumnPredicate(const BitmapFilterColumnPredicate& other, uint32_t col_id)
+            : ColumnPredicate(other, col_id),
+              _filter(other._filter),
+              _specific_filter(assert_cast<SpecificFilter*>(_filter.get())) {}
+    BitmapFilterColumnPredicate(const BitmapFilterColumnPredicate& other) = delete;
+    std::shared_ptr<ColumnPredicate> clone(uint32_t col_id) const override {
+        return BitmapFilterColumnPredicate<T>::create_shared(*this, col_id);
+    }
+    std::string debug_string() const override {
+        fmt::memory_buffer debug_string_buffer;
+        fmt::format_to(debug_string_buffer, "BitmapFilterColumnPredicate({})",
+                       ColumnPredicate::debug_string());
+        return fmt::to_string(debug_string_buffer);
+    }
 
     PredicateType type() const override { return PredicateType::BITMAP_FILTER; }
 
@@ -85,10 +100,6 @@ class BitmapFilterColumnPredicate : public ColumnPredicate {
         return new_size;
     }
 
-    std::string _debug_string() const override {
-        return "BitmapFilterColumnPredicate(" + type_to_string(T) + ")";
-    }
-
     std::shared_ptr<BitmapFilterFuncBase> _filter;
     SpecificFilter* _specific_filter; // owned by _filter
 

be/src/olap/block_column_predicate.h

@@ -33,7 +33,7 @@
 #include "olap/column_predicate.h"
 #include "olap/olap_common.h"
 #include "vec/columns/column.h"
-#include "vec/exec/format/parquet/parquet_pred_cmp.h"
+#include "vec/exec/format/parquet/parquet_predicate.h"
 
 namespace roaring {
 class Roaring;
@@ -60,7 +60,7 @@ class BlockColumnPredicate {
     virtual void get_all_column_ids(std::set<ColumnId>& column_id_set) const = 0;
 
     virtual void get_all_column_predicate(
-            std::set<const ColumnPredicate*>& predicate_set) const = 0;
+            std::set<std::shared_ptr<const ColumnPredicate>>& predicate_set) const = 0;
 
     virtual uint16_t evaluate(vectorized::MutableColumns& block, uint16_t* sel,
                               uint16_t selected_size) const {
@@ -118,13 +118,15 @@ class SingleColumnBlockPredicate : public BlockColumnPredicate {
     ENABLE_FACTORY_CREATOR(SingleColumnBlockPredicate);
 
 public:
-    explicit SingleColumnBlockPredicate(const ColumnPredicate* pre) : _predicate(pre) {}
+    explicit SingleColumnBlockPredicate(const std::shared_ptr<const ColumnPredicate>& pre)
+            : _predicate(pre) {}
 
     void get_all_column_ids(std::set<ColumnId>& column_id_set) const override {
         column_id_set.insert(_predicate->column_id());
     }
 
-    void get_all_column_predicate(std::set<const ColumnPredicate*>& predicate_set) const override {
+    void get_all_column_predicate(
+            std::set<std::shared_ptr<const ColumnPredicate>>& predicate_set) const override {
         predicate_set.insert(_predicate);
     }
 
@@ -154,7 +156,7 @@ class SingleColumnBlockPredicate : public BlockColumnPredicate {
     }
 
 private:
-    const ColumnPredicate* _predicate = nullptr;
+    const std::shared_ptr<const ColumnPredicate> _predicate = nullptr;
 };
 
 class MutilColumnBlockPredicate : public BlockColumnPredicate {
@@ -185,7 +187,8 @@ class MutilColumnBlockPredicate : public BlockColumnPredicate {
         }
     }
 
-    void get_all_column_predicate(std::set<const ColumnPredicate*>& predicate_set) const override {
+    void get_all_column_predicate(
+            std::set<std::shared_ptr<const ColumnPredicate>>& predicate_set) const override {
         for (auto& child_block_predicate : _block_column_predicate_vec) {
             child_block_predicate->get_all_column_predicate(predicate_set);
         }

be/src/olap/bloom_filter_predicate.h

@@ -30,16 +30,31 @@
 namespace doris {
 
 template <PrimitiveType T>
-class BloomFilterColumnPredicate : public ColumnPredicate {
+class BloomFilterColumnPredicate final : public ColumnPredicate {
 public:
+    ENABLE_FACTORY_CREATOR(BloomFilterColumnPredicate);
     using SpecificFilter = BloomFilterFunc<T>;
 
-    BloomFilterColumnPredicate(uint32_t column_id,
+    BloomFilterColumnPredicate(uint32_t column_id, std::string col_name,
                                const std::shared_ptr<BloomFilterFuncBase>& filter)
-            : ColumnPredicate(column_id),
+            : ColumnPredicate(column_id, col_name, T),
               _filter(filter),
               _specific_filter(assert_cast<SpecificFilter*>(_filter.get())) {}
     ~BloomFilterColumnPredicate() override = default;
+    BloomFilterColumnPredicate(const BloomFilterColumnPredicate& other, uint32_t col_id)
+            : ColumnPredicate(other, col_id),
+              _filter(other._filter),
+              _specific_filter(assert_cast<SpecificFilter*>(_filter.get())) {}
+    BloomFilterColumnPredicate(const BloomFilterColumnPredicate& other) = delete;
+    std::shared_ptr<ColumnPredicate> clone(uint32_t col_id) const override {
+        return BloomFilterColumnPredicate<T>::create_shared(*this, col_id);
+    }
+    std::string debug_string() const override {
+        fmt::memory_buffer debug_string_buffer;
+        fmt::format_to(debug_string_buffer, "BloomFilterColumnPredicate({})",
+                       ColumnPredicate::debug_string());
+        return fmt::to_string(debug_string_buffer);
+    }
 
     PredicateType type() const override { return PredicateType::BF; }
 
@@ -76,8 +91,6 @@ class BloomFilterColumnPredicate : public ColumnPredicate {
         return new_size;
     }
 
-    std::string _debug_string() const override { return "BloomFilter(" + type_to_string(T) + ")"; }
-
     std::shared_ptr<BloomFilterFuncBase> _filter;
     SpecificFilter* _specific_filter; // owned by _filter
 };