Backport ClickHouse#91664 to 25.8: Fix memory usage in join with additional filter

Author: robot-clickhouse

src/Interpreters/ConcurrentHashJoin.cpp

@@ -320,17 +320,23 @@ class ConcurrentHashJoinResult : public IJoinResult
         if (!current_result)
         {
             if (next_block >= dispatched_blocks.size())
-                return {Block(), true};
+                return {Block(), nullptr, true};
 
             current_result = hash_joins[next_block]->data->joinScatteredBlock(std::move(dispatched_blocks[next_block]));
-            ++next_block;
         }
 
         auto data = current_result->next();
         if (data.is_last)
+        {
+            if (data.next_block)
+                dispatched_blocks[next_block] = std::move(*data.next_block);
+            else
+                ++next_block;
             current_result.reset();
+        }
+
         bool is_last = next_block >= dispatched_blocks.size() && data.is_last;
-        return {std::move(data.block), is_last};
+        return {std::move(data.block), nullptr, is_last};
     }
 };
 

src/Interpreters/HashJoin/AddedColumns.h

@@ -265,14 +265,4 @@ class AddedColumns
     }
 };
 
-/// Adapter class to pass into addFoundRowAll
-/// In joinRightColumnsWithAdditionalFilter we don't want to add rows directly into AddedColumns,
-/// because they need to be filtered by additional_filter_expression.
-class PreSelectedRows : public std::vector<const RowRef *>
-{
-public:
-    void appendFromBlock(const RowRef * row_ref, bool /* has_default */) { this->emplace_back(row_ref); }
-    static constexpr bool isLazy() { return false; }
-};
-
 }

src/Interpreters/HashJoin/HashJoin.cpp

@@ -1036,7 +1036,7 @@ IJoinResult::JoinResultBlock CrossJoinResult::next()
 
     bool is_last = left_row >= rows_total;
     res = res.cloneWithColumns(std::move(dst_columns));
-    return {res, is_last};
+    return {res, nullptr, is_last};
 }
 
 JoinResultPtr HashJoin::joinBlockImplCross(Block block) const

src/Interpreters/HashJoin/HashJoinMethods.h

@@ -114,23 +114,23 @@ class HashJoinMethods
         bool & all_values_unique);
 
     template <typename AddedColumns>
-    static void switchJoinRightColumns(
+    static size_t switchJoinRightColumns(
         const std::vector<const MapsTemplate *> & mapv,
         AddedColumns & added_columns,
         const ScatteredBlock::Selector & selector,
         HashJoin::Type type,
         JoinStuff::JoinUsedFlags & used_flags);
 
     template <typename KeyGetter, typename Map, typename AddedColumns>
-    static void joinRightColumnsSwitchNullability(
+    static size_t joinRightColumnsSwitchNullability(
         std::vector<KeyGetter> && key_getter_vector,
         const std::vector<const Map *> & mapv,
         AddedColumns & added_columns,
         const ScatteredBlock::Selector & selector,
         JoinStuff::JoinUsedFlags & used_flags);
 
     template <typename KeyGetter, typename Map, bool need_filter, typename AddedColumns>
-    static void joinRightColumnsSwitchMultipleDisjuncts(
+    static size_t joinRightColumnsSwitchMultipleDisjuncts(
         std::vector<KeyGetter> && key_getter_vector,
         const std::vector<const Map *> & mapv,
         AddedColumns & added_columns,
@@ -147,7 +147,7 @@ class HashJoinMethods
         JoinCommon::JoinMask::Kind join_mask_kind,
         typename AddedColumns,
         typename Selector>
-    static void joinRightColumns(
+    static size_t joinRightColumns(
         std::vector<KeyGetter> && key_getter_vector,
         const std::vector<const Map *> & mapv,
         AddedColumns & added_columns,
@@ -161,7 +161,7 @@ class HashJoinMethods
         bool check_null_map,
         typename AddedColumns,
         typename Selector>
-    static void joinRightColumnsSwitchJoinMaskKind(
+    static size_t joinRightColumnsSwitchJoinMaskKind(
         std::vector<KeyGetter> && key_getter_vector,
         const std::vector<const Map *> & mapv,
         AddedColumns & added_columns,
@@ -176,31 +176,24 @@ class HashJoinMethods
         JoinCommon::JoinMask::Kind join_mask_kind,
         typename AddedColumns,
         typename Selector>
-    static void joinRightColumns(
+    static size_t joinRightColumns(
         KeyGetter & key_getter,
         const Map * map,
         AddedColumns & added_columns,
         JoinStuff::JoinUsedFlags & used_flags,
         const Selector & selector);
 
     template <typename KeyGetter, typename Map, bool need_filter, bool check_null_map, typename AddedColumns, typename Selector>
-    static void joinRightColumnsSwitchJoinMaskKind(
+    static size_t joinRightColumnsSwitchJoinMaskKind(
         KeyGetter & key_getter,
         const Map * map,
         AddedColumns & added_columns,
         JoinStuff::JoinUsedFlags & used_flags,
         const Selector & selector);
 
-    template <typename AddedColumns, typename Selector>
-    static ColumnPtr buildAdditionalFilter(
-        const Selector & selector,
-        const std::vector<const RowRef *> & selected_rows,
-        const std::vector<size_t> & row_replicate_offset,
-        AddedColumns & added_columns);
-
     /// First to collect all matched rows refs by join keys, then filter out rows which are not true in additional filter expression.
     template <typename KeyGetter, typename Map, typename AddedColumns>
-    static void joinRightColumnsWithAddtitionalFilter(
+    static size_t joinRightColumnsWithAddtitionalFilter(
         std::vector<KeyGetter> && key_getter_vector,
         const std::vector<const Map *> & mapv,
         AddedColumns & added_columns,