Merge pull request ClickHouse#80360 from ClickHouse/logical-join-predicate-push-down-pre-filter

Fix predicate-push-down for the logical JOIN step if the pre-join exp…

Author: nikitamikhaylov

src/Processors/QueryPlan/Optimizations/filterPushDown.cpp

@@ -387,8 +387,33 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan::
 
     size_t updated_steps = 0;
 
+    /// For the logical join step, we need to merge pre-join actions to filter dag.
+    /// TODO: this should be refactored and replaced with optimizations which
+    /// 1. push filter/expression into JOIN (as post-filter)
+    /// 2. move filter within JOIN step, potentially changing JoinKind
+    /// 3. push filter/expression out of JOIN (from pre-filter)
+    auto fix_predicate_for_join_logical_step = [&](ActionsDAG filter_dag, const ActionsDAG & side_dag)
+    {
+        filter_dag = ActionsDAG::merge(side_dag.clone(), std::move(filter_dag));
+        auto & outputs = filter_dag.getOutputs();
+        outputs.resize(1);
+        outputs.insert(outputs.end(), filter_dag.getInputs().begin(), filter_dag.getInputs().end());
+        filter_dag.removeUnusedActions();
+        return filter_dag;
+    };
+
     if (join_filter_push_down_actions.left_stream_filter_to_push_down)
     {
+        if (logical_join)
+        {
+
+            join_filter_push_down_actions.left_stream_filter_to_push_down = fix_predicate_for_join_logical_step(
+                std::move(*join_filter_push_down_actions.left_stream_filter_to_push_down),
+                *logical_join->getExpressionActions().left_pre_join_actions
+            );
+            join_filter_push_down_actions.left_stream_filter_removes_filter = true;
+        }
+
         const auto & result_name = join_filter_push_down_actions.left_stream_filter_to_push_down->getOutputs()[0]->result_name;
         updated_steps += addNewFilterStepOrThrow(parent_node,
             nodes,
@@ -404,6 +429,16 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan::
 
     if (join_filter_push_down_actions.right_stream_filter_to_push_down && allow_push_down_to_right)
     {
+        if (logical_join)
+        {
+
+            join_filter_push_down_actions.right_stream_filter_to_push_down = fix_predicate_for_join_logical_step(
+                std::move(*join_filter_push_down_actions.right_stream_filter_to_push_down),
+                *logical_join->getExpressionActions().right_pre_join_actions
+            );
+            join_filter_push_down_actions.right_stream_filter_removes_filter = true;
+        }
+
         const auto & result_name = join_filter_push_down_actions.right_stream_filter_to_push_down->getOutputs()[0]->result_name;
         updated_steps += addNewFilterStepOrThrow(parent_node,
             nodes,

tests/queries/0_stateless/03393_join_bug_77848.reference

@@ -0,0 +1,2 @@
+1
+1

tests/queries/0_stateless/03393_join_bug_77848.sql

@@ -13,6 +13,6 @@ SET enable_analyzer = 1;
 -- TODO(@vdimir): NOT_FOUND_COLUMN_IN_BLOCK is a bug, should be fixed
 -- This tests ensures that query does not crash at least
 
-SELECT 1 FROM BadTable i LEFT JOIN BadJoin c ON i.id_uint = toUInt128(c.id) WHERE equals(i.id_uint, 12); --  { serverError NOT_FOUND_COLUMN_IN_BLOCK}
+SELECT 1 FROM BadTable i LEFT JOIN BadJoin c ON i.id_uint = toUInt128(c.id) WHERE equals(i.id_uint, 12);
 
-SELECT equals(i.id_uint, 12) FROM BadTable i LEFT JOIN BadJoin c ON i.id_uint = toUInt128(c.id) WHERE equals(i.id_uint, 12); --  { serverError NOT_FOUND_COLUMN_IN_BLOCK}
+SELECT equals(i.id_uint, 12) FROM BadTable i LEFT JOIN BadJoin c ON i.id_uint = toUInt128(c.id) WHERE equals(i.id_uint, 12);

tests/queries/0_stateless/03517_logical_join_predicate_push_down_with_pre_expression_bug.reference

@@ -0,0 +1,5 @@
+2025-04-01
+2025-04-01
+2025-04-01
+2025-04-01
+2025-04-01

tests/queries/0_stateless/03517_logical_join_predicate_push_down_with_pre_expression_bug.sql

@@ -0,0 +1,24 @@
+drop table if exists td;
+drop table if exists tdt;
+
+CREATE TABLE td (id Int16, d Date) ENGINE=MergeTree() order by id;
+CREATE TABLE tdt (id Int16, dt DateTime) ENGINE=MergeTree() order by id;
+
+insert into td values (1,'2025-03-01'),(2,'2025-04-01');
+insert into tdt values (1,'2025-03-01 01:01:01'),(2,'2025-03-01 02:01:01'),(3,'2025-04-01 03:01:01'),(4,'2025-04-01 04:01:01'),(5,'2025-04-01 05:01:01');
+
+SELECT td_d FROM (SELECT t.id td_id, t.d td_d, uniqExact(tdt.id) as cnt FROM td as t LEFT JOIN tdt ON toDate(tdt.dt) = t.d GROUP BY td_id, td_d) WHERE td_d = '2025-04-01';
+SELECT td_d FROM (SELECT t.id td_id, t.d td_d, uniqExact(tdt.id) as cnt FROM tdt RIGHT JOIN td as t ON toDate(tdt.dt) = t.d GROUP BY td_id, td_d) WHERE td_d = '2025-04-01';
+
+SELECT td_d FROM (SELECT t.id td_id, t.d td_d, uniqExact(tdt.id) as cnt FROM td as t INNER JOIN tdt ON toDate(tdt.dt) = t.d GROUP BY td_id, td_d) WHERE td_d = '2025-04-01';
+SELECT td_d FROM (SELECT t.id td_id, t.d td_d, uniqExact(tdt.id) as cnt FROM tdt INNER JOIN td as t ON toDate(tdt.dt) = t.d GROUP BY td_id, td_d) WHERE td_d = '2025-04-01';
+
+CREATE VIEW v AS
+SELECT
+  t.id td_id, t.d td_d, uniqExact(tdt.id) as cnt
+FROM
+  td as t
+  LEFT JOIN tdt ON toDate(tdt.dt) = t.d
+GROUP BY td_id, td_d;
+
+SELECT td_d FROM v WHERE td_d = '2025-04-01';