Skip to content

Commit bc601ac

Browse files
forsaken628forsaken628
authored
fix(query): tpcds q69 fail (#17516)
* fix tpcds q69 fail * test * test
1 parent d450a1a commit bc601ac

File tree

60 files changed

+496
-17
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+496
-17
lines changed

‎src/query/service/src/pipelines/processors/transforms/hash_join/probe_state.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ use databend_common_hashtable::RowPtr;
2424
use super::desc::MARKER_KIND_FALSE;
2525
use crate::sql::plans::JoinType;
2626

27+
#[derive(Debug)]
2728
pub struct ProcessState {
2829
pub input: DataBlock,
2930
pub keys_state: KeysState,

‎src/query/sql/src/executor/format.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1427,6 +1427,7 @@ fn hash_join_to_format_tree(
14271427
.map(|scalar| scalar.as_expr(&BUILTIN_FUNCTIONS).sql_display())
14281428
.collect::<Vec<_>>()
14291429
.join(", ");
1430+
let is_null_equal = plan.is_null_equal.iter().map(|b| format!("{b}")).join(", ");
14301431
let filters = plan
14311432
.non_equi_conditions
14321433
.iter()
@@ -1448,6 +1449,7 @@ fn hash_join_to_format_tree(
14481449
FormatTreeNode::new(format!("join type: {}", plan.join_type)),
14491450
FormatTreeNode::new(format!("build keys: [{build_keys}]")),
14501451
FormatTreeNode::new(format!("probe keys: [{probe_keys}]")),
1452+
FormatTreeNode::new(format!("keys is null equal: [{is_null_equal}]")),
14511453
FormatTreeNode::new(format!("filters: [{filters}]")),
14521454
];
14531455

‎src/query/sql/src/planner/optimizer/decorrelate/decorrelate.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,16 @@ impl SubqueryRewriter {
342342
&mut left_conditions,
343343
&mut right_conditions,
344344
)?;
345+
let mut is_null_equal = Vec::new();
346+
for (i, (l, r)) in left_conditions
347+
.iter()
348+
.zip(right_conditions.iter())
349+
.enumerate()
350+
{
351+
if l.data_type()?.is_nullable() || r.data_type()?.is_nullable() {
352+
is_null_equal.push(i);
353+
}
354+
}
345355

346356
let marker_index = if let Some(idx) = subquery.projection_index {
347357
idx
@@ -356,7 +366,7 @@ impl SubqueryRewriter {
356366
equi_conditions: JoinEquiCondition::new_conditions(
357367
right_conditions,
358368
left_conditions,
359-
vec![],
369+
is_null_equal,
360370
),
361371
non_equi_conditions: vec![],
362372
join_type: JoinType::RightMark,

‎src/query/sql/src/planner/optimizer/decorrelate/subquery_rewriter.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -376,7 +376,12 @@ impl SubqueryRewriter {
376376
span: subquery.span,
377377
func_name: "not".to_string(),
378378
params: vec![],
379-
arguments: vec![column_ref],
379+
arguments: vec![ScalarExpr::FunctionCall(FunctionCall {
380+
span: subquery.span,
381+
func_name: "is_true".to_string(),
382+
params: vec![],
383+
arguments: vec![column_ref],
384+
})],
380385
})
381386
} else {
382387
column_ref

‎src/query/sql/src/planner/optimizer/rule/rewrite/push_down_filter_join/mark_join_to_semi_join.rs‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -37,22 +37,19 @@ pub fn convert_mark_to_semi_join(s_expr: &SExpr) -> Result<(SExpr, bool)> {
3737

3838
// remove mark index filter
3939
for (idx, predicate) in filter.predicates.iter().enumerate() {
40-
match predicate {
41-
ScalarExpr::BoundColumnRef(col) if col.column.index == mark_index => {
40+
if !predicate.used_columns().contains(&mark_index) {
41+
continue;
42+
}
43+
44+
if let ScalarExpr::BoundColumnRef(col) = predicate {
45+
if col.column.index == mark_index {
4246
find_mark_index = true;
4347
filter.predicates.remove(idx);
4448
break;
4549
}
46-
ScalarExpr::FunctionCall(func) if func.func_name == "not" => {
47-
// Check if the argument is mark index, if so, we won't convert it to semi join
48-
if let ScalarExpr::BoundColumnRef(col) = &func.arguments[0] {
49-
if col.column.index == mark_index {
50-
return Ok((s_expr.clone(), false));
51-
}
52-
}
53-
}
54-
_ => (),
5550
}
51+
// Check if the predicate used mark, if so, we won't convert it to semi join
52+
return Ok((s_expr.clone(), false));
5653
}
5754

5855
if !find_mark_index {

‎src/query/sql/src/planner/plans/join.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,11 @@ pub enum JoinType {
5555
LeftAnti,
5656
RightAnti,
5757
/// Mark Join is a special case of join that is used to process Any subquery and correlated Exists subquery.
58-
/// Left Mark Join use subquery as probe side, it's blocked at `mark_join_blocks`
58+
/// Left Mark output build fields and marker
59+
/// Left Mark Join use subquery as probe(left) side, it's blocked at `mark_join_blocks`
5960
LeftMark,
60-
/// Right Mark Join use subquery as build side, it's executed by streaming.
61+
/// Right Mark output probe fields and marker
62+
/// Right Mark Join use subquery as build(right) side, it's executed by streaming.
6163
RightMark,
6264
/// Single Join is a special kind of join that is used to process correlated scalar subquery.
6365
LeftSingle,

‎tests/sqllogictests/suites/mode/cluster/exchange.test‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ Exchange
99
├── join type: INNER
1010
├── build keys: [t.number (#0)]
1111
├── probe keys: [t1.number (#1)]
12+
├── keys is null equal: [false]
1213
├── filters: []
1314
├── estimated rows: 2.00
1415
├── Exchange(Build)
@@ -44,6 +45,7 @@ Exchange
4445
├── join type: INNER
4546
├── build keys: [t.number (#0)]
4647
├── probe keys: [t2.number (#2)]
48+
├── keys is null equal: [false]
4749
├── filters: []
4850
├── estimated rows: 6.00
4951
├── Exchange(Build)
@@ -54,6 +56,7 @@ Exchange
5456
│ ├── join type: INNER
5557
│ ├── build keys: [t.number (#0)]
5658
│ ├── probe keys: [t1.number (#1)]
59+
│ ├── keys is null equal: [false]
5760
│ ├── filters: []
5861
│ ├── estimated rows: 2.00
5962
│ ├── Exchange(Build)
@@ -98,6 +101,7 @@ Exchange
98101
├── join type: INNER
99102
├── build keys: [t.b (#1)]
100103
├── probe keys: [t2.number (#3)]
104+
├── keys is null equal: [false]
101105
├── filters: []
102106
├── estimated rows: 6.00
103107
├── Exchange(Build)
@@ -108,6 +112,7 @@ Exchange
108112
│ ├── join type: INNER
109113
│ ├── build keys: [t.a (#0)]
110114
│ ├── probe keys: [t1.number (#2)]
115+
│ ├── keys is null equal: [false]
111116
│ ├── filters: []
112117
│ ├── estimated rows: 2.00
113118
│ ├── Exchange(Build)
@@ -156,6 +161,7 @@ Exchange
156161
├── join type: INNER
157162
├── build keys: [t.number (#1)]
158163
├── probe keys: [CAST(t1.number (#2) AS UInt64 NULL)]
164+
├── keys is null equal: [false]
159165
├── filters: []
160166
├── estimated rows: 2.00
161167
├── Exchange(Build)
@@ -240,6 +246,7 @@ Fragment 2:
240246
├── join type: INNER
241247
├── build keys: [t.number (#1)]
242248
├── probe keys: [CAST(t1.number (#2) AS UInt64 NULL)]
249+
├── keys is null equal: [false]
243250
├── filters: []
244251
├── estimated rows: 2.00
245252
├── ExchangeSource(Build)
@@ -294,6 +301,7 @@ AggregateFinal
294301
│ ├── join type: INNER
295302
│ ├── build keys: [t1.a (#0)]
296303
│ ├── probe keys: [t2.a (#1)]
304+
│ ├── keys is null equal: [false]
297305
│ ├── filters: []
298306
│ ├── estimated rows: 10000.00
299307
│ ├── Exchange(Build)
@@ -322,6 +330,7 @@ AggregateFinal
322330
├── join type: INNER
323331
├── build keys: [t3.a (#3)]
324332
├── probe keys: [t1.a (#2)]
333+
├── keys is null equal: [false]
325334
├── filters: []
326335
├── estimated rows: 100.00
327336
├── Exchange(Build)
@@ -379,6 +388,7 @@ AggregateFinal
379388
│ ├── join type: INNER
380389
│ ├── build keys: [t1.a (#0)]
381390
│ ├── probe keys: [t2.a (#1)]
391+
│ ├── keys is null equal: [false]
382392
│ ├── filters: []
383393
│ ├── estimated rows: 10000.00
384394
│ ├── Exchange(Build)
@@ -419,6 +429,7 @@ AggregateFinal
419429
├── join type: INNER
420430
├── build keys: [t3.a (#3)]
421431
├── probe keys: [t1.a (#2)]
432+
├── keys is null equal: [false]
422433
├── filters: []
423434
├── estimated rows: 100.00
424435
├── Exchange(Build)
@@ -475,6 +486,7 @@ AggregateFinal
475486
│ ├── join type: INNER
476487
│ ├── build keys: [t1.a (#0)]
477488
│ ├── probe keys: [t2.a (#1)]
489+
│ ├── keys is null equal: [false]
478490
│ ├── filters: []
479491
│ ├── estimated rows: 10000.00
480492
│ ├── Exchange(Build)
@@ -524,6 +536,7 @@ Exchange
524536
├── join type: CROSS
525537
├── build keys: []
526538
├── probe keys: []
539+
├── keys is null equal: []
527540
├── filters: []
528541
├── estimated rows: 100000.00
529542
├── Exchange(Build)
@@ -543,6 +556,7 @@ Exchange
543556
├── join type: CROSS
544557
├── build keys: []
545558
├── probe keys: []
559+
├── keys is null equal: []
546560
├── filters: []
547561
├── estimated rows: 10000.00
548562
├── Exchange(Build)
@@ -581,6 +595,7 @@ Exchange
581595
├── join type: INNER
582596
├── build keys: [t2.number (#1)]
583597
├── probe keys: [t1.number (#0)]
598+
├── keys is null equal: [false]
584599
├── filters: []
585600
├── estimated rows: 200.00
586601
├── Exchange(Build)

‎tests/sqllogictests/suites/mode/cluster/explain_v2.test‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ Exchange
4949
├── join type: INNER
5050
├── build keys: [t2.a (#2)]
5151
├── probe keys: [t1.a (#0)]
52+
├── keys is null equal: [false]
5253
├── filters: []
5354
├── estimated rows: 99.92
5455
├── Exchange(Build)
@@ -94,6 +95,7 @@ Exchange
9495
├── join type: INNER
9596
├── build keys: [t2.a (#2)]
9697
├── probe keys: [t1.a (#0)]
98+
├── keys is null equal: [false]
9799
├── filters: []
98100
├── estimated rows: 100.00
99101
├── Exchange(Build)
@@ -218,6 +220,7 @@ Limit
218220
├── join type: INNER
219221
├── build keys: [t2.a (#2)]
220222
├── probe keys: [t1.a (#0)]
223+
├── keys is null equal: [false]
221224
├── filters: []
222225
├── estimated rows: 100.00
223226
├── Exchange(Build)
@@ -258,6 +261,7 @@ Exchange
258261
├── join type: INNER
259262
├── build keys: [t2.a (#2)]
260263
├── probe keys: [t1.a (#0)]
264+
├── keys is null equal: [false]
261265
├── filters: []
262266
├── estimated rows: 100.00
263267
├── Exchange(Build)
@@ -336,6 +340,7 @@ AggregateFinal
336340
├── join type: RIGHT OUTER
337341
├── build keys: [CAST(y.a (#1) AS UInt64 NULL)]
338342
├── probe keys: [x.a (#2)]
343+
├── keys is null equal: [false]
339344
├── filters: []
340345
├── estimated rows: 50.00
341346
├── Exchange(Build)
@@ -393,6 +398,7 @@ Exchange
393398
├── join type: INNER
394399
├── build keys: [CAST(CAST(subquery_2 (#2) AS UInt8 NULL) AS Int32 NULL)]
395400
├── probe keys: [t1.a (#0)]
401+
├── keys is null equal: [false]
396402
├── filters: []
397403
├── estimated rows: 2.00
398404
├── Exchange(Build)
@@ -435,6 +441,7 @@ Exchange
435441
├── join type: LEFT OUTER
436442
├── build keys: [b.query_node (#63)]
437443
├── probe keys: [CAST(a.cluster_node (#0) AS String NULL)]
444+
├── keys is null equal: [false]
438445
├── filters: []
439446
├── estimated rows: 0.00
440447
├── Exchange(Build)

‎tests/sqllogictests/suites/mode/cluster/filter_nulls.test‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ Exchange
4343
├── join type: INNER
4444
├── build keys: [table2.value (#1)]
4545
├── probe keys: [table1.value (#0)]
46+
├── keys is null equal: [false]
4647
├── filters: []
4748
├── estimated rows: 250.00
4849
├── Exchange(Build)
@@ -88,6 +89,7 @@ Exchange
8889
├── join type: INNER
8990
├── build keys: [table3.value (#2)]
9091
├── probe keys: [table1.value (#0)]
92+
├── keys is null equal: [false]
9193
├── filters: []
9294
├── estimated rows: 200.00
9395
├── Exchange(Build)
@@ -98,6 +100,7 @@ Exchange
98100
│ ├── join type: INNER
99101
│ ├── build keys: [table2.value (#1)]
100102
│ ├── probe keys: [table3.value (#2)]
103+
│ ├── keys is null equal: [false]
101104
│ ├── filters: []
102105
│ ├── estimated rows: 250.00
103106
│ ├── Exchange(Build)
@@ -158,6 +161,7 @@ Exchange
158161
├── join type: LEFT SEMI
159162
├── build keys: [table2.value (#1)]
160163
├── probe keys: [table1.value (#0)]
164+
├── keys is null equal: [false]
161165
├── filters: []
162166
├── estimated rows: 250.00
163167
├── Exchange(Build)
@@ -203,6 +207,7 @@ Exchange
203207
├── join type: RIGHT SEMI
204208
├── build keys: [table2.value (#1)]
205209
├── probe keys: [table1.value (#0)]
210+
├── keys is null equal: [false]
206211
├── filters: []
207212
├── estimated rows: 250.00
208213
├── Exchange(Build)

‎tests/sqllogictests/suites/mode/cluster/merge_into_non_equal_distributed.test‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ CommitSink
7272
├── join type: LEFT OUTER
7373
├── build keys: []
7474
├── probe keys: []
75+
├── keys is null equal: []
7576
├── filters: []
7677
├── estimated rows: 1.00
7778
├── Exchange(Build)
@@ -121,6 +122,7 @@ CommitSink
121122
├── join type: RIGHT OUTER
122123
├── build keys: []
123124
├── probe keys: []
125+
├── keys is null equal: []
124126
├── filters: [t1.a (#1) > t.a (#0)]
125127
├── estimated rows: 15.00
126128
├── TableScan(Build)
@@ -158,6 +160,7 @@ CommitSink
158160
├── join type: RIGHT OUTER
159161
├── build keys: []
160162
├── probe keys: []
163+
├── keys is null equal: []
161164
├── filters: [t1.a (#1) < t2.a (#0)]
162165
├── estimated rows: 15.00
163166
├── TableScan(Build)
@@ -255,6 +258,7 @@ CommitSink
255258
├── join type: RIGHT OUTER
256259
├── build keys: [CAST(t2.a (#0) AS Int64 NULL)]
257260
├── probe keys: [CAST(t1.a (#1) AS Int64 NULL)]
261+
├── keys is null equal: [false]
258262
├── filters: []
259263
├── estimated rows: 0.00
260264
├── Exchange(Build)

0 commit comments

Comments
 (0)