Skip to content

Commit 4bfa47c

Browse files
AvogarAvogar
authored
Merge pull request ClickHouse#79489 from Avogar/subcolumns-prewhere
Allow moving conditions with subcolumns to prewhere
2 parents 2a5d0b0 + 68ac381 commit 4bfa47c

7 files changed

+295
-33
lines changed

‎src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,21 @@ static Int64 findMinPosition(const NameSet & condition_table_columns, const Name
5959
return min_position;
6060
}
6161

62+
static NameSet getTableColumns(const StorageMetadataPtr & metadata_snapshot, const Names & queried_columns)
63+
{
64+
const auto & columns_description = metadata_snapshot->getColumns();
65+
NameSet table_columns = collections::map<std::unordered_set>(columns_description.getAllPhysical(), [](const NameAndTypePair & col) { return col.name; });
66+
67+
/// Add also requested subcolumns to known table columns.
68+
for (const auto & column : queried_columns)
69+
{
70+
if (columns_description.hasSubcolumn(column))
71+
table_columns.insert(column);
72+
}
73+
74+
return table_columns;
75+
}
76+
6277
MergeTreeWhereOptimizer::MergeTreeWhereOptimizer(
6378
std::unordered_map<std::string, UInt64> column_sizes_,
6479
const StorageMetadataPtr & metadata_snapshot,
@@ -67,8 +82,7 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer(
6782
const std::optional<NameSet> & supported_columns_,
6883
LoggerPtr log_)
6984
: estimator(estimator_)
70-
, table_columns{collections::map<std::unordered_set>(
71-
metadata_snapshot->getColumns().getAllPhysical(), [](const NameAndTypePair & col) { return col.name; })}
85+
, table_columns(getTableColumns(metadata_snapshot, queried_columns_))
7286
, queried_columns{queried_columns_}
7387
, supported_columns{supported_columns_}
7488
, sorting_key_names{NameSet(

‎tests/performance/prewhere_subcolumns.xml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
<test>
2+
<settings>
3+
<max_threads>1</max_threads>
4+
<use_variant_as_common_type>1</use_variant_as_common_type>
5+
</settings>
6+
7+
<create_query>create table test (x UInt32, n Nullable(UInt32), s String, t Tuple(a UInt32, b String), json JSON) engine=MergeTree order by tuple()</create_query>
8+
<fill_query>insert into test select number, number = 0 ? NULL : number, repeat('a', 1000), tuple(number, randomString(1000)), toJSONString(map('a', number::Int32, 'b', repeat('a', 1000))) from numbers(1000000)</fill_query>
9+
10+
<query>select * from test where t.a = 42 format Null</query>
11+
<query>select * from test where n.null format Null</query>
12+
<query>select * from test where json.a = 42 format Null</query>
13+
14+
<drop_query>drop table if exists test</drop_query>
15+
</test>

‎tests/queries/0_stateless/03275_subcolumns_in_primary_key.reference‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ Sorting: a.b.c DESC, a.b.d DESC
8686
0 ((0,'str_0',[])) Hello, World!
8787
2 ((0,'str_2',[0,1])) Hello, World!
8888
Expression ((Project names + Projection))
89-
Filter ((WHERE + Change column names to column identifiers))
89+
Expression
9090
ReadFromMergeTree (default.test)
9191
Indexes:
9292
PrimaryKey
@@ -97,7 +97,7 @@ Parts: 1/2
9797
Granules: 2/8
9898
0 ((0,'str_0',[])) Hello, World!
9999
Expression ((Project names + Projection))
100-
Filter ((WHERE + Change column names to column identifiers))
100+
Expression
101101
ReadFromMergeTree (default.test)
102102
Indexes:
103103
PrimaryKey
@@ -125,7 +125,7 @@ ReadFromPreparedSource (_minmax_count_projection)
125125
Expression ((Project names + Projection))
126126
Aggregating
127127
Expression (Before GROUP BY)
128-
Filter ((WHERE + Change column names to column identifiers))
128+
Expression
129129
ReadFromMergeTree (default.test)
130130
Indexes:
131131
PrimaryKey
@@ -249,7 +249,7 @@ Sorting: a.b.c DESC, a.b.d DESC
249249
0 ((0,'str_0',[])) Hello, World!
250250
2 ((0,'str_2',[0,1])) Hello, World!
251251
Expression ((Project names + Projection))
252-
Filter ((WHERE + Change column names to column identifiers))
252+
Expression
253253
ReadFromMergeTree (default.test)
254254
Indexes:
255255
PrimaryKey
@@ -260,7 +260,7 @@ Parts: 1/1
260260
Granules: 2/8
261261
0 ((0,'str_0',[])) Hello, World!
262262
Expression ((Project names + Projection))
263-
Filter ((WHERE + Change column names to column identifiers))
263+
Expression
264264
ReadFromMergeTree (default.test)
265265
Indexes:
266266
PrimaryKey
@@ -288,7 +288,7 @@ ReadFromPreparedSource (_minmax_count_projection)
288288
Expression ((Project names + Projection))
289289
Aggregating
290290
Expression (Before GROUP BY)
291-
Filter ((WHERE + Change column names to column identifiers))
291+
Expression
292292
ReadFromMergeTree (default.test)
293293
Indexes:
294294
PrimaryKey
@@ -386,7 +386,7 @@ Sorting: a.b.c DESC, a.b.d DESC
386386
0 ((0,'str_0',[])) Hello, World!
387387
2 ((0,'str_2',[0,1])) Hello, World!
388388
Expression ((Project names + Projection))
389-
Filter ((WHERE + Change column names to column identifiers))
389+
Expression
390390
ReadFromMergeTree (default.test)
391391
Indexes:
392392
PrimaryKey
@@ -397,7 +397,7 @@ Parts: 1/2
397397
Granules: 2/8
398398
0 ((0,'str_0',[])) Hello, World!
399399
Expression ((Project names + Projection))
400-
Filter ((WHERE + Change column names to column identifiers))
400+
Expression
401401
ReadFromMergeTree (default.test)
402402
Indexes:
403403
PrimaryKey
@@ -425,7 +425,7 @@ ReadFromPreparedSource (_minmax_count_projection)
425425
Expression ((Project names + Projection))
426426
Aggregating
427427
Expression (Before GROUP BY)
428-
Filter ((WHERE + Change column names to column identifiers))
428+
Expression
429429
ReadFromMergeTree (default.test)
430430
Indexes:
431431
PrimaryKey
@@ -549,7 +549,7 @@ Sorting: a.b.c DESC, a.b.d DESC
549549
0 ((0,'str_0',[])) Hello, World!
550550
2 ((0,'str_2',[0,1])) Hello, World!
551551
Expression ((Project names + Projection))
552-
Filter ((WHERE + Change column names to column identifiers))
552+
Expression
553553
ReadFromMergeTree (default.test)
554554
Indexes:
555555
PrimaryKey
@@ -560,7 +560,7 @@ Parts: 1/1
560560
Granules: 2/8
561561
0 ((0,'str_0',[])) Hello, World!
562562
Expression ((Project names + Projection))
563-
Filter ((WHERE + Change column names to column identifiers))
563+
Expression
564564
ReadFromMergeTree (default.test)
565565
Indexes:
566566
PrimaryKey
@@ -588,7 +588,7 @@ ReadFromPreparedSource (_minmax_count_projection)
588588
Expression ((Project names + Projection))
589589
Aggregating
590590
Expression (Before GROUP BY)
591-
Filter ((WHERE + Change column names to column identifiers))
591+
Expression
592592
ReadFromMergeTree (default.test)
593593
Indexes:
594594
PrimaryKey
@@ -686,7 +686,7 @@ Sorting: a.b.c DESC, a.b.d DESC
686686
0 ((0,'str_0',[])) Hello, World!
687687
2 ((0,'str_2',[0,1])) Hello, World!
688688
Expression ((Project names + Projection))
689-
Filter ((WHERE + Change column names to column identifiers))
689+
Expression
690690
ReadFromMergeTree (default.test)
691691
Indexes:
692692
PrimaryKey
@@ -697,7 +697,7 @@ Parts: 1/2
697697
Granules: 2/8
698698
0 ((0,'str_0',[])) Hello, World!
699699
Expression ((Project names + Projection))
700-
Filter ((WHERE + Change column names to column identifiers))
700+
Expression
701701
ReadFromMergeTree (default.test)
702702
Indexes:
703703
PrimaryKey
@@ -725,7 +725,7 @@ ReadFromPreparedSource (_minmax_count_projection)
725725
Expression ((Project names + Projection))
726726
Aggregating
727727
Expression (Before GROUP BY)
728-
Filter ((WHERE + Change column names to column identifiers))
728+
Expression
729729
ReadFromMergeTree (default.test)
730730
Indexes:
731731
PrimaryKey
@@ -849,7 +849,7 @@ Sorting: a.b.c DESC, a.b.d DESC
849849
0 ((0,'str_0',[])) Hello, World!
850850
2 ((0,'str_2',[0,1])) Hello, World!
851851
Expression ((Project names + Projection))
852-
Filter ((WHERE + Change column names to column identifiers))
852+
Expression
853853
ReadFromMergeTree (default.test)
854854
Indexes:
855855
PrimaryKey
@@ -860,7 +860,7 @@ Parts: 1/1
860860
Granules: 2/8
861861
0 ((0,'str_0',[])) Hello, World!
862862
Expression ((Project names + Projection))
863-
Filter ((WHERE + Change column names to column identifiers))
863+
Expression
864864
ReadFromMergeTree (default.test)
865865
Indexes:
866866
PrimaryKey
@@ -888,7 +888,7 @@ ReadFromPreparedSource (_minmax_count_projection)
888888
Expression ((Project names + Projection))
889889
Aggregating
890890
Expression (Before GROUP BY)
891-
Filter ((WHERE + Change column names to column identifiers))
891+
Expression
892892
ReadFromMergeTree (default.test)
893893
Indexes:
894894
PrimaryKey

‎tests/queries/0_stateless/03277_json_subcolumns_in_primary_key.reference‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ Sorting: json.a DESC
8686
0 {"a":0,"b":"str_0","c":[]} Hello, World!
8787
2 {"a":0,"b":"str_2","c":["0","1"]} Hello, World!
8888
Expression ((Project names + Projection))
89-
Filter ((WHERE + Change column names to column identifiers))
89+
Expression
9090
ReadFromMergeTree (default.test)
9191
Indexes:
9292
PrimaryKey
@@ -97,7 +97,7 @@ Parts: 1/2
9797
Granules: 2/8
9898
0 {"a":0,"b":"str_0","c":[]} Hello, World!
9999
Expression ((Project names + Projection))
100-
Filter ((WHERE + Change column names to column identifiers))
100+
Expression
101101
ReadFromMergeTree (default.test)
102102
Indexes:
103103
PrimaryKey
@@ -125,7 +125,7 @@ ReadFromPreparedSource (_minmax_count_projection)
125125
Expression ((Project names + Projection))
126126
Aggregating
127127
Expression (Before GROUP BY)
128-
Filter ((WHERE + Change column names to column identifiers))
128+
Expression
129129
ReadFromMergeTree (default.test)
130130
Indexes:
131131
PrimaryKey
@@ -237,7 +237,7 @@ ReadFromPreparedSource (_minmax_count_projection)
237237
Expression ((Project names + Projection))
238238
Aggregating
239239
Expression (Before GROUP BY)
240-
Filter ((WHERE + Change column names to column identifiers))
240+
Expression
241241
ReadFromMergeTree (default.test)
242242
Indexes:
243243
PrimaryKey
@@ -334,7 +334,7 @@ Sorting: json.a DESC
334334
0 {"a":0,"b":"str_0","c":[]} Hello, World!
335335
2 {"a":0,"b":"str_2","c":["0","1"]} Hello, World!
336336
Expression ((Project names + Projection))
337-
Filter ((WHERE + Change column names to column identifiers))
337+
Expression
338338
ReadFromMergeTree (default.test)
339339
Indexes:
340340
PrimaryKey
@@ -345,7 +345,7 @@ Parts: 1/2
345345
Granules: 2/8
346346
0 {"a":0,"b":"str_0","c":[]} Hello, World!
347347
Expression ((Project names + Projection))
348-
Filter ((WHERE + Change column names to column identifiers))
348+
Expression
349349
ReadFromMergeTree (default.test)
350350
Indexes:
351351
PrimaryKey
@@ -373,7 +373,7 @@ ReadFromPreparedSource (_minmax_count_projection)
373373
Expression ((Project names + Projection))
374374
Aggregating
375375
Expression (Before GROUP BY)
376-
Filter ((WHERE + Change column names to column identifiers))
376+
Expression
377377
ReadFromMergeTree (default.test)
378378
Indexes:
379379
PrimaryKey
@@ -485,7 +485,7 @@ ReadFromPreparedSource (_minmax_count_projection)
485485
Expression ((Project names + Projection))
486486
Aggregating
487487
Expression (Before GROUP BY)
488-
Filter ((WHERE + Change column names to column identifiers))
488+
Expression
489489
ReadFromMergeTree (default.test)
490490
Indexes:
491491
PrimaryKey
@@ -582,7 +582,7 @@ Sorting: json.a DESC
582582
0 {"a":0,"b":"str_0","c":[]} Hello, World!
583583
2 {"a":0,"b":"str_2","c":["0","1"]} Hello, World!
584584
Expression ((Project names + Projection))
585-
Filter ((WHERE + Change column names to column identifiers))
585+
Expression
586586
ReadFromMergeTree (default.test)
587587
Indexes:
588588
PrimaryKey
@@ -593,7 +593,7 @@ Parts: 1/2
593593
Granules: 2/8
594594
0 {"a":0,"b":"str_0","c":[]} Hello, World!
595595
Expression ((Project names + Projection))
596-
Filter ((WHERE + Change column names to column identifiers))
596+
Expression
597597
ReadFromMergeTree (default.test)
598598
Indexes:
599599
PrimaryKey
@@ -621,7 +621,7 @@ ReadFromPreparedSource (_minmax_count_projection)
621621
Expression ((Project names + Projection))
622622
Aggregating
623623
Expression (Before GROUP BY)
624-
Filter ((WHERE + Change column names to column identifiers))
624+
Expression
625625
ReadFromMergeTree (default.test)
626626
Indexes:
627627
PrimaryKey
@@ -733,7 +733,7 @@ ReadFromPreparedSource (_minmax_count_projection)
733733
Expression ((Project names + Projection))
734734
Aggregating
735735
Expression (Before GROUP BY)
736-
Filter ((WHERE + Change column names to column identifiers))
736+
Expression
737737
ReadFromMergeTree (default.test)
738738
Indexes:
739739
PrimaryKey

‎tests/queries/0_stateless/03356_array_join_subcolumns_indexes.reference‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
[0,1,2]
55
Expression ((Project names + (Projection + )))
66
ArrayJoin (ARRAY JOIN)
7-
Filter ((( + WHERE) + (DROP unused columns before ARRAY JOIN + (ARRAY JOIN actions + Change column names to column identifiers))))
7+
Expression
88
ReadFromMergeTree (default.test)
99
Indexes:
1010
Skip

0 commit comments

Comments
 (0)