Skip to content

Commit 3663793

Browse files
azatazat
authored
Merge pull request ClickHouse#88487 from ClickHouse/backport/25.3/87987
Backport ClickHouse#87987 to 25.3: Fix index analysis with session_timezone specified
2 parents 6cecffc + 565280b commit 3663793

File tree

4 files changed

+180
-23
lines changed

4 files changed

+180
-23
lines changed

‎src/Storages/MergeTree/KeyCondition.cpp‎

Lines changed: 32 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ namespace Setting
5353
{
5454
extern const SettingsBool analyze_index_with_space_filling_curves;
5555
extern const SettingsDateTimeOverflowBehavior date_time_overflow_behavior;
56+
extern const SettingsTimezone session_timezone;
5657
}
5758

5859
namespace ErrorCodes
@@ -1048,7 +1049,7 @@ bool applyFunctionChainToColumn(
10481049
}
10491050

10501051
// And cast it to the argument type of the first function in the chain
1051-
auto in_argument_type = getArgumentTypeOfMonotonicFunction(*functions[0]);
1052+
auto in_argument_type = removeLowCardinality(getArgumentTypeOfMonotonicFunction(*functions[0]));
10521053
if (canBeSafelyCast(result_type, in_argument_type))
10531054
{
10541055
result_column = castColumnAccurate({result_column, result_type, ""}, in_argument_type);
@@ -1077,13 +1078,13 @@ bool applyFunctionChainToColumn(
10771078
if (func->getArgumentTypes().empty())
10781079
return false;
10791080

1080-
auto argument_type = getArgumentTypeOfMonotonicFunction(*func);
1081+
auto argument_type = removeLowCardinality(getArgumentTypeOfMonotonicFunction(*func));
10811082
if (!canBeSafelyCast(result_type, argument_type))
10821083
return false;
10831084

10841085
result_column = castColumnAccurate({result_column, result_type, ""}, argument_type);
1085-
result_column = func->execute({{result_column, argument_type, ""}}, func->getResultType(), result_column->size(), /* dry_run = */ false);
1086-
result_type = func->getResultType();
1086+
result_type = removeLowCardinality(func->getResultType());
1087+
result_column = func->execute({{result_column, argument_type, ""}}, result_type, result_column->size(), /* dry_run = */ false);
10871088

10881089
// Transforming nullable columns to the nested ones, in case no nulls found
10891090
if (result_column->isNullable())
@@ -1096,7 +1097,7 @@ bool applyFunctionChainToColumn(
10961097
return false;
10971098
}
10981099
result_column = result_column_nullable.getNestedColumnPtr();
1099-
result_type = removeNullable(func->getResultType());
1100+
result_type = removeNullable(result_type);
11001101
}
11011102
}
11021103
out_column = result_column;
@@ -1854,48 +1855,57 @@ bool KeyCondition::extractMonotonicFunctionsChainFromKey(
18541855
auto func_name = func->function_base->getName();
18551856
auto func_base = func->function_base;
18561857

1857-
ColumnsWithTypeAndName arguments;
18581858
ColumnWithTypeAndName const_arg;
18591859
FunctionWithOptionalConstArg::Kind kind = FunctionWithOptionalConstArg::Kind::NO_CONST;
18601860

18611861
if (date_time_parsing_functions.contains(func_name))
18621862
{
1863-
const auto & arg_types = func_base->getArgumentTypes();
1864-
if (!arg_types.empty() && isStringOrFixedString(arg_types[0]))
1865-
func_name = func_name + "OrNull";
1866-
}
1863+
const auto & func_arg_types = func_base->getArgumentTypes();
18671864

1868-
auto func_builder = FunctionFactory::instance().tryGet(func_name, context);
1865+
const bool has_string_argument = !func_arg_types.empty() && isStringOrFixedString(func_arg_types[0]);
1866+
const bool has_session_timezone = !context->getSettingsRef()[Setting::session_timezone].value.empty();
18691867

1870-
if (func->children.size() == 1)
1871-
{
1872-
arguments.push_back({nullptr, removeLowCardinality(func->children[0]->result_type), ""});
1868+
// Skipping analysis in case when is requires parsing datetime from string
1869+
// with `session_timezone` specified
1870+
if (has_string_argument && has_session_timezone)
1871+
return false;
1872+
1873+
// Otherwise, in case when datetime parsing is required, rebuilding the function,
1874+
// to get its "-OrNull" version required for safe parsing, and not failing on
1875+
// values with incorrect format
1876+
if (has_string_argument)
1877+
{
1878+
ColumnsWithTypeAndName new_args;
1879+
for (const auto & type : func->function_base->getArgumentTypes())
1880+
new_args.push_back({nullptr, type, ""});
1881+
1882+
const auto func_builder = FunctionFactory::instance().tryGet(func_name + "OrNull", context);
1883+
func_base = func_builder->build(new_args);
1884+
}
18731885
}
1874-
else if (func->children.size() == 2)
1886+
1887+
// For single argument functions, the input may be used as-is, for binary functions,
1888+
// we'll produce a partially applied version of `func` with the reduced arity
1889+
if (func->children.size() == 2)
18751890
{
18761891
const auto * left = func->children[0];
18771892
const auto * right = func->children[1];
18781893
if (left->column && isColumnConst(*left->column))
18791894
{
18801895
const_arg = {left->result_type->createColumnConst(0, (*left->column)[0]), left->result_type, ""};
1881-
arguments.push_back(const_arg);
1882-
arguments.push_back({nullptr, removeLowCardinality(right->result_type), ""});
18831896
kind = FunctionWithOptionalConstArg::Kind::LEFT_CONST;
18841897
}
18851898
else
18861899
{
18871900
const_arg = {right->result_type->createColumnConst(0, (*right->column)[0]), right->result_type, ""};
1888-
arguments.push_back({nullptr, removeLowCardinality(left->result_type), ""});
1889-
arguments.push_back(const_arg);
18901901
kind = FunctionWithOptionalConstArg::Kind::RIGHT_CONST;
18911902
}
18921903
}
18931904

1894-
auto out_func = func_builder->build(arguments);
18951905
if (kind == FunctionWithOptionalConstArg::Kind::NO_CONST)
1896-
out_functions_chain.push_back(out_func);
1906+
out_functions_chain.push_back(func_base);
18971907
else
1898-
out_functions_chain.push_back(std::make_shared<FunctionWithOptionalConstArg>(out_func, const_arg, kind));
1908+
out_functions_chain.push_back(std::make_shared<FunctionWithOptionalConstArg>(func_base, const_arg, kind));
18991909
}
19001910

19011911
out_key_column_num = it->second;

‎tests/queries/0_stateless/03173_set_transformed_partition_pruning.sql‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
-- Tags: no-msan, long
2+
-- msan: too slow
3+
14
SELECT '-- Single partition by function';
25

36
DROP TABLE IF EXISTS 03173_single_function;
@@ -231,7 +234,7 @@ SELECT toString(toDate('2000-01-01') + 10 * number) FROM numbers(50)
231234
UNION ALL
232235
SELECT toString(toDate('2100-01-01') + 10 * number) FROM numbers(50);
233236

234-
SELECT count() FROM 03173_nested_date_parsing WHERE id IN ('2000-01-21', '2023-05-02') SETTINGS log_comment='03173_nested_date_parsing';
237+
SELECT count() FROM 03173_nested_date_parsing WHERE id IN ('2000-01-21', '2023-05-02') SETTINGS log_comment='03173_nested_date_parsing', session_timezone = '';
235238
SYSTEM FLUSH LOGS;
236239
SELECT ProfileEvents['SelectedParts'] FROM system.query_log WHERE type = 'QueryFinish' AND current_database = currentDatabase() AND log_comment = '03173_nested_date_parsing';
237240
SELECT count() FROM 03173_nested_date_parsing WHERE id IN ('not a date');

‎tests/queries/0_stateless/03636_index_analysis_with_session_tz.reference‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
-- PK UTC timezone
2+
1
3+
Condition: (toStartOfDay(ts) in [1756857600, 1756857600])
4+
Parts: 1/1
5+
Granules: 1/1
6+
7+
-- PK EST timezone
8+
1
9+
Condition: (toStartOfDay(ts) in [1756857600, 1756857600])
10+
Parts: 1/1
11+
Granules: 1/1
12+
13+
-- Partitions UTC timezone
14+
1
15+
Condition: (ts in [1756882680, 1756882680])
16+
Parts: 1/1
17+
Granules: 1/1
18+
Condition: (toStartOfDay(ts) in [1756857600, 1756857600])
19+
Parts: 1/1
20+
Granules: 1/1
21+
22+
-- Partitions EST timezone
23+
1
24+
Condition: (ts in [1756882680, 1756882680])
25+
Parts: 1/1
26+
Granules: 1/1
27+
Condition: (toStartOfDay(ts) in [1756857600, 1756857600])
28+
Parts: 1/1
29+
Granules: 1/1
30+
31+
-- Partitions UTC timezone
32+
1
33+
Condition: true
34+
Parts: 1/1
35+
Granules: 1/1
36+
37+
-- Partitions EST timezone
38+
1
39+
Condition: true
40+
Parts: 1/1
41+
Granules: 1/1

‎tests/queries/0_stateless/03636_index_analysis_with_session_tz.sql‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
SET session_timezone = 'UTC';
2+
-- For explain with indexes and key condition values verification
3+
SET parallel_replicas_local_plan = 1;
4+
5+
DROP TABLE IF EXISTS 03636_data_pk, 03636_data_partitions, 03636_data_parsed;
6+
7+
CREATE TABLE 03636_data_pk (ts DateTime) ENGINE = MergeTree ORDER BY toStartOfDay(ts)
8+
AS
9+
SELECT 1756882680;
10+
11+
SELECT '-- PK UTC timezone';
12+
13+
SELECT count() FROM 03636_data_pk WHERE ts = 1756882680;
14+
15+
SELECT trim(explain)
16+
FROM (
17+
EXPLAIN indexes = 1 SELECT count() FROM 03636_data_pk WHERE ts = 1756882680
18+
)
19+
WHERE trim(explain) ilike 'condition: %'
20+
OR trim(explain) ilike 'parts: %'
21+
OR trim(explain) ilike 'granules: %';
22+
23+
SELECT '';
24+
SELECT '-- PK EST timezone';
25+
26+
SELECT count() FROM 03636_data_pk WHERE ts = 1756882680 SETTINGS session_timezone = 'EST';
27+
28+
SELECT trim(explain)
29+
FROM (
30+
EXPLAIN indexes = 1 SELECT count() FROM 03636_data_pk WHERE ts = 1756882680
31+
)
32+
WHERE trim(explain) ilike 'condition: %'
33+
OR trim(explain) ilike 'parts: %'
34+
OR trim(explain) ilike 'granules: %'
35+
SETTINGS session_timezone = 'EST';
36+
37+
DROP TABLE 03636_data_pk;
38+
39+
CREATE TABLE 03636_data_partitions (ts DateTime) ENGINE = MergeTree ORDER BY tuple() PARTITION BY toStartOfDay(ts)
40+
AS
41+
SELECT 1756882680;
42+
43+
SELECT '';
44+
SELECT '-- Partitions UTC timezone';
45+
46+
SELECT count() FROM 03636_data_partitions WHERE ts = 1756882680;
47+
48+
SELECT trim(explain)
49+
FROM (
50+
EXPLAIN indexes = 1 SELECT count() FROM 03636_data_partitions WHERE ts = 1756882680
51+
)
52+
WHERE trim(explain) ilike 'condition: %'
53+
OR trim(explain) ilike 'parts: %'
54+
OR trim(explain) ilike 'granules: %';
55+
56+
SELECT '';
57+
SELECT '-- Partitions EST timezone';
58+
59+
SELECT count() FROM 03636_data_partitions WHERE ts = 1756882680 SETTINGS session_timezone = 'EST';
60+
61+
SELECT trim(explain)
62+
FROM (
63+
EXPLAIN indexes = 1 SELECT count() FROM 03636_data_partitions WHERE ts = 1756882680
64+
)
65+
WHERE trim(explain) ilike 'condition: %'
66+
OR trim(explain) ilike 'parts: %'
67+
OR trim(explain) ilike 'granules: %'
68+
SETTINGS session_timezone = 'EST';
69+
70+
DROP TABLE 03636_data_partitions;
71+
72+
CREATE TABLE 03636_data_parsed (ts String) ENGINE = MergeTree ORDER BY toStartOfDay(toDateTime(ts))
73+
AS
74+
SELECT '2025-09-02 19:00:00';
75+
76+
SELECT '';
77+
SELECT '-- Partitions UTC timezone';
78+
79+
SELECT count() FROM 03636_data_parsed WHERE ts = '2025-09-02 19:00:00';
80+
81+
SELECT trim(explain)
82+
FROM (
83+
EXPLAIN indexes = 1 SELECT count() FROM 03636_data_parsed WHERE ts = '2025-09-02 19:00:00'
84+
)
85+
WHERE trim(explain) ilike 'condition: %'
86+
OR trim(explain) ilike 'parts: %'
87+
OR trim(explain) ilike 'granules: %';
88+
89+
SELECT '';
90+
SELECT '-- Partitions EST timezone';
91+
92+
SELECT count() FROM 03636_data_parsed WHERE ts = '2025-09-02 19:00:00' SETTINGS session_timezone = 'EST';
93+
94+
SELECT trim(explain)
95+
FROM (
96+
EXPLAIN indexes = 1 SELECT count() FROM 03636_data_parsed WHERE ts = '2025-09-02 19:00:00'
97+
)
98+
WHERE trim(explain) ilike 'condition: %'
99+
OR trim(explain) ilike 'parts: %'
100+
OR trim(explain) ilike 'granules: %'
101+
SETTINGS session_timezone = 'EST';
102+
103+
DROP TABLE 03636_data_parsed;

0 commit comments

Comments
 (0)