Skip to content

Commit 27f3f97

Browse files
robot-clickhouserobot-clickhouse
committed
Backport ClickHouse#86184 to 25.8: Fix inferring Date/DateTime/DateTime64 on dates that are out of supported range
1 parent ab168b0 commit 27f3f97

22 files changed

+266
-121
lines changed

‎src/Common/DateLUT.cpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,11 @@ ExtendedDayNum makeDayNum(const DateLUTImpl & date_lut, Int16 year, UInt8 month,
209209
return date_lut.makeDayNum(year, month, day_of_month, default_error_day_num);
210210
}
211211

212+
std::optional<ExtendedDayNum> tryToMakeDayNum(const DateLUTImpl & date_lut, Int16 year, UInt8 month, UInt8 day_of_month)
213+
{
214+
return date_lut.tryToMakeDayNum(year, month, day_of_month);
215+
}
216+
212217
Int64 makeDate(const DateLUTImpl & date_lut, Int16 year, UInt8 month, UInt8 day_of_month)
213218
{
214219
static_assert(std::same_as<Int64, DateLUTImpl::Time>);
@@ -221,6 +226,12 @@ Int64 makeDateTime(const DateLUTImpl & date_lut, Int16 year, UInt8 month, UInt8
221226
return date_lut.makeDateTime(year, month, day_of_month, hour, minute, second);
222227
}
223228

229+
std::optional<Int64> tryToMakeDateTime(const DateLUTImpl & date_lut, Int16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second)
230+
{
231+
static_assert(std::same_as<Int64, DateLUTImpl::Time>);
232+
return date_lut.tryToMakeDateTime(year, month, day_of_month, hour, minute, second);
233+
}
234+
224235
const std::string & getDateLUTTimeZone(const DateLUTImpl & date_lut)
225236
{
226237
return date_lut.getTimeZone();

‎src/Common/DateLUT.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,11 @@ inline UInt64 timeInNanoseconds(std::chrono::time_point<std::chrono::system_cloc
8787
/// A few helper functions to avoid having to include DateLUTImpl.h in some heavy headers
8888

8989
ExtendedDayNum makeDayNum(const DateLUTImpl & date_lut, Int16 year, UInt8 month, UInt8 day_of_month, Int32 default_error_day_num = 0);
90+
std::optional<ExtendedDayNum> tryToMakeDayNum(const DateLUTImpl & date_lut, Int16 year, UInt8 month, UInt8 day_of_month);
9091

9192
Int64 makeDate(const DateLUTImpl & date_lut, Int16 year, UInt8 month, UInt8 day_of_month);
9293
Int64 makeDateTime(const DateLUTImpl & date_lut, Int16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second);
94+
std::optional<Int64> tryToMakeDateTime(const DateLUTImpl & date_lut, Int16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second);
9395

9496
const std::string & getDateLUTTimeZone(const DateLUTImpl & date_lut);
9597
UInt32 getDayNumOffsetEpoch();

‎src/Common/DateLUTImpl.h‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1177,6 +1177,20 @@ class DateLUTImpl
11771177
return LUTIndex{std::min(index, static_cast<UInt32>(DATE_LUT_SIZE - 1))};
11781178
}
11791179

1180+
std::optional<LUTIndex> tryToMakeLUTIndex(Int16 year, UInt8 month, UInt8 day_of_month) const
1181+
{
1182+
if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31))
1183+
return std::nullopt;
1184+
1185+
auto year_lut_index = (year - DATE_LUT_MIN_YEAR) * 12 + month - 1;
1186+
UInt32 index = years_months_lut[year_lut_index].toUnderType() + day_of_month - 1;
1187+
1188+
if (index >= DATE_LUT_SIZE)
1189+
return std::nullopt;
1190+
1191+
return LUTIndex(index);
1192+
}
1193+
11801194
/// Create DayNum from year, month, day of month.
11811195
ExtendedDayNum makeDayNum(Int16 year, UInt8 month, UInt8 day_of_month, Int32 default_error_day_num = 0) const
11821196
{
@@ -1186,6 +1200,18 @@ class DateLUTImpl
11861200
return toDayNum(makeLUTIndex(year, month, day_of_month));
11871201
}
11881202

1203+
std::optional<ExtendedDayNum> tryToMakeDayNum(Int16 year, UInt8 month, UInt8 day_of_month) const
1204+
{
1205+
if (unlikely(year < DATE_LUT_MIN_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31))
1206+
return std::nullopt;
1207+
1208+
auto index = tryToMakeLUTIndex(year, month, day_of_month);
1209+
if (!index)
1210+
return std::nullopt;
1211+
1212+
return toDayNum(*index);
1213+
}
1214+
11891215
Time makeDate(Int16 year, UInt8 month, UInt8 day_of_month) const
11901216
{
11911217
return lut[makeLUTIndex(year, month, day_of_month)].date;
@@ -1204,6 +1230,20 @@ class DateLUTImpl
12041230
return lut[index].date + time_offset;
12051231
}
12061232

1233+
std::optional<Time> tryToMakeDateTime(Int16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second) const
1234+
{
1235+
auto index = tryToMakeLUTIndex(year, month, day_of_month);
1236+
if (!index)
1237+
return std::nullopt;
1238+
1239+
Time time_offset = hour * 3600 + minute * 60 + second;
1240+
1241+
if (time_offset >= lut[*index].time_at_offset_change())
1242+
time_offset -= lut[*index].amount_of_offset_change();
1243+
1244+
return lut[*index].date + time_offset;
1245+
}
1246+
12071247
Time makeTime(int64_t hour, UInt8 minute, UInt8 second) const
12081248
{
12091249
Time time_offset = hour * 3600 + minute * 60 + second;

‎src/IO/ReadHelpers.cpp‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1567,10 +1567,25 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
15671567
second = (s[6] - '0') * 10 + (s[7] - '0');
15681568
}
15691569

1570-
if (unlikely(year == 0))
1571-
datetime = 0;
1570+
if constexpr (throw_exception)
1571+
{
1572+
if (unlikely(year == 0))
1573+
datetime = 0;
1574+
else
1575+
datetime = makeDateTime(date_lut, year, month, day, hour, minute, second);
1576+
}
15721577
else
1573-
datetime = makeDateTime(date_lut, year, month, day, hour, minute, second);
1578+
{
1579+
auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second);
1580+
if (!datetime_maybe)
1581+
return false;
1582+
1583+
/// For usual DateTime check if value is within supported range
1584+
if (!dt64_mode && (*datetime_maybe < 0 || *datetime_maybe > UINT32_MAX))
1585+
return false;
1586+
1587+
datetime = *datetime_maybe;
1588+
}
15741589
}
15751590
else
15761591
{

‎src/IO/ReadHelpers.h‎

Lines changed: 44 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -609,6 +609,15 @@ inline void convertToDayNum(DayNum & date, ExtendedDayNum & from)
609609
date = from;
610610
}
611611

612+
inline bool tryToConvertToDayNum(DayNum & date, ExtendedDayNum & from)
613+
{
614+
if (unlikely(from < 0 || from > 0xFFFF))
615+
return false;
616+
617+
date = from;
618+
return true;
619+
}
620+
612621
template <typename ReturnType = void>
613622
inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_delimiters = nullptr)
614623
{
@@ -617,13 +626,25 @@ inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf, const DateLU
617626
LocalDate local_date;
618627

619628
if constexpr (throw_exception)
629+
{
620630
readDateTextImpl<ReturnType>(local_date, buf, allowed_delimiters);
621-
else if (!readDateTextImpl<ReturnType>(local_date, buf, allowed_delimiters))
622-
return false;
631+
ExtendedDayNum ret = makeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day());
632+
convertToDayNum(date, ret);
633+
}
634+
else
635+
{
636+
if (!readDateTextImpl<ReturnType>(local_date, buf, allowed_delimiters))
637+
return false;
623638

624-
ExtendedDayNum ret = makeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day());
625-
convertToDayNum(date, ret);
626-
return ReturnType(true);
639+
auto ret = tryToMakeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day());
640+
if (!ret)
641+
return false;
642+
643+
if (!tryToConvertToDayNum(date, *ret))
644+
return false;
645+
646+
return true;
647+
}
627648
}
628649

629650
template <typename ReturnType = void>
@@ -868,10 +889,25 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons
868889
second = (s[17] - '0') * 10 + (s[18] - '0');
869890
}
870891

871-
if (unlikely(year == 0))
872-
datetime = 0;
892+
if constexpr (throw_exception)
893+
{
894+
if (unlikely(year == 0))
895+
datetime = 0;
896+
else
897+
datetime = makeDateTime(date_lut, year, month, day, hour, minute, second);
898+
}
873899
else
874-
datetime = makeDateTime(date_lut, year, month, day, hour, minute, second);
900+
{
901+
auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second);
902+
if (!datetime_maybe)
903+
return false;
904+
905+
/// For usual DateTime check if value is within supported range
906+
if (!dt64_mode && (*datetime_maybe < 0 || *datetime_maybe > UINT32_MAX))
907+
return false;
908+
909+
datetime = *datetime_maybe;
910+
}
875911

876912
if (dt_long)
877913
buf.position() += date_time_broken_down_length;

‎src/IO/parseDateTimeBestEffort.cpp‎

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -753,31 +753,49 @@ ReturnType parseDateTimeBestEffortImpl(
753753
}
754754
};
755755

756-
if constexpr (strict)
756+
if constexpr (std::is_same_v<ReturnType, void>)
757757
{
758-
if constexpr (is_64)
758+
if (has_time_zone_offset)
759759
{
760-
if (year < 1900)
761-
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime64: year {} is less than minimum supported year 1900", year);
760+
res = utc_time_zone.makeDateTime(year, month, day_of_month, hour, minute, second);
761+
adjust_time_zone();
762762
}
763763
else
764764
{
765-
if (year < 1970)
766-
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: year {} is less than minimum supported year 1970", year);
765+
res = local_time_zone.makeDateTime(year, month, day_of_month, hour, minute, second);
767766
}
768767
}
769-
770-
if (has_time_zone_offset)
771-
{
772-
res = utc_time_zone.makeDateTime(year, month, day_of_month, hour, minute, second);
773-
adjust_time_zone();
774-
}
775768
else
776769
{
777-
res = local_time_zone.makeDateTime(year, month, day_of_month, hour, minute, second);
778-
}
779770

780-
return ReturnType(true);
771+
if (has_time_zone_offset)
772+
{
773+
auto res_maybe = utc_time_zone.tryToMakeDateTime(year, month, day_of_month, hour, minute, second);
774+
if (!res_maybe)
775+
return false;
776+
777+
/// For usual DateTime check if value is within supported range
778+
if (!is_64 && (*res_maybe < 0 || *res_maybe > UINT32_MAX))
779+
return false;
780+
781+
res = *res_maybe;
782+
adjust_time_zone();
783+
}
784+
else
785+
{
786+
auto res_maybe = local_time_zone.tryToMakeDateTime(year, month, day_of_month, hour, minute, second);
787+
if (!res_maybe)
788+
return false;
789+
790+
/// For usual DateTime check if value is within supported range
791+
if (!is_64 && (*res_maybe < 0 || *res_maybe > UINT32_MAX))
792+
return false;
793+
794+
res = *res_maybe;
795+
}
796+
797+
return true;
798+
}
781799
}
782800

783801
template <typename ReturnType, bool is_us_style, bool strict = false>

‎tests/integration/test_storage_kafka/test_batch_fast.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1368,7 +1368,7 @@ def test_kafka_virtual_columns_with_materialized_view(
13681368
ENGINE = MergeTree()
13691369
ORDER BY key;
13701370
CREATE MATERIALIZED VIEW test.consumer TO test.view AS
1371-
SELECT *, _key as kafka_key, _topic as topic, _offset as offset, _partition as partition, _timestamp = 0 ? '0000-00-00 00:00:00' : toString(_timestamp) as timestamp FROM test.kafka;
1371+
SELECT *, _key as kafka_key, _topic as topic, _offset as offset, _partition as partition, _timestamp = 0 ? '1970-01-01 00:00:00' : toString(_timestamp) as timestamp FROM test.kafka;
13721372
"""
13731373
)
13741374

‎tests/queries/0_stateless/01186_conversion_to_nullable.reference‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@
1010
256
1111
2020-12-24
1212
\N
13-
1970-01-01
1413
\N
15-
2149-06-06
14+
\N
15+
\N
1616
2020-12-24 01:02:03
1717
\N
18-
1970-01-01 02:00:00
18+
\N
1919
\N
2020
2020-12-24 01:02:03.00
2121
\N

‎tests/queries/0_stateless/01556_accurate_cast_or_null.reference‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,13 @@
3636
2023-05-30 14:38:20
3737
1970-01-01 00:00:19
3838
1970-01-01 19:26:40
39-
1970-01-01 00:00:00
40-
2106-02-07 06:28:15
39+
\N
40+
\N
4141
\N
4242
\N
4343
\N
4444
2023-05-30
45-
2149-06-06
45+
\N
4646
1970-01-20
4747
\N
4848
\N

‎tests/queries/0_stateless/02404_data.CSV‎

Lines changed: 0 additions & 10 deletions
This file was deleted.

0 commit comments

Comments
 (0)