Skip to content

Commit 1d2e117

Browse files
zvonandzvonand
authored
Merge pull request #1225 from Altinity/backports/25.8.12/91870
25.8.12 Stable backport of ClickHouse#90490: Fix a bug about `toDateTimeOrNull` with negative argument
2 parents 7055c13 + 03c361d commit 1d2e117

9 files changed

+210
-62
lines changed

‎src/Formats/SchemaInferenceUtils.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -791,7 +791,7 @@ namespace
791791

792792
ReadBufferFromString buf(field);
793793
DayNum tmp;
794-
return tryReadDateText(tmp, buf, DateLUT::instance(), /*allowed_delimiters=*/"-/:") && buf.eof();
794+
return tryReadDateText(tmp, buf, DateLUT::instance(), /*allowed_delimiters=*/"-/:", /*saturate_on_overflow=*/false) && buf.eof();
795795
}
796796

797797
DataTypePtr tryInferDateTimeOrDateTime64(std::string_view field, const FormatSettings & settings)
@@ -827,7 +827,7 @@ namespace
827827
switch (settings.date_time_input_format)
828828
{
829829
case FormatSettings::DateTimeInputFormat::Basic:
830-
if (tryReadDateTimeText(tmp, buf, DateLUT::instance(), /*allowed_date_delimiters=*/"-/:", /*allowed_time_delimiters=*/":") && buf.eof())
830+
if (tryReadDateTimeText(tmp, buf, DateLUT::instance(), /*allowed_date_delimiters=*/"-/:", /*allowed_time_delimiters=*/":", /*saturate_on_overflow=*/false) && buf.eof())
831831
return std::make_shared<DataTypeDateTime>();
832832
break;
833833
case FormatSettings::DateTimeInputFormat::BestEffort:
@@ -846,7 +846,7 @@ namespace
846846
switch (settings.date_time_input_format)
847847
{
848848
case FormatSettings::DateTimeInputFormat::Basic:
849-
if (tryReadDateTime64Text(tmp, 9, buf, DateLUT::instance(), /*allowed_date_delimiters=*/"-/:", /*allowed_time_delimiters=*/":") && buf.eof())
849+
if (tryReadDateTime64Text(tmp, 9, buf, DateLUT::instance(), /*allowed_date_delimiters=*/"-/:", /*allowed_time_delimiters=*/":", /*saturate_on_overflow=*/false) && buf.eof())
850850
return std::make_shared<DataTypeDateTime64>(9);
851851
break;
852852
case FormatSettings::DateTimeInputFormat::BestEffort:

‎src/IO/ReadHelpers.cpp‎

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1458,7 +1458,13 @@ template bool readDateTextFallback<bool>(LocalDate &, ReadBuffer &, const char *
14581458

14591459

14601460
template <typename ReturnType, bool dt64_mode>
1461-
ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters, const char * allowed_time_delimiters)
1461+
ReturnType readDateTimeTextFallback(
1462+
time_t & datetime,
1463+
ReadBuffer & buf,
1464+
const DateLUTImpl & date_lut,
1465+
const char * allowed_date_delimiters,
1466+
const char * allowed_time_delimiters,
1467+
bool saturate_on_overflow)
14621468
{
14631469
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
14641470

@@ -1576,15 +1582,29 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
15761582
}
15771583
else
15781584
{
1579-
auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second);
1580-
if (!datetime_maybe)
1581-
return false;
1585+
if (saturate_on_overflow)
1586+
{
1587+
/// Use saturating version - makeDateTime saturates out-of-range years
1588+
if (unlikely(year == 0))
1589+
datetime = 0;
1590+
else
1591+
datetime = makeDateTime(date_lut, year, month, day, hour, minute, second);
1592+
}
1593+
else
1594+
{
1595+
/// Use non-saturating version - return false for out-of-range values
1596+
auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second);
1597+
if (!datetime_maybe)
1598+
return false;
15821599

1583-
/// For usual DateTime check if value is within supported range
1584-
if (!dt64_mode && (*datetime_maybe < 0 || *datetime_maybe > UINT32_MAX))
1585-
return false;
1600+
if constexpr (!dt64_mode)
1601+
{
1602+
if (*datetime_maybe < 0 || *datetime_maybe > static_cast<Int64>(UINT32_MAX))
1603+
return false;
1604+
}
15861605

1587-
datetime = *datetime_maybe;
1606+
datetime = *datetime_maybe;
1607+
}
15881608
}
15891609
}
15901610
else
@@ -1620,10 +1640,10 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
16201640
return ReturnType(true);
16211641
}
16221642

1623-
template void readDateTimeTextFallback<void, false>(time_t &, ReadBuffer &, const DateLUTImpl &, const char *, const char *);
1624-
template void readDateTimeTextFallback<void, true>(time_t &, ReadBuffer &, const DateLUTImpl &, const char *, const char *);
1625-
template bool readDateTimeTextFallback<bool, false>(time_t &, ReadBuffer &, const DateLUTImpl &, const char *, const char *);
1626-
template bool readDateTimeTextFallback<bool, true>(time_t &, ReadBuffer &, const DateLUTImpl &, const char *, const char *);
1643+
template void readDateTimeTextFallback<void, false>(time_t &, ReadBuffer &, const DateLUTImpl &, const char *, const char *, bool);
1644+
template void readDateTimeTextFallback<void, true>(time_t &, ReadBuffer &, const DateLUTImpl &, const char *, const char *, bool);
1645+
template bool readDateTimeTextFallback<bool, false>(time_t &, ReadBuffer &, const DateLUTImpl &, const char *, const char *, bool);
1646+
template bool readDateTimeTextFallback<bool, true>(time_t &, ReadBuffer &, const DateLUTImpl &, const char *, const char *, bool);
16271647

16281648
template <typename ReturnType, bool t64_mode>
16291649
ReturnType readTimeTextFallback(time_t & time, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters, const char * allowed_time_delimiters)

‎src/IO/ReadHelpers.h‎

Lines changed: 52 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -619,7 +619,7 @@ inline bool tryToConvertToDayNum(DayNum & date, ExtendedDayNum & from)
619619
}
620620

621621
template <typename ReturnType = void>
622-
inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_delimiters = nullptr)
622+
inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_delimiters = nullptr, bool saturate_on_overflow = true)
623623
{
624624
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
625625

@@ -636,12 +636,22 @@ inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf, const DateLU
636636
if (!readDateTextImpl<ReturnType>(local_date, buf, allowed_delimiters))
637637
return false;
638638

639-
auto ret = tryToMakeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day());
640-
if (!ret)
641-
return false;
639+
if (saturate_on_overflow)
640+
{
641+
/// Use saturating versions - makeDayNum saturates out-of-range years, convertToDayNum saturates to 0 or 0xFFFF
642+
ExtendedDayNum ret = makeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day());
643+
convertToDayNum(date, ret);
644+
}
645+
else
646+
{
647+
/// Use non-saturating versions - return false for out-of-range values
648+
auto ret = tryToMakeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day());
649+
if (!ret)
650+
return false;
642651

643-
if (!tryToConvertToDayNum(date, *ret))
644-
return false;
652+
if (!tryToConvertToDayNum(date, *ret))
653+
return false;
654+
}
645655

646656
return true;
647657
}
@@ -685,9 +695,9 @@ inline bool tryReadDateText(LocalDate & date, ReadBuffer & buf, const char * all
685695
return readDateTextImpl<bool>(date, buf, allowed_delimiters);
686696
}
687697

688-
inline bool tryReadDateText(DayNum & date, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), const char * allowed_delimiters = nullptr)
698+
inline bool tryReadDateText(DayNum & date, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), const char * allowed_delimiters = nullptr, bool saturate_on_overflow = true)
689699
{
690-
return readDateTextImpl<bool>(date, buf, time_zone, allowed_delimiters);
700+
return readDateTextImpl<bool>(date, buf, time_zone, allowed_delimiters, saturate_on_overflow);
691701
}
692702

693703
inline bool tryReadDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), const char * allowed_delimiters = nullptr)
@@ -815,7 +825,7 @@ inline T parseFromStringWithoutAssertEOF(std::string_view str)
815825
}
816826

817827
template <typename ReturnType = void, bool dt64_mode = false>
818-
ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr);
828+
ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr, bool saturate_on_overflow = true);
819829

820830
template <typename ReturnType = void, bool t64_mode = false>
821831
ReturnType readTimeTextFallback(time_t & time, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr);
@@ -824,7 +834,7 @@ ReturnType readTimeTextFallback(time_t & time, ReadBuffer & buf, const DateLUTIm
824834
* As an exception, also supported parsing of unix timestamp in form of decimal number.
825835
*/
826836
template <typename ReturnType = void, bool dt64_mode = false>
827-
inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr)
837+
inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr, bool saturate_on_overflow = true)
828838
{
829839
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
830840

@@ -898,15 +908,30 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons
898908
}
899909
else
900910
{
901-
auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second);
902-
if (!datetime_maybe)
903-
return false;
904-
905-
/// For usual DateTime check if value is within supported range
906-
if (!dt64_mode && (*datetime_maybe < 0 || *datetime_maybe > UINT32_MAX))
907-
return false;
908-
909-
datetime = *datetime_maybe;
911+
if (saturate_on_overflow)
912+
{
913+
/// Use saturating version - makeDateTime saturates out-of-range years
914+
if (unlikely(year == 0))
915+
datetime = 0;
916+
else
917+
datetime = makeDateTime(date_lut, year, month, day, hour, minute, second);
918+
}
919+
else
920+
{
921+
/// Use non-saturating version - return false for out-of-range values
922+
auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second);
923+
if (!datetime_maybe)
924+
return false;
925+
926+
/// For usual DateTime check if value is within supported range
927+
if constexpr (!dt64_mode)
928+
{
929+
if (*datetime_maybe < 0 || *datetime_maybe > static_cast<Int64>(UINT32_MAX))
930+
return false;
931+
}
932+
933+
datetime = *datetime_maybe;
934+
}
910935
}
911936

912937
if (dt_long)
@@ -919,7 +944,7 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons
919944
/// Why not readIntTextUnsafe? Because for needs of AdFox, parsing of unix timestamp with leading zeros is supported: 000...NNNN.
920945
return readIntTextImpl<time_t, ReturnType, ReadIntTextCheckOverflow::CHECK_OVERFLOW>(datetime, buf);
921946
}
922-
return readDateTimeTextFallback<ReturnType, dt64_mode>(datetime, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters);
947+
return readDateTimeTextFallback<ReturnType, dt64_mode>(datetime, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters, saturate_on_overflow);
923948
}
924949

925950
/** In hhh:mm:ss format, according to specified time zone.
@@ -1107,7 +1132,7 @@ inline ReturnType readTimeTextImpl(time_t & time, ReadBuffer & buf, const DateLU
11071132
}
11081133

11091134
template <typename ReturnType>
1110-
inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr)
1135+
inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr, bool saturate_on_overflow = true)
11111136
{
11121137
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
11131138

@@ -1121,7 +1146,7 @@ inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, Re
11211146
{
11221147
try
11231148
{
1124-
readDateTimeTextImpl<ReturnType, true>(whole, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters);
1149+
readDateTimeTextImpl<ReturnType, true>(whole, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters, saturate_on_overflow);
11251150
}
11261151
catch (const DB::Exception &)
11271152
{
@@ -1131,8 +1156,7 @@ inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, Re
11311156
}
11321157
else
11331158
{
1134-
auto ok = readDateTimeTextImpl<ReturnType, true>(whole, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters);
1135-
if (!ok && (buf.eof() || *buf.position() != '.'))
1159+
if (!readDateTimeTextImpl<ReturnType, true>(whole, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters, saturate_on_overflow))
11361160
return ReturnType(false);
11371161
}
11381162
}
@@ -1367,14 +1391,14 @@ inline bool tryReadTimeText(time_t & time, ReadBuffer & buf, const DateLUTImpl &
13671391
return readTimeTextImpl<bool>(time, buf, time_zone, allowed_date_delimiters, allowed_time_delimiters);
13681392
}
13691393

1370-
inline bool tryReadDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr)
1394+
inline bool tryReadDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr, bool saturate_on_overflow = true)
13711395
{
1372-
return readDateTimeTextImpl<bool>(datetime, buf, time_zone, allowed_date_delimiters, allowed_time_delimiters);
1396+
return readDateTimeTextImpl<bool>(datetime, buf, time_zone, allowed_date_delimiters, allowed_time_delimiters, saturate_on_overflow);
13731397
}
13741398

1375-
inline bool tryReadDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance(), const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr)
1399+
inline bool tryReadDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance(), const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr, bool saturate_on_overflow = true)
13761400
{
1377-
return readDateTimeTextImpl<bool>(datetime64, scale, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters);
1401+
return readDateTimeTextImpl<bool>(datetime64, scale, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters, saturate_on_overflow);
13781402
}
13791403

13801404
inline bool tryReadTime64Text(Time64 & time64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance(), const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr)

‎src/IO/parseDateTimeBestEffort.cpp‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ namespace DB
1313

1414
namespace ErrorCodes
1515
{
16-
extern const int LOGICAL_ERROR;
17-
extern const int CANNOT_PARSE_DATETIME;
16+
extern const int LOGICAL_ERROR;
17+
extern const int CANNOT_PARSE_DATETIME;
1818
}
1919

2020

@@ -753,7 +753,7 @@ ReturnType parseDateTimeBestEffortImpl(
753753
}
754754
};
755755

756-
if constexpr (std::is_same_v<ReturnType, void>)
756+
if constexpr (!strict || std::is_same_v<ReturnType, void>)
757757
{
758758
if (has_time_zone_offset)
759759
{
@@ -764,20 +764,24 @@ ReturnType parseDateTimeBestEffortImpl(
764764
{
765765
res = local_time_zone.makeDateTime(year, month, day_of_month, hour, minute, second);
766766
}
767+
768+
if constexpr (std::is_same_v<ReturnType, bool>)
769+
return true;
767770
}
768771
else
769772
{
770-
771773
if (has_time_zone_offset)
772774
{
773775
auto res_maybe = utc_time_zone.tryToMakeDateTime(year, month, day_of_month, hour, minute, second);
774776
if (!res_maybe)
775777
return false;
776778

777779
/// For usual DateTime check if value is within supported range
778-
if (!is_64 && (*res_maybe < 0 || *res_maybe > UINT32_MAX))
779-
return false;
780-
780+
if constexpr (!is_64)
781+
{
782+
if (*res_maybe < 0 || *res_maybe > UINT32_MAX)
783+
return false;
784+
}
781785
res = *res_maybe;
782786
adjust_time_zone();
783787
}
@@ -788,9 +792,11 @@ ReturnType parseDateTimeBestEffortImpl(
788792
return false;
789793

790794
/// For usual DateTime check if value is within supported range
791-
if (!is_64 && (*res_maybe < 0 || *res_maybe > UINT32_MAX))
792-
return false;
793-
795+
if constexpr (!is_64)
796+
{
797+
if (*res_maybe < 0 || *res_maybe > UINT32_MAX)
798+
return false;
799+
}
794800
res = *res_maybe;
795801
}
796802

‎tests/queries/0_stateless/01186_conversion_to_nullable.reference‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@
1010
256
1111
2020-12-24
1212
\N
13+
1970-01-01
1314
\N
14-
\N
15-
\N
15+
2149-06-06
1616
2020-12-24 01:02:03
1717
\N
18-
\N
18+
1970-01-01 02:00:00
1919
\N
2020
2020-12-24 01:02:03.00
2121
\N

‎tests/queries/0_stateless/01556_accurate_cast_or_null.reference‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,13 @@
3636
2023-05-30 14:38:20
3737
1970-01-01 00:00:19
3838
1970-01-01 19:26:40
39-
\N
40-
\N
39+
1970-01-01 00:00:00
40+
2106-02-07 06:28:15
4141
\N
4242
\N
4343
\N
4444
2023-05-30
45-
\N
45+
2149-06-06
4646
1970-01-20
4747
\N
4848
\N

‎tests/queries/0_stateless/03149_asof_join_ddb_timestamps.reference‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
2023-03-21 19:00:00 3
88
2023-03-21 20:00:00 3
99
2023-03-21 21:00:00 3
10+
2106-02-07 06:28:15 9
1011
2023-03-21 13:00:00 0
1112
2023-03-21 14:00:00 1
1213
2023-03-21 15:00:00 2
@@ -16,6 +17,7 @@
1617
2023-03-21 19:00:00 3
1718
2023-03-21 20:00:00 3
1819
2023-03-21 21:00:00 3
20+
2106-02-07 06:28:15 9
1921
2023-03-21 12:00:00 \N
2022
2023-03-21 13:00:00 0
2123
2023-03-21 14:00:00 1
@@ -26,7 +28,7 @@
2628
2023-03-21 19:00:00 3
2729
2023-03-21 20:00:00 3
2830
2023-03-21 21:00:00 3
29-
\N \N
31+
2106-02-07 06:28:15 9
3032
\N \N
3133
2023-03-21 12:00:00 0
3234
2023-03-21 13:00:00 0
@@ -38,7 +40,7 @@
3840
2023-03-21 19:00:00 3
3941
2023-03-21 20:00:00 3
4042
2023-03-21 21:00:00 3
41-
\N 0
43+
2106-02-07 06:28:15 9
4244
\N 0
4345
2023-03-21 12:00:00 \N
4446
2023-03-21 13:00:00 \N
@@ -50,5 +52,5 @@
5052
2023-03-21 19:00:00 \N
5153
2023-03-21 20:00:00 \N
5254
2023-03-21 21:00:00 \N
53-
\N \N
55+
2106-02-07 06:28:15 \N
5456
\N \N

0 commit comments

Comments
 (0)