Skip to content

Commit 1b81208

Browse files
zclllyybbzclllyybb
authored
[Fix](function) fix skip null value for some date functions (#59616)
fix skip null value for some date functions
1 parent e3fc71b commit 1b81208

File tree

9 files changed

+159
-119
lines changed

9 files changed

+159
-119
lines changed

‎be/src/vec/functions/function_date_or_datetime_computation.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,10 +119,10 @@ using FunctionDatetimeSubQuarters =
119119
using FunctionDatetimeSubYears =
120120
FunctionDateOrDateTimeComputation<SubtractYearsImpl<TYPE_DATETIMEV2>>;
121121

122-
using FunctionAddTimeDatetime = FunctionAddTime<TYPE_DATETIMEV2, AddTimeImpl>;
123-
using FunctionAddTimeTime = FunctionAddTime<TYPE_TIMEV2, AddTimeImpl>;
124-
using FunctionSubTimeDatetime = FunctionAddTime<TYPE_DATETIMEV2, SubTimeImpl>;
125-
using FunctionSubTimeTime = FunctionAddTime<TYPE_TIMEV2, SubTimeImpl>;
122+
using FunctionAddTimeDatetime = FunctionNeedsToHandleNull<AddTimeDatetimeImpl, TYPE_DATETIMEV2>;
123+
using FunctionAddTimeTime = FunctionNeedsToHandleNull<AddTimeTimeImpl, TYPE_TIMEV2>;
124+
using FunctionSubTimeDatetime = FunctionNeedsToHandleNull<SubTimeDatetimeImpl, TYPE_DATETIMEV2>;
125+
using FunctionSubTimeTime = FunctionNeedsToHandleNull<SubTimeTimeImpl, TYPE_TIMEV2>;
126126

127127
#define FUNCTION_TIME_DIFF(NAME, IMPL, TYPE) using NAME##_##TYPE = FunctionTimeDiff<IMPL<TYPE>>;
128128

‎be/src/vec/functions/function_date_or_datetime_computation.h‎

Lines changed: 51 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include <cstddef>
2424
#include <cstdint>
2525
#include <memory>
26+
#include <string>
2627
#include <string_view>
2728
#include <type_traits>
2829
#include <utility>
@@ -1819,118 +1820,71 @@ class PeriodDiffImpl {
18191820
}
18201821
};
18211822

1822-
struct AddTimeImpl {
1823-
static constexpr auto name = "add_time";
1824-
static bool is_negative() { return false; }
1825-
};
1826-
1827-
struct SubTimeImpl {
1828-
static constexpr auto name = "sub_time";
1829-
static bool is_negative() { return true; }
1830-
};
1831-
1832-
template <PrimitiveType PType, typename Impl>
1833-
class FunctionAddTime : public IFunction {
1823+
template <PrimitiveType PType, bool IsNegative>
1824+
class AddTimeImplBase {
18341825
public:
1835-
static constexpr auto name = Impl::name;
1836-
static constexpr PrimitiveType ReturnType = PType;
1837-
static constexpr PrimitiveType ArgType1 = PType;
1838-
static constexpr PrimitiveType ArgType2 = TYPE_TIMEV2;
1839-
using ColumnType1 = typename PrimitiveTypeTraits<PType>::ColumnType;
1840-
using ColumnType2 = typename PrimitiveTypeTraits<TYPE_TIMEV2>::ColumnType;
1826+
static constexpr auto name = IsNegative ? "sub_time" : "add_time";
18411827
using InputType1 = typename PrimitiveTypeTraits<PType>::DataType::FieldType;
18421828
using InputType2 = typename PrimitiveTypeTraits<TYPE_TIMEV2>::DataType::FieldType;
1843-
using ReturnNativeType = InputType1;
1844-
using ReturnDataType = typename PrimitiveTypeTraits<PType>::DataType;
1829+
using ResultColumnType = typename PrimitiveTypeTraits<PType>::ColumnType;
18451830

1846-
String get_name() const override { return name; }
1847-
size_t get_number_of_arguments() const override { return 2; }
1848-
DataTypes get_variadic_argument_types_impl() const override {
1831+
static size_t get_number_of_arguments() { return 2; }
1832+
static bool is_variadic() { return true; }
1833+
static DataTypes get_variadic_argument_types_impl() {
18491834
return {std::make_shared<typename PrimitiveTypeTraits<PType>::DataType>(),
18501835
std::make_shared<typename PrimitiveTypeTraits<TYPE_TIMEV2>::DataType>()};
18511836
}
1852-
DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override {
1853-
return std::make_shared<ReturnDataType>();
1854-
}
1855-
1856-
ReturnNativeType compute(const InputType1& arg1, const InputType2& arg2) const {
1857-
if constexpr (PType == TYPE_DATETIMEV2) {
1858-
DateV2Value<DateTimeV2ValueType> dtv1 =
1859-
binary_cast<InputType1, DateV2Value<DateTimeV2ValueType>>(arg1);
1860-
auto tv2 = static_cast<TimeValue::TimeType>(arg2);
1861-
TimeInterval interval(TimeUnit::MICROSECOND, tv2, Impl::is_negative());
1862-
bool out_range = dtv1.template date_add_interval<TimeUnit::MICROSECOND>(interval);
1863-
if (UNLIKELY(!out_range)) {
1864-
throw Exception(ErrorCode::INVALID_ARGUMENT,
1865-
"datetime value is out of range in function {}", name);
1866-
}
1867-
return binary_cast<DateV2Value<DateTimeV2ValueType>, ReturnNativeType>(dtv1);
1868-
} else if constexpr (PType == TYPE_TIMEV2) {
1869-
auto tv1 = static_cast<TimeValue::TimeType>(arg1);
1870-
auto tv2 = static_cast<TimeValue::TimeType>(arg2);
1871-
double res = TimeValue::limit_with_bound(Impl::is_negative() ? tv1 - tv2 : tv1 + tv2);
1872-
return res;
1873-
} else {
1874-
throw Exception(ErrorCode::FATAL_ERROR, "not support type for function {}", name);
1837+
static DataTypePtr get_return_type_impl(const DataTypes& arguments) {
1838+
if (arguments[0]->is_nullable() || arguments[1]->is_nullable()) {
1839+
return make_nullable(std::make_shared<typename PrimitiveTypeTraits<PType>::DataType>());
18751840
}
1841+
return std::make_shared<typename PrimitiveTypeTraits<PType>::DataType>();
18761842
}
18771843

1878-
static FunctionPtr create() { return std::make_shared<FunctionAddTime>(); }
1879-
1880-
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
1881-
uint32_t result, size_t input_rows_count) const override {
1882-
DCHECK_EQ(arguments.size(), 2);
1883-
const auto& [left_col, left_const] =
1884-
unpack_if_const(block.get_by_position(arguments[0]).column);
1885-
const auto& [right_col, right_const] =
1886-
unpack_if_const(block.get_by_position(arguments[1]).column);
1887-
ColumnPtr nest_col1 = remove_nullable(left_col);
1888-
ColumnPtr nest_col2 = remove_nullable(right_col);
1889-
auto res = ColumnVector<ReturnType>::create(input_rows_count, 0);
1890-
1891-
if (left_const) {
1892-
execute_constant_vector(assert_cast<const ColumnType1&>(*nest_col1).get_element(0),
1893-
assert_cast<const ColumnType2&>(*nest_col2).get_data(),
1894-
res->get_data(), input_rows_count);
1895-
} else if (right_const) {
1896-
execute_vector_constant(assert_cast<const ColumnType1&>(*nest_col1).get_data(),
1897-
assert_cast<const ColumnType2&>(*nest_col2).get_element(0),
1898-
res->get_data(), input_rows_count);
1899-
} else {
1900-
execute_vector_vector(assert_cast<const ColumnType1&>(*nest_col1).get_data(),
1901-
assert_cast<const ColumnType2&>(*nest_col2).get_data(),
1902-
res->get_data(), input_rows_count);
1903-
}
1904-
1905-
block.replace_by_position(result, std::move(res));
1906-
return Status::OK();
1907-
}
1908-
void execute_vector_vector(const PaddedPODArray<InputType1>& left_col,
1909-
const PaddedPODArray<InputType2>& right_col,
1910-
PaddedPODArray<ReturnNativeType>& res_data,
1911-
size_t input_rows_count) const {
1912-
for (size_t i = 0; i < input_rows_count; ++i) {
1913-
res_data[i] = compute(left_col[i], right_col[i]);
1914-
}
1915-
}
1844+
static void execute(const std::vector<ColumnWithConstAndNullMap>& cols_info,
1845+
typename ResultColumnType::MutablePtr& res_col,
1846+
PaddedPODArray<UInt8>& res_null_map_data, size_t input_rows_count) {
1847+
const auto& left_data =
1848+
assert_cast<const ResultColumnType*>(cols_info[0].nested_col)->get_data();
1849+
const auto& right_data =
1850+
assert_cast<const ColumnVector<TYPE_TIMEV2>*>(cols_info[1].nested_col)->get_data();
19161851

1917-
void execute_vector_constant(const PaddedPODArray<InputType1>& left_col,
1918-
const InputType2 right_value,
1919-
PaddedPODArray<ReturnNativeType>& res_data,
1920-
size_t input_rows_count) const {
19211852
for (size_t i = 0; i < input_rows_count; ++i) {
1922-
res_data[i] = compute(left_col[i], right_value);
1923-
}
1924-
}
1853+
if (cols_info[0].is_null_at(i) || cols_info[1].is_null_at(i)) {
1854+
res_col->insert_default();
1855+
res_null_map_data[i] = 1;
1856+
continue;
1857+
}
19251858

1926-
void execute_constant_vector(const InputType1 left_value,
1927-
const PaddedPODArray<InputType2>& right_col,
1928-
PaddedPODArray<ReturnNativeType>& res_data,
1929-
size_t input_rows_count) const {
1930-
for (size_t i = 0; i < input_rows_count; ++i) {
1931-
res_data[i] = compute(left_value, right_col[i]);
1859+
const auto& arg1 = left_data[index_check_const(i, cols_info[0].is_const)];
1860+
const auto& arg2 = right_data[index_check_const(i, cols_info[1].is_const)];
1861+
1862+
if constexpr (PType == TYPE_DATETIMEV2) {
1863+
DateV2Value<DateTimeV2ValueType> dtv1 =
1864+
binary_cast<InputType1, DateV2Value<DateTimeV2ValueType>>(arg1);
1865+
auto tv2 = static_cast<TimeValue::TimeType>(arg2);
1866+
TimeInterval interval(TimeUnit::MICROSECOND, tv2, IsNegative);
1867+
bool out_range = dtv1.template date_add_interval<TimeUnit::MICROSECOND>(interval);
1868+
if (!out_range) [[unlikely]] {
1869+
throw_invalid_strings(name, dtv1.to_string(), std::to_string(arg2));
1870+
}
1871+
res_col->insert_value(
1872+
binary_cast<DateV2Value<DateTimeV2ValueType>, InputType1>(dtv1));
1873+
} else if constexpr (PType == TYPE_TIMEV2) {
1874+
auto tv1 = static_cast<TimeValue::TimeType>(arg1);
1875+
auto tv2 = static_cast<TimeValue::TimeType>(arg2);
1876+
double res = TimeValue::limit_with_bound(IsNegative ? tv1 - tv2 : tv1 + tv2);
1877+
res_col->insert_value(res);
1878+
} else {
1879+
throw Exception(ErrorCode::FATAL_ERROR, "not support type for function {}", name);
1880+
}
19321881
}
19331882
}
19341883
};
1884+
1885+
using AddTimeDatetimeImpl = AddTimeImplBase<TYPE_DATETIMEV2, false>;
1886+
using AddTimeTimeImpl = AddTimeImplBase<TYPE_TIMEV2, false>;
1887+
using SubTimeDatetimeImpl = AddTimeImplBase<TYPE_DATETIMEV2, true>;
1888+
using SubTimeTimeImpl = AddTimeImplBase<TYPE_TIMEV2, true>;
19351889
#include "common/compile_check_avoid_end.h"
19361890
} // namespace doris::vectorized

‎be/src/vec/functions/function_date_or_datetime_to_string.cpp‎

Lines changed: 36 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,6 @@ class FunctionDateOrDateTimeToString : public IFunction {
7777
return {};
7878
}
7979

80-
ColumnNumbers get_arguments_that_are_always_constant() const override { return {1}; }
81-
8280
// In ICU, Week_array: {"", "Sunday", "Monday", ..., "Saturday"}, size = 8
8381
// Month_array: {"January", "February", ..., "December"}, size = 12
8482
static constexpr size_t DAY_NUM_IN_ICU = 8;
@@ -140,28 +138,48 @@ class FunctionDateOrDateTimeToString : public IFunction {
140138
return IFunction::open(context, scope);
141139
}
142140

141+
bool use_default_implementation_for_nulls() const override { return false; }
142+
143143
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
144144
uint32_t result, size_t input_rows_count) const override {
145145
const ColumnPtr source_col = block.get_by_position(arguments[0]).column;
146+
const NullMap* null_map = nullptr;
147+
ColumnPtr actual_col = source_col;
148+
149+
if (is_column_nullable(*source_col)) {
150+
const auto* nullable_col = check_and_get_column<ColumnNullable>(source_col.get());
151+
actual_col = nullable_col->get_nested_column_ptr();
152+
null_map = &nullable_col->get_null_map_data();
153+
}
154+
146155
const auto* sources =
147-
check_and_get_column<ColumnVector<Transform::OpArgType>>(source_col.get());
148-
auto col_res = ColumnString::create();
156+
check_and_get_column<ColumnVector<Transform::OpArgType>>(actual_col.get());
157+
if (!sources) [[unlikely]] {
158+
return Status::FatalError("Illegal column {} of first argument of function {}",
159+
block.get_by_position(arguments[0]).column->get_name(), name);
160+
}
149161

150-
// Support all input of datetime is valind to make sure not null return
151-
if (sources) {
152-
vector(context, sources->get_data(), col_res->get_chars(), col_res->get_offsets());
153-
block.replace_by_position(result, std::move(col_res));
162+
auto col_res = ColumnString::create();
163+
vector(context, sources->get_data(), col_res->get_chars(), col_res->get_offsets(),
164+
null_map);
165+
166+
if (null_map) {
167+
const auto* nullable_col = check_and_get_column<ColumnNullable>(source_col.get());
168+
block.replace_by_position(
169+
result,
170+
ColumnNullable::create(std::move(col_res),
171+
nullable_col->get_null_map_column_ptr()->clone_resized(
172+
input_rows_count)));
154173
} else {
155-
return Status::InternalError("Illegal column {} of first argument of function {}",
156-
block.get_by_position(arguments[0]).column->get_name(),
157-
name);
174+
block.replace_by_position(result, std::move(col_res));
158175
}
159176
return Status::OK();
160177
}
161178

162179
private:
163180
static void vector(FunctionContext* context, const PaddedPODArray<NativeType>& ts,
164-
ColumnString::Chars& res_data, ColumnString::Offsets& res_offsets) {
181+
ColumnString::Chars& res_data, ColumnString::Offsets& res_offsets,
182+
const NullMap* null_map = nullptr) {
165183
const auto len = ts.size();
166184
res_data.resize(len * Transform::max_size);
167185
res_offsets.resize(len);
@@ -176,7 +194,12 @@ class FunctionDateOrDateTimeToString : public IFunction {
176194
names_ptr = state->month_names;
177195
}
178196

179-
for (int i = 0; i < len; ++i) {
197+
for (size_t i = 0; i < len; ++i) {
198+
if (null_map && (*null_map)[i]) {
199+
res_offsets[i] = cast_set<UInt32>(offset);
200+
continue;
201+
}
202+
180203
const auto& t = ts[i];
181204
const auto date_time_value = binary_cast<NativeType, DateType>(t);
182205
res_offsets[i] = cast_set<UInt32>(

‎be/src/vec/functions/function_needs_to_handle_null.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@ class FunctionNeedsToHandleNull : public IFunction {
5353
}
5454
return false;
5555
}
56+
DataTypes get_variadic_argument_types_impl() const override {
57+
if constexpr (requires { Impl::get_variadic_argument_types_impl(); }) {
58+
return Impl::get_variadic_argument_types_impl();
59+
}
60+
return {};
61+
}
5662

5763
bool use_default_implementation_for_nulls() const override { return false; }
5864

regression-test/data/nereids_function_p0/scalar_function/Map.out renamed to regression-test/data/nereids_function_p0/scalar_function/nereids_scalar_fn_map.out

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3722,3 +3722,11 @@ false
37223722
-- !sql --
37233723
{"{"zip":10001}":"{"city":"NY"}", "{"code":10002}":"{"state":"NY"}"}
37243724

3725+
-- !element_at_tint_date_dayname --
3726+
\N
3727+
Sunday
3728+
3729+
-- !element_at_tint_date_dayname_notnull --
3730+
\N
3731+
Saturday
3732+

‎regression-test/data/nereids_p0/join/test_outer_join.out‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,8 @@
1010

1111
-- !join --
1212

13+
-- !join --
14+
\N \N
15+
1 2023-12-18T23:30:01
16+
2 2023-12-15T23:30:01
17+

regression-test/suites/nereids_function_p0/scalar_function/Map.groovy renamed to regression-test/suites/nereids_function_p0/scalar_function/nereids_scalar_fn_map.groovy

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ suite("nereids_scalar_fn_map") {
3535
order_qt_element_at_str_tint """ select km_str_tint[kstr] from fn_test """
3636
order_qt_element_at_date_tint """ select km_date_tint[kdt] from fn_test """
3737
order_qt_element_at_dtm_tint """ select km_dtm_tint[kdtm] from fn_test """
38+
3839
order_qt_element_at_bool_tint_notnull """ select km_bool_tint[kbool] from fn_test_not_nullable """
3940
order_qt_element_at_tint_tint_notnull """ select km_tint_tint[ktint] from fn_test_not_nullable """
4041
order_qt_element_at_sint_tint_notnull """ select km_sint_tint[ksint] from fn_test_not_nullable """
@@ -314,4 +315,24 @@ suite("nereids_scalar_fn_map") {
314315
qt_sql "select map('postal_code', 10001, 'area_code', 10002, 'zip_plus_4', 10003)"
315316
qt_sql "select map('{\"zip\":10001}', '{\"city\":\"NY\"}', '{\"code\":10002}', '{\"state\":\"NY\"}')"
316317

318+
sql """ drop table if exists fn_test_element_at_map_date """
319+
sql """
320+
create table fn_test_element_at_map_date (
321+
id int null,
322+
m map<tinyint, date> null
323+
) engine=olap
324+
distributed by hash(id) buckets 1
325+
properties('replication_num' = '1')
326+
"""
327+
sql """
328+
insert into fn_test_element_at_map_date values
329+
(1, map(1,null, 2,'2023-12-16')),
330+
(2, map(1,'2023-01-01', 2,null));
331+
"""
332+
order_qt_element_at_tint_date_dayname """
333+
select dayname(element_at(m, 1)) from fn_test_element_at_map_date
334+
"""
335+
order_qt_element_at_tint_date_dayname_notnull """
336+
select dayname(element_at(m, 2)) from fn_test_element_at_map_date
337+
"""
317338
}

‎regression-test/suites/nereids_p0/join/test_outer_join.groovy‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,30 @@ suite("test_outer_join", "nereids_p0") {
8484
contains "FULL OUTER JOIN(PARTITIONED)"
8585
}
8686

87-
sql "DROP TABLE IF EXISTS ${tbl1}"
88-
sql "DROP TABLE IF EXISTS ${tbl2}"
89-
sql "DROP TABLE IF EXISTS ${tbl3}"
87+
def tbl4 = "test_outer_join4"
88+
def tbl5 = "test_outer_join5"
89+
90+
sql "DROP TABLE IF EXISTS test_outer_join4"
91+
sql """
92+
CREATE TABLE test_outer_join4 (
93+
k INT,
94+
d DATEV2
95+
)
96+
DUPLICATE KEY(k)
97+
DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES ("replication_num" = "1");
98+
"""
99+
sql "DROP TABLE IF EXISTS test_outer_join5"
100+
sql """
101+
CREATE TABLE test_outer_join5 (
102+
k INT
103+
)
104+
DUPLICATE KEY(k)
105+
DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES ("replication_num" = "1");
106+
"""
107+
sql """INSERT INTO test_outer_join4 VALUES (1, '2023-12-18'), (2, '2023-12-15');"""
108+
sql """INSERT INTO test_outer_join5 VALUES (1), (2), (3);"""
109+
110+
order_qt_join """
111+
SELECT k, add_time(d, '23:30:01') FROM test_outer_join4 RIGHT JOIN test_outer_join5 USING (k) ORDER BY k;
112+
"""
90113
}

‎regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ suite("test_date_function_v2") {
175175

176176
test{
177177
sql("select add_time('9999-12-29 00:00:00', '122:35:22.123456');")
178-
exception "datetime value is out of range in function add_time"
178+
exception "Operation add_time of"
179179
}
180180

181181
qt_sql_subtime1("select sub_time('2023-10-14 00:00:00', '22:35:22');")
@@ -193,7 +193,7 @@ suite("test_date_function_v2") {
193193

194194
test{
195195
sql("select sub_time('0000-01-01 00:00:00', '122:35:22.123456');")
196-
exception "datetime value is out of range in function sub_time"
196+
exception "Operation sub_time of"
197197
}
198198

199199
//test computetimearithmetic regular

0 commit comments

Comments
 (0)