Skip to content

Commit ca5927a

Browse files
ti-chi-botti-chi-bot
authored
Support other integer types for SubstringUTF8 & RightUTF8 functions (#9507) (#9512)
close #9473 Support other integer types for SubstringUTF8 & RightUTF8 functions Signed-off-by: gengliqi <[email protected]>
1 parent bbc6bf0 commit ca5927a

File tree

6 files changed

+565
-292
lines changed

6 files changed

+565
-292
lines changed

‎dbms/src/Functions/FunctionsString.cpp‎

Lines changed: 282 additions & 203 deletions
Large diffs are not rendered by default.

‎dbms/src/Functions/tests/gtest_string_left.cpp‎

Lines changed: 21 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -67,61 +67,55 @@ class StringLeftTest : public DB::tests::FunctionTest
6767
for (bool is_length_const : is_consts)
6868
inner_test(is_str_const, is_length_const);
6969
}
70-
71-
template <typename Integer>
72-
void testInvalidLengthType()
73-
{
74-
static_assert(!std::is_same_v<Integer, Int64> && !std::is_same_v<Integer, UInt64>);
75-
auto inner_test = [&](bool is_str_const, bool is_length_const) {
76-
ASSERT_THROW(
77-
executeFunction(
78-
func_name,
79-
is_str_const ? createConstColumn<Nullable<String>>(1, "") : createColumn<Nullable<String>>({""}),
80-
is_length_const ? createConstColumn<Nullable<Integer>>(1, 0) : createColumn<Nullable<Integer>>({0})),
81-
Exception);
82-
};
83-
std::vector<bool> is_consts = {true, false};
84-
for (bool is_str_const : is_consts)
85-
for (bool is_length_const : is_consts)
86-
inner_test(is_str_const, is_length_const);
87-
}
8870
};
8971

9072
TEST_F(StringLeftTest, testBoundary)
9173
try
9274
{
75+
testBoundary<Int8>();
76+
testBoundary<Int16>();
77+
testBoundary<Int32>();
9378
testBoundary<Int64>();
79+
testBoundary<UInt8>();
80+
testBoundary<UInt16>();
81+
testBoundary<UInt32>();
9482
testBoundary<UInt64>();
9583
}
9684
CATCH
9785

9886
TEST_F(StringLeftTest, testMoreCases)
9987
try
10088
{
89+
#define CALL(A, B, C) \
90+
test<Int8>(A, B, C); \
91+
test<Int16>(A, B, C); \
92+
test<Int32>(A, B, C); \
93+
test<Int64>(A, B, C); \
94+
test<UInt8>(A, B, C); \
95+
test<UInt16>(A, B, C); \
96+
test<UInt32>(A, B, C); \
97+
test<UInt64>(A, B, C);
98+
10199
// test big string
102100
// big_string.size() > length
103101
String big_string;
104102
// unit_string length = 22
105103
String unit_string = "big string is 我!!!!!!!";
106104
for (size_t i = 0; i < 1000; ++i)
107105
big_string += unit_string;
108-
test<Int64>(big_string, 22, unit_string);
109-
test<UInt64>(big_string, 22, unit_string);
106+
CALL(big_string, 22, unit_string);
110107

111108
// test origin_str.size() == length
112109
String origin_str = "我的 size = 12";
113-
test<Int64>(origin_str, 12, origin_str);
114-
test<UInt64>(origin_str, 12, origin_str);
110+
CALL(origin_str, 12, origin_str);
115111

116112
// test origin_str.size() < length
117-
test<Int64>(origin_str, 22, origin_str);
118-
test<UInt64>(origin_str, 22, origin_str);
113+
CALL(origin_str, 22, origin_str);
119114

120115
// Mixed language
121116
String english_str = "This is English";
122117
String mixed_language_str = english_str + ",这是中文,C'est français,これが日本の";
123-
test<Int64>(mixed_language_str, english_str.size(), english_str);
124-
test<UInt64>(mixed_language_str, english_str.size(), english_str);
118+
CALL(mixed_language_str, english_str.size(), english_str);
125119

126120
// column size != 1
127121
// case 1
@@ -145,18 +139,8 @@ try
145139
func_name,
146140
createConstColumn<Nullable<String>>(8, second_case_string),
147141
createColumn<Nullable<Int64>>({0, 1, 0, 1, 0, 0, 1, 1})));
148-
}
149-
CATCH
150142

151-
TEST_F(StringLeftTest, testInvalidLengthType)
152-
try
153-
{
154-
testInvalidLengthType<Int8>();
155-
testInvalidLengthType<Int16>();
156-
testInvalidLengthType<Int32>();
157-
testInvalidLengthType<UInt8>();
158-
testInvalidLengthType<UInt16>();
159-
testInvalidLengthType<UInt32>();
143+
#undef CALL
160144
}
161145
CATCH
162146

‎dbms/src/Functions/tests/gtest_strings_right.cpp‎

Lines changed: 21 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -66,61 +66,55 @@ class StringRightTest : public DB::tests::FunctionTest
6666
for (bool is_length_const : is_consts)
6767
inner_test(is_str_const, is_length_const);
6868
}
69-
70-
template <typename Integer>
71-
void testInvalidLengthType()
72-
{
73-
static_assert(!std::is_same_v<Integer, Int64> && !std::is_same_v<Integer, UInt64>);
74-
auto inner_test = [&](bool is_str_const, bool is_length_const) {
75-
ASSERT_THROW(
76-
executeFunction(
77-
func_name,
78-
is_str_const ? createConstColumn<Nullable<String>>(1, "") : createColumn<Nullable<String>>({""}),
79-
is_length_const ? createConstColumn<Nullable<Integer>>(1, 0) : createColumn<Nullable<Integer>>({0})),
80-
Exception);
81-
};
82-
std::vector<bool> is_consts = {true, false};
83-
for (bool is_str_const : is_consts)
84-
for (bool is_length_const : is_consts)
85-
inner_test(is_str_const, is_length_const);
86-
}
8769
};
8870

8971
TEST_F(StringRightTest, testBoundary)
9072
try
9173
{
74+
testBoundary<Int8>();
75+
testBoundary<Int16>();
76+
testBoundary<Int32>();
9277
testBoundary<Int64>();
78+
testBoundary<UInt8>();
79+
testBoundary<UInt16>();
80+
testBoundary<UInt32>();
9381
testBoundary<UInt64>();
9482
}
9583
CATCH
9684

9785
TEST_F(StringRightTest, testMoreCases)
9886
try
9987
{
88+
#define CALL(A, B, C) \
89+
test<Int8>(A, B, C); \
90+
test<Int16>(A, B, C); \
91+
test<Int32>(A, B, C); \
92+
test<Int64>(A, B, C); \
93+
test<UInt8>(A, B, C); \
94+
test<UInt16>(A, B, C); \
95+
test<UInt32>(A, B, C); \
96+
test<UInt64>(A, B, C);
97+
10098
// test big string
10199
// big_string.size() > length
102100
String big_string;
103101
// unit_string length = 22
104102
String unit_string = "big string is 我!!!!!!!";
105103
for (size_t i = 0; i < 1000; ++i)
106104
big_string += unit_string;
107-
test<Int64>(big_string, 22, unit_string);
108-
test<UInt64>(big_string, 22, unit_string);
105+
CALL(big_string, 22, unit_string);
109106

110107
// test origin_str.size() == length
111108
String origin_str = "我的 size = 12";
112-
test<Int64>(origin_str, 12, origin_str);
113-
test<UInt64>(origin_str, 12, origin_str);
109+
CALL(origin_str, 12, origin_str);
114110

115111
// test origin_str.size() < length
116-
test<Int64>(origin_str, 22, origin_str);
117-
test<UInt64>(origin_str, 22, origin_str);
112+
CALL(origin_str, 22, origin_str);
118113

119114
// Mixed language
120115
String english_str = "This is English";
121116
String mixed_language_str = "这是中文,C'est français,これが日本の," + english_str;
122-
test<Int64>(mixed_language_str, english_str.size(), english_str);
123-
test<UInt64>(mixed_language_str, english_str.size(), english_str);
117+
CALL(mixed_language_str, english_str.size(), english_str);
124118

125119
// column size != 1
126120
// case 1
@@ -144,18 +138,8 @@ try
144138
func_name,
145139
createConstColumn<Nullable<String>>(8, second_case_string),
146140
createColumn<Nullable<Int64>>({0, 1, 0, 1, 0, 0, 1, 1})));
147-
}
148-
CATCH
149141

150-
TEST_F(StringRightTest, testInvalidLengthType)
151-
try
152-
{
153-
testInvalidLengthType<Int8>();
154-
testInvalidLengthType<Int16>();
155-
testInvalidLengthType<Int32>();
156-
testInvalidLengthType<UInt8>();
157-
testInvalidLengthType<UInt16>();
158-
testInvalidLengthType<UInt32>();
142+
#undef CALL
159143
}
160144
CATCH
161145

‎dbms/src/Functions/tests/gtest_substring.cpp‎

Lines changed: 153 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,160 @@ class SubString : public DB::tests::FunctionTest
2828
{
2929
};
3030

31+
template <typename T1, typename T2>
32+
class TestNullableSigned
33+
{
34+
public:
35+
static void run(SubString & sub_string)
36+
{
37+
ASSERT_COLUMN_EQ(
38+
createColumn<Nullable<String>>({"p.co", "ww.p", "pingcap", "com", ".com", "", "", "", {}, {}, {}}),
39+
sub_string.executeFunction(
40+
"substringUTF8",
41+
createColumn<Nullable<String>>(
42+
{"www.pingcap.com",
43+
"ww.pingcap.com",
44+
"w.pingcap.com",
45+
".pingcap.com",
46+
"pingcap.com",
47+
"pingcap.com",
48+
"pingcap.com",
49+
"pingcap.com",
50+
{},
51+
"pingcap",
52+
"pingcap"}),
53+
createColumn<T1>({-5, 1, 3, -3, 8, 2, -100, 0, 2, {}, -3}),
54+
createColumn<T2>({4, 4, 7, 4, 5, -5, 2, 3, 6, 4, {}})));
55+
}
56+
};
57+
58+
template <typename T1, typename T2>
59+
class TestSigned
60+
{
61+
public:
62+
static void run(SubString & sub_string)
63+
{
64+
ASSERT_COLUMN_EQ(
65+
createColumn<Nullable<String>>({"p.co", "ww.p", "pingcap", "com", ".com", "", "", "", {}}),
66+
sub_string.executeFunction(
67+
"substringUTF8",
68+
createColumn<Nullable<String>>(
69+
{"www.pingcap.com",
70+
"ww.pingcap.com",
71+
"w.pingcap.com",
72+
".pingcap.com",
73+
"pingcap.com",
74+
"pingcap.com",
75+
"pingcap.com",
76+
"pingcap.com",
77+
{}}),
78+
createColumn<T1>({-5, 1, 3, -3, 8, 2, -100, 0, 2}),
79+
createColumn<T2>({4, 4, 7, 4, 5, -5, 2, 3, 6})));
80+
}
81+
};
82+
83+
template <typename T1, typename T2>
84+
class TestNullableUnsigned
85+
{
86+
public:
87+
static void run(SubString & sub_string)
88+
{
89+
ASSERT_COLUMN_EQ(
90+
createColumn<Nullable<String>>({"p.co", "ww.p", "pingcap", "com", ".com", "", "", {}, {}, {}}),
91+
sub_string.executeFunction(
92+
"substringUTF8",
93+
createColumn<Nullable<String>>(
94+
{"www.pingcap.com",
95+
"ww.pingcap.com",
96+
"w.pingcap.com",
97+
".pingcap.com",
98+
"pingcap.com",
99+
"pingcap.com",
100+
"pingcap.com",
101+
{},
102+
"pingcap",
103+
"pingcap"}),
104+
createColumn<T1>({11, 1, 3, 10, 8, 2, 0, 9, {}, 7}),
105+
createColumn<T2>({4, 4, 7, 4, 5, 0, 3, 6, 1, {}})));
106+
}
107+
};
108+
109+
template <typename T1, typename T2>
110+
class TestUnsigned
111+
{
112+
public:
113+
static void run(SubString & sub_string)
114+
{
115+
ASSERT_COLUMN_EQ(
116+
createColumn<Nullable<String>>({"p.co", "ww.p", "pingcap", "com", ".com", "", "", {}}),
117+
sub_string.executeFunction(
118+
"substringUTF8",
119+
createColumn<Nullable<String>>(
120+
{"www.pingcap.com",
121+
"ww.pingcap.com",
122+
"w.pingcap.com",
123+
".pingcap.com",
124+
"pingcap.com",
125+
"pingcap.com",
126+
"pingcap.com",
127+
{}}),
128+
createColumn<T1>({11, 1, 3, 10, 8, 2, 0, 2}),
129+
createColumn<T2>({4, 4, 7, 4, 5, 0, 3, 1})));
130+
}
131+
};
132+
133+
template <typename T1, typename T2>
134+
class TestConstPos
135+
{
136+
public:
137+
static void run(SubString & sub_string)
138+
{
139+
ASSERT_COLUMN_EQ(
140+
createColumn<Nullable<String>>({"w", "ww", "w.p", ".pin"}),
141+
sub_string.executeFunction(
142+
"substringUTF8",
143+
createColumn<Nullable<String>>({"www.pingcap.com", "ww.pingcap.com", "w.pingcap.com", ".pingcap.com"}),
144+
createConstColumn<T1>(4, 1),
145+
createColumn<T2>({1, 2, 3, 4})));
146+
}
147+
};
148+
149+
template <typename T1, typename T2>
150+
class TestConstLength
151+
{
152+
public:
153+
static void run(SubString & sub_string)
154+
{
155+
ASSERT_COLUMN_EQ(
156+
createColumn<Nullable<String>>({"www.", "w.pi", "ping", "ngca"}),
157+
sub_string.executeFunction(
158+
"substringUTF8",
159+
createColumn<Nullable<String>>({"www.pingcap.com", "ww.pingcap.com", "w.pingcap.com", ".pingcap.com"}),
160+
createColumn<T1>({1, 2, 3, 4}),
161+
createConstColumn<T1>(4, 4)));
162+
}
163+
};
164+
31165
TEST_F(SubString, subStringUTF8Test)
32166
try
33167
{
168+
TestTypePair<TestNullableIntTypes, TestNullableIntTypes, TestNullableSigned, SubString>::run(*this);
169+
TestTypePair<TestAllIntTypes, TestAllIntTypes, TestSigned, SubString>::run(*this);
170+
171+
TestTypePair<TestNullableIntTypes, TestNullableUIntTypes, TestNullableUnsigned, SubString>::run(*this);
172+
TestTypePair<TestNullableUIntTypes, TestNullableIntTypes, TestNullableUnsigned, SubString>::run(*this);
173+
TestTypePair<TestNullableUIntTypes, TestNullableUIntTypes, TestNullableUnsigned, SubString>::run(*this);
174+
175+
TestTypePair<TestAllIntTypes, TestAllUIntTypes, TestUnsigned, SubString>::run(*this);
176+
TestTypePair<TestAllUIntTypes, TestAllIntTypes, TestUnsigned, SubString>::run(*this);
177+
TestTypePair<TestAllUIntTypes, TestAllUIntTypes, TestUnsigned, SubString>::run(*this);
178+
179+
TestTypePair<TestAllIntTypes, TestAllIntTypes, TestConstPos, SubString>::run(*this);
180+
TestTypePair<TestAllUIntTypes, TestAllUIntTypes, TestConstPos, SubString>::run(*this);
181+
182+
TestTypePair<TestAllIntTypes, TestAllIntTypes, TestConstLength, SubString>::run(*this);
183+
TestTypePair<TestAllUIntTypes, TestAllUIntTypes, TestConstLength, SubString>::run(*this);
184+
34185
// column, const, const
35186
ASSERT_COLUMN_EQ(
36187
createColumn<Nullable<String>>({"www.", "ww.p", "w.pi", ".pin"}),
@@ -39,6 +190,7 @@ try
39190
createColumn<Nullable<String>>({"www.pingcap.com", "ww.pingcap.com", "w.pingcap.com", ".pingcap.com"}),
40191
createConstColumn<Nullable<Int64>>(4, 1),
41192
createConstColumn<Nullable<Int64>>(4, 4)));
193+
42194
// const, const, const
43195
ASSERT_COLUMN_EQ(
44196
createConstColumn<String>(1, "www."),
@@ -47,17 +199,8 @@ try
47199
createConstColumn<Nullable<String>>(1, "www.pingcap.com"),
48200
createConstColumn<Nullable<Int64>>(1, 1),
49201
createConstColumn<Nullable<Int64>>(1, 4)));
50-
// Test Null
51-
ASSERT_COLUMN_EQ(
52-
createColumn<Nullable<String>>({{}, "www."}),
53-
executeFunction(
54-
"substringUTF8",
55-
createColumn<Nullable<String>>(
56-
{{}, "www.pingcap.com"}),
57-
createConstColumn<Nullable<Int64>>(2, 1),
58-
createConstColumn<Nullable<Int64>>(2, 4)));
59202
}
60203
CATCH
61204

62205
} // namespace tests
63-
} // namespace DB
206+
} // namespace DB

0 commit comments

Comments
 (0)