Skip to content

Commit 340dddc

Browse files
xinyualxinyual
committed
Add string builtin functions (#3393)
* add string udfs Signed-off-by: xinyual <xinyual@amazon.com> * add it to string Signed-off-by: xinyual <xinyual@amazon.com> * add IT for string function Signed-off-by: xinyual <xinyual@amazon.com> * remove change for local test Signed-off-by: xinyual <xinyual@amazon.com> * revert change Signed-off-by: xinyual <xinyual@amazon.com> --------- Signed-off-by: xinyual <xinyual@amazon.com>
1 parent cec9634 commit 340dddc

File tree

2 files changed

+284
-2
lines changed

2 files changed

+284
-2
lines changed

‎core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.apache.calcite.sql.SqlOperator;
1616
import org.apache.calcite.sql.fun.SqlLibraryOperators;
1717
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
18+
import org.apache.calcite.sql.fun.SqlTrimFunction;
1819
import org.apache.calcite.sql.type.ReturnTypes;
1920
import org.opensearch.sql.calcite.CalcitePlanContext;
2021
import org.opensearch.sql.calcite.udf.mathUDF.SqrtFunction;
@@ -53,10 +54,28 @@ static SqlOperator translate(String op) {
5354
case "/":
5455
return SqlStdOperatorTable.DIVIDE;
5556
// Built-in String Functions
57+
case "CONCAT":
58+
return SqlLibraryOperators.CONCAT_FUNCTION;
59+
case "CONCAT_WS":
60+
return SqlLibraryOperators.CONCAT_WS;
61+
case "LIKE":
62+
return SqlLibraryOperators.ILIKE;
63+
case "LTRIM", "RTRIM", "TRIM":
64+
return SqlStdOperatorTable.TRIM;
65+
case "LENGTH":
66+
return SqlStdOperatorTable.CHAR_LENGTH;
5667
case "LOWER":
5768
return SqlStdOperatorTable.LOWER;
58-
case "LIKE":
59-
return SqlStdOperatorTable.LIKE;
69+
case "POSITION":
70+
return SqlStdOperatorTable.POSITION;
71+
case "REVERSE":
72+
return SqlLibraryOperators.REVERSE;
73+
case "RIGHT":
74+
return SqlLibraryOperators.RIGHT;
75+
case "SUBSTRING":
76+
return SqlStdOperatorTable.SUBSTRING;
77+
case "UPPER":
78+
return SqlStdOperatorTable.UPPER;
6079
// Built-in Math Functions
6180
case "ABS":
6281
return SqlStdOperatorTable.ABS;
@@ -99,6 +118,30 @@ static SqlOperator translate(String op) {
99118
static List<RexNode> translateArgument(
100119
String op, List<RexNode> argList, CalcitePlanContext context) {
101120
switch (op.toUpperCase(Locale.ROOT)) {
121+
case "TRIM":
122+
List<RexNode> trimArgs =
123+
new ArrayList<>(
124+
List.of(
125+
context.rexBuilder.makeFlag(SqlTrimFunction.Flag.BOTH),
126+
context.rexBuilder.makeLiteral(" ")));
127+
trimArgs.addAll(argList);
128+
return trimArgs;
129+
case "LTRIM":
130+
List<RexNode> LTrimArgs =
131+
new ArrayList<>(
132+
List.of(
133+
context.rexBuilder.makeFlag(SqlTrimFunction.Flag.LEADING),
134+
context.rexBuilder.makeLiteral(" ")));
135+
LTrimArgs.addAll(argList);
136+
return LTrimArgs;
137+
case "RTRIM":
138+
List<RexNode> RTrimArgs =
139+
new ArrayList<>(
140+
List.of(
141+
context.rexBuilder.makeFlag(SqlTrimFunction.Flag.TRAILING),
142+
context.rexBuilder.makeLiteral(" ")));
143+
RTrimArgs.addAll(argList);
144+
return RTrimArgs;
102145
case "ATAN":
103146
List<RexNode> AtanArgs = new ArrayList<>(argList);
104147
if (AtanArgs.size() == 1) {

‎integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import java.io.IOException;
1212
import org.json.JSONObject;
1313
import org.junit.jupiter.api.Test;
14+
import org.opensearch.client.Request;
1415

1516
public class CalcitePPLBuiltinFunctionIT extends CalcitePPLIntegTestCase {
1617
@Override
@@ -31,4 +32,242 @@ public void testSqrtAndPow() {
3132

3233
verifyDataRows(actual, rows("Hello", 30));
3334
}
35+
36+
// Test
37+
@Test
38+
public void testConcat() {
39+
JSONObject actual =
40+
executeQuery(
41+
String.format(
42+
"source=%s | where name=concat('He', 'llo') | fields name, age",
43+
TEST_INDEX_STATE_COUNTRY));
44+
45+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
46+
47+
verifyDataRows(actual, rows("Hello", 30));
48+
}
49+
50+
@Test
51+
public void testConcatWithField() throws IOException {
52+
Request request1 =
53+
new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true");
54+
request1.setJsonEntity(
55+
"{\"name\":\"HelloWay\",\"age\":27,\"state\":\"Way\",\"country\":\"Canada\",\"year\":2023,\"month\":4}");
56+
client().performRequest(request1);
57+
JSONObject actual =
58+
executeQuery(
59+
String.format(
60+
"source=%s | where name=concat('Hello', state) | fields name, age",
61+
TEST_INDEX_STATE_COUNTRY));
62+
63+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
64+
65+
verifyDataRows(actual, rows("HelloWay", 27));
66+
}
67+
68+
@Test
69+
public void testConcatWs() throws IOException {
70+
Request request1 =
71+
new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true");
72+
request1.setJsonEntity(
73+
"{\"name\":\"John,Way\",\"age\":27,\"state\":\"Way\",\"country\":\"Canada\",\"year\":2023,\"month\":4}");
74+
client().performRequest(request1);
75+
JSONObject actual =
76+
executeQuery(
77+
String.format(
78+
"source=%s | where name=concat_ws(',', 'John', state) | fields name, age",
79+
TEST_INDEX_STATE_COUNTRY));
80+
81+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
82+
83+
verifyDataRows(actual, rows("John,Way", 27));
84+
}
85+
86+
@Test
87+
public void testLength() {
88+
JSONObject actual =
89+
executeQuery(
90+
String.format(
91+
"source=%s | where length(name) = 5 | fields name, age", TEST_INDEX_STATE_COUNTRY));
92+
93+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
94+
95+
verifyDataRows(actual, rows("Hello", 30));
96+
}
97+
98+
@Test
99+
public void testLengthShouldBeInsensitive() {
100+
JSONObject actual =
101+
executeQuery(
102+
String.format(
103+
"source=%s | where leNgTh(name) = 5 | fields name, age", TEST_INDEX_STATE_COUNTRY));
104+
105+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
106+
107+
verifyDataRows(actual, rows("Hello", 30));
108+
}
109+
110+
@Test
111+
public void testLower() {
112+
JSONObject actual =
113+
executeQuery(
114+
String.format(
115+
"source=%s | where lower(name) = 'hello' | fields name, age",
116+
TEST_INDEX_STATE_COUNTRY));
117+
118+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
119+
120+
verifyDataRows(actual, rows("Hello", 30));
121+
}
122+
123+
@Test
124+
public void testUpper() {
125+
JSONObject actual =
126+
executeQuery(
127+
String.format(
128+
"source=%s | where upper(name) = upper('hello') | fields name, age",
129+
TEST_INDEX_STATE_COUNTRY));
130+
131+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
132+
133+
verifyDataRows(actual, rows("Hello", 30));
134+
}
135+
136+
@Test
137+
public void testLike() {
138+
JSONObject actual =
139+
executeQuery(
140+
String.format(
141+
"source=%s | where like(name, '_ello%%') | fields name, age",
142+
TEST_INDEX_STATE_COUNTRY));
143+
144+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
145+
146+
verifyDataRows(actual, rows("Hello", 30));
147+
}
148+
149+
@Test
150+
public void testSubstring() {
151+
JSONObject actual =
152+
executeQuery(
153+
String.format(
154+
"source=%s | where substring(name, 2, 2) = 'el' | fields name, age",
155+
TEST_INDEX_STATE_COUNTRY));
156+
157+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
158+
159+
verifyDataRows(actual, rows("Hello", 30));
160+
}
161+
162+
@Test
163+
public void testPosition() {
164+
JSONObject actual =
165+
executeQuery(
166+
String.format(
167+
"source=%s | where position('He' in name) = 1 | fields name, age",
168+
TEST_INDEX_STATE_COUNTRY));
169+
170+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
171+
172+
verifyDataRows(actual, rows("Hello", 30));
173+
}
174+
175+
@Test
176+
public void testTrim() throws IOException {
177+
prepareTrim();
178+
JSONObject actual =
179+
executeQuery(
180+
String.format(
181+
"source=%s | where Trim(name) = 'Jim' | fields name, age",
182+
TEST_INDEX_STATE_COUNTRY));
183+
184+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
185+
186+
verifyDataRows(actual, rows(" Jim", 27), rows("Jim ", 57), rows(" Jim ", 70));
187+
}
188+
189+
@Test
190+
public void testRTrim() throws IOException {
191+
prepareTrim();
192+
JSONObject actual =
193+
executeQuery(
194+
String.format(
195+
"source=%s | where RTrim(name) = 'Jim' | fields name, age",
196+
TEST_INDEX_STATE_COUNTRY));
197+
198+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
199+
200+
verifyDataRows(actual, rows("Jim ", 57));
201+
}
202+
203+
@Test
204+
public void testLTrim() throws IOException {
205+
prepareTrim();
206+
JSONObject actual =
207+
executeQuery(
208+
String.format(
209+
"source=%s | where LTrim(name) = 'Jim' | fields name, age",
210+
TEST_INDEX_STATE_COUNTRY));
211+
212+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
213+
214+
verifyDataRows(actual, rows(" Jim", 27));
215+
}
216+
217+
@Test
218+
public void testReverse() throws IOException {
219+
Request request1 =
220+
new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true");
221+
request1.setJsonEntity(
222+
"{\"name\":\"DeD\",\"age\":27,\"state\":\"B.C\",\"country\":\"Canada\",\"year\":2023,\"month\":4}");
223+
client().performRequest(request1);
224+
JSONObject actual =
225+
executeQuery(
226+
String.format(
227+
"source=%s | where Reverse(name) = name | fields name, age",
228+
TEST_INDEX_STATE_COUNTRY));
229+
230+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
231+
232+
verifyDataRows(actual, rows("DeD", 27));
233+
}
234+
235+
@Test
236+
public void testRight() throws IOException {
237+
Request request1 =
238+
new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true");
239+
request1.setJsonEntity(
240+
"{\"name\":\"DeD\",\"age\":27,\"state\":\"B.C\",\"country\":\"Canada\",\"year\":2023,\"month\":4}");
241+
client().performRequest(request1);
242+
JSONObject actual =
243+
executeQuery(
244+
String.format(
245+
"source=%s | where right(name, 2) = 'lo' | fields name, age",
246+
TEST_INDEX_STATE_COUNTRY));
247+
248+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
249+
250+
verifyDataRows(actual, rows("Hello", 30));
251+
}
252+
253+
private void prepareTrim() throws IOException {
254+
Request request1 =
255+
new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true");
256+
request1.setJsonEntity(
257+
"{\"name\":\" "
258+
+ " Jim\",\"age\":27,\"state\":\"B.C\",\"country\":\"Canada\",\"year\":2023,\"month\":4}");
259+
client().performRequest(request1);
260+
Request request2 =
261+
new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/6?refresh=true");
262+
request2.setJsonEntity(
263+
"{\"name\":\"Jim "
264+
+ " \",\"age\":57,\"state\":\"B.C\",\"country\":\"Canada\",\"year\":2023,\"month\":4}");
265+
client().performRequest(request2);
266+
Request request3 =
267+
new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/7?refresh=true");
268+
request3.setJsonEntity(
269+
"{\"name\":\" Jim "
270+
+ " \",\"age\":70,\"state\":\"B.C\",\"country\":\"Canada\",\"year\":2023,\"month\":4}");
271+
client().performRequest(request3);
272+
}
34273
}

0 commit comments

Comments
 (0)