Skip to content

Commit 5909e80

Browse files
ivanceaivancea
authored
ESQL: Test KQL and QSTR with timezones in a LOOKUP JOIN expression (elastic#138979)
As the LOOKUP JOIN execution code doesn't have the configuration object, functions using it on translation to Lucene queries would fail with an NPE. Right now, only KQL and QSTR are both translatable and using the Configuration. However, on serialization, they don't need the configuration anymore, and instead they use a processed `queryBuilder`. See: https://github.com/elastic/elasticsearch/blob/f969d7de63663ef498300b44a91d30dea7af9aef/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java#L175-L177
1 parent e1b32f3 commit 5909e80

File tree

6 files changed

+114
-25
lines changed

6 files changed

+114
-25
lines changed

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@
180180
import java.util.TreeMap;
181181
import java.util.function.Predicate;
182182
import java.util.jar.JarInputStream;
183+
import java.util.regex.Pattern;
183184
import java.util.stream.DoubleStream;
184185
import java.util.stream.IntStream;
185186
import java.util.zip.ZipEntry;
@@ -1341,20 +1342,27 @@ private static PhysicalPlan ignoreIdsInPhysicalPlan(PhysicalPlan plan) {
13411342
});
13421343
}
13431344

1345+
private static final Pattern SET_SPLIT_PATTERN = Pattern.compile("^(\\s*SET\\b[^;]+;)+\\s*\\b", Pattern.CASE_INSENSITIVE);
1346+
13441347
public static String addRemoteIndices(String query, Set<String> lookupIndices, boolean onlyRemotes) {
13451348
String[] commands = query.split("\\|");
13461349
// remove subqueries
13471350
String first = commands[0].split(",\\s+\\(")[0].trim();
13481351

1349-
// Split "SET a=b; FROM x" into "SET a=b" and "FROM x"
1350-
int lastSetDelimiterPosition = first.lastIndexOf(';');
1351-
String setStatements = lastSetDelimiterPosition == -1 ? "" : first.substring(0, lastSetDelimiterPosition + 1);
1352-
String afterSetStatements = lastSetDelimiterPosition == -1 ? first : first.substring(lastSetDelimiterPosition + 1);
1352+
// Split "SET a=b; FROM x" into "SET a=b; " and "FROM x"
1353+
var setMatcher = SET_SPLIT_PATTERN.matcher(first);
1354+
int lastSetDelimiterPosition = -1;
1355+
if (setMatcher.find()) {
1356+
lastSetDelimiterPosition = setMatcher.end();
1357+
}
1358+
String setStatements = lastSetDelimiterPosition == -1 ? "" : first.substring(0, lastSetDelimiterPosition);
1359+
String afterSetStatements = lastSetDelimiterPosition == -1 ? first : first.substring(lastSetDelimiterPosition);
13531360

13541361
// Split "FROM a, b, c" into "FROM" and "a, b, c"
13551362
String[] commandParts = afterSetStatements.trim().split("\\s+", 2);
13561363

13571364
String command = commandParts[0].trim();
1365+
assert command.equalsIgnoreCase("set") == false : "didn't correctly extract the SET statement from the query";
13581366
if (SourceCommand.isSourceCommand(command)) {
13591367
String commandArgs = commandParts[1].trim();
13601368
String[] indices = EsqlParser.INSTANCE.parseQuery(afterSetStatements)
@@ -1386,7 +1394,10 @@ public static String addRemoteIndices(String query, Set<String> lookupIndices, b
13861394
i += newIndex.length();
13871395
}
13881396
String newFirstCommand = command + " " + commandArgs;
1389-
return (setStatements + " " + newFirstCommand.trim() + query.substring(first.length())).trim();
1397+
String finalQuery = (setStatements + newFirstCommand.trim() + query.substring(first.length()));
1398+
assert query.split("\n").length == finalQuery.split("\n").length
1399+
: "the final query should have the same lines for warnings to work";
1400+
return finalQuery;
13901401
}
13911402
}
13921403
return query;

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/multi_column_joinable_lookup.csv‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
1-
id_int,name_str,is_active_bool,ip_addr,other1,other2
2-
1,Alice,true,192.168.1.1,alpha,1000
3-
1,Alice,true,192.168.1.2,beta,2000
4-
2,Bob,false,192.168.1.3,gamma,3000
5-
3,Charlie,true,192.168.1.3,delta,4000
6-
3,Charlie,false,192.168.1.3,epsilon,5000
7-
4,David,false,192.168.1.4,zeta,6000
8-
5,Eve,true,192.168.1.5,eta,7000
9-
5,Eve,true,192.168.1.5,theta,8000
10-
6,,true,192.168.1.6,iota,9000
11-
7,Grace,false,,kappa,10000
12-
8,Hank,true,192.168.1.8,lambda,11000
13-
,Kate,false,192.168.1.11,mu,12000
14-
12,Liam,true,192.168.1.12,nu,13000
15-
13,Mia,false,192.168.1.13,xi,14000
16-
[14],Nina,true,192.168.1.14,omicron,15000
17-
16,Paul,true,192.168.1.16,pi,16000
18-
[17,18],Olivia,true,192.168.1.17,rho,17000
19-
[1,19,20],Sophia,true,192.168.1.21,sigma,21000
1+
id_int,name_str,is_active_bool,ip_addr,date,date_nanos,other1,other2
2+
1,Alice,true,192.168.1.1,1980-02-29T23:59:59.999,1980-02-29T23:59:59.999999999,alpha,1000
3+
1,Alice,true,192.168.1.2,1999-05-02T00:00:01.000,1999-05-02T00:00:01.000000001,beta,2000
4+
2,Bob,false,192.168.1.3,2001-05-02T00:00:02.000,2001-05-02T00:00:02.000000002,gamma,3000
5+
3,Charlie,true,192.168.1.3,2003-05-02T00:00:03.123,2003-05-02T00:00:03.123456789,delta,4000
6+
3,Charlie,false,192.168.1.3,2005-05-02T00:00:04.000,2005-05-02T00:00:04.000000000,epsilon,5000
7+
4,David,false,192.168.1.4,2007-05-02T00:00:05.000,2007-05-02T00:00:05.000000500,zeta,6000
8+
5,Eve,true,192.168.1.5,2009-05-02T00:00:06.987,2009-05-02T00:00:06.987654321,eta,7000
9+
5,Eve,true,192.168.1.5,2011-05-02T00:00:07.000,2011-05-02T00:00:07.000000000,theta,8000
10+
6,,true,192.168.1.6,2013-05-02T00:00:08.000,2013-05-02T00:00:08.000000010,iota,9000
11+
7,Grace,false,,2015-05-02T00:00:09.000,2015-05-02T00:00:09.000000000,kappa,10000
12+
8,Hank,true,192.168.1.8,2016-05-02T00:00:10.314,2016-05-02T00:00:10.314159265,lambda,11000
13+
,Kate,false,192.168.1.11,2018-05-02T00:00:11.000,2018-05-02T00:00:11.000000000,mu,12000
14+
12,Liam,true,192.168.1.12,2020-05-02T00:00:12.000,2020-05-02T00:00:12.000000123,nu,13000
15+
13,Mia,false,192.168.1.13,2022-05-02T00:00:13.000,2022-05-02T00:00:13.000000000,xi,14000
16+
[14],Nina,true,192.168.1.14,2024-05-02T00:00:14.555,2024-05-02T00:00:14.555555555,omicron,15000
17+
16,Paul,true,192.168.1.16,2026-05-02T00:00:15.000,2026-05-02T00:00:15.000000000,pi,16000
18+
[17,18],Olivia,true,192.168.1.17,2028-05-02T00:00:16.000,2028-05-02T00:00:16.000000777,rho,17000
19+
[1,19,20],Sophia,true,192.168.1.21,2030-05-02T00:00:17.000,2030-05-02T00:00:17.000000000,sigma,21000

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1014,6 +1014,48 @@ id_left:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:in
10141014
1 | Alice | foo | beta | 2000
10151015
;
10161016

1017+
lookupJoinExpressionWithKqlAndTimezones
1018+
required_capability: join_lookup_v12
1019+
required_capability: lookup_join_with_full_text_function
1020+
required_capability: kql_qstr_timezone_support
1021+
1022+
SET time_zone = "Europe/Madrid"\;
1023+
FROM multi_column_joinable
1024+
| RENAME id_int AS id_left
1025+
| LOOKUP JOIN multi_column_joinable_lookup ON KQL("date:\"1980-03-01T00:59:59.999\" AND date_nanos:\"1980-03-01T00:59:59.999999999\"") AND id_int == id_left
1026+
| WHERE date IS NOT NULL
1027+
| KEEP id_left, date, date_nanos
1028+
| SORT id_left, date, date_nanos
1029+
;
1030+
1031+
warning:Line 4:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON KQL(\"date:\\\"1980-03-01T00:59:59.999\\\" AND date_nanos:\\\"1980-03-01T00:59:59.999999999\\\"\") AND id_int == id_left] failed, treating result as null. Only first 20 failures recorded.
1032+
warning:Line 4:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value
1033+
1034+
id_left:integer | date:date | date_nanos:date_nanos
1035+
1 | 1980-02-29T23:59:59.999Z | 1980-02-29T23:59:59.999999999Z
1036+
;
1037+
1038+
lookupJoinExpressionWithQstrAndTimezones
1039+
required_capability: join_lookup_v12
1040+
required_capability: lookup_join_with_full_text_function
1041+
required_capability: kql_qstr_timezone_support
1042+
1043+
SET time_zone = "Europe/Madrid"\;
1044+
FROM multi_column_joinable
1045+
| RENAME id_int AS id_left
1046+
| LOOKUP JOIN multi_column_joinable_lookup ON QSTR("date:[1980-03-01T00:59:59 TO 1980-03-01T00:59:59.999] AND date_nanos:[1980-03-01T00:59:59 TO 1980-03-01T00:59:59.999999999]") AND id_int == id_left
1047+
| WHERE date IS NOT NULL
1048+
| KEEP id_left, date, date_nanos
1049+
| SORT id_left, date, date_nanos
1050+
;
1051+
1052+
warning:Line 4:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON QSTR(\"date:[1980-03-01T00:59:59 TO 1980-03-01T00:59:59.999] AND date_nanos:[1980-03-01T00:59:59 TO 1980-03-01T00:59:59.999999999]\") AND id_int == id_left] failed, treating result as null. Only first 20 failures recorded.
1053+
warning:Line 4:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value
1054+
1055+
id_left:integer | date:date | date_nanos:date_nanos
1056+
1 | 1980-02-29T23:59:59.999Z | 1980-02-29T23:59:59.999999999Z
1057+
;
1058+
10171059
lookupJoinExpressionWithMultiMatch
10181060
required_capability: join_lookup_v12
10191061
required_capability: lookup_join_with_full_text_function

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5704,6 +5704,6 @@ from languages_lookup_non_unique_ke*
57045704
warning:Line 13:24: evaluation of [other2 == 50] failed, treating result as null. Only first 20 failures recorded.
57055705
warning:Line 13:24: java.lang.IllegalArgumentException: single-value function encountered multi-value
57065706

5707-
language_name:keyword | country:text | other2:integer | YEcxuCOHsGzb:keyword | other1:keyword |id_int:integer | ip_addr:ip |is_active_bool:boolean | name_str:keyword
5707+
language_name:keyword | country:text | other2:integer | YEcxuCOHsGzb:keyword | other1:keyword | date:datetime | date_nanos:date_nanos | id_int:integer | ip_addr:ip | is_active_bool:boolean | name_str:keyword
57085708

57095709
;

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-multi_column_joinable_lookup.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,12 @@
1212
"ip_addr": {
1313
"type": "ip"
1414
},
15+
"date": {
16+
"type": "date"
17+
},
18+
"date_nanos": {
19+
"type": "date_nanos"
20+
},
1521
"other1": {
1622
"type": "keyword"
1723
},

‎x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/EsqlTestUtilsTests.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,20 @@ public void testSet() {
5555
);
5656
}
5757

58+
public void testSetMultiline() {
59+
assertThat(EsqlTestUtils.addRemoteIndices("""
60+
SET a=b;
61+
SET c=d;
62+
FROM foo
63+
| SORT bar
64+
""", Set.of(), false), equalTo("""
65+
SET a=b;
66+
SET c=d;
67+
FROM *:foo,foo
68+
| SORT bar
69+
"""));
70+
}
71+
5872
public void testMetadata() {
5973
assertThat(
6074
EsqlTestUtils.addRemoteIndices("FROM foo METADATA _source | SORT bar", Set.of(), false),
@@ -98,6 +112,22 @@ public void testSubquery() {
98112
| KEEP _index, emp_no, languages, language_name"""));
99113
}
100114

115+
public void testSubqueryWithSet() {
116+
assertThat(EsqlTestUtils.addRemoteIndices("""
117+
SET a = b;
118+
SET x = y; FROM employees, (FROM employees_incompatible
119+
| ENRICH languages_policy on languages with language_name )
120+
metadata _index
121+
| EVAL emp_no = emp_no::long
122+
""", Set.of(), false), equalTo("""
123+
SET a = b;
124+
SET x = y; FROM *:employees,employees, (FROM employees_incompatible
125+
| ENRICH languages_policy on languages with language_name )
126+
metadata _index
127+
| EVAL emp_no = emp_no::long
128+
"""));
129+
}
130+
101131
public void testTripleQuotes() {
102132
assertThat(
103133
EsqlTestUtils.addRemoteIndices("from \"\"\"employees\"\"\" | limit 2", Set.of(), false),

0 commit comments

Comments
 (0)