Skip to content

Commit a328b58

Browse files
ivanceaivancea
authored
ESQL: Time_zone tests (#136748)
Continuation of #136637 Configure tests to work with SET, and add/update tests for: - Request "time_zone" parameter - "SET" "time_zone" parameter - Functions currently affected by `Configuration.zoneId` Also, use random configurations by default for unit tests, so we catch and update functions not testing configuration changes.
1 parent c5b0360 commit a328b58

File tree

22 files changed

+481
-92
lines changed

22 files changed

+481
-92
lines changed

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,15 @@ static CsvSpecReader.CsvTestCase convertToRemoteIndices(CsvSpecReader.CsvTestCas
301301
String first = commands[0].trim();
302302
// If true, we're using *:index, otherwise we're using *:index,index
303303
boolean onlyRemotes = canUseRemoteIndicesOnly() && randomBoolean();
304-
String[] commandParts = first.split("\\s+", 2);
304+
305+
// Split "SET a=b; FROM x" into "SET a=b" and "FROM x"
306+
int lastSetDelimiterPosition = first.lastIndexOf(';');
307+
String setStatements = lastSetDelimiterPosition == -1 ? "" : first.substring(0, lastSetDelimiterPosition + 1);
308+
String afterSetStatements = lastSetDelimiterPosition == -1 ? first : first.substring(lastSetDelimiterPosition + 1);
309+
310+
// Split "FROM a, b, c" into "FROM" and "a, b, c"
311+
String[] commandParts = afterSetStatements.trim().split("\\s+", 2);
312+
305313
String command = commandParts[0].trim();
306314
if (command.equalsIgnoreCase("from") || command.equalsIgnoreCase("ts")) {
307315
String[] indexMetadataParts = commandParts[1].split("(?i)\\bmetadata\\b", 2);
@@ -323,7 +331,7 @@ static CsvSpecReader.CsvTestCase convertToRemoteIndices(CsvSpecReader.CsvTestCas
323331
+ remoteIndices
324332
+ " "
325333
+ (indexMetadataParts.length == 1 ? "" : "metadata " + indexMetadataParts[1]);
326-
testCase.query = newFirstCommand + query.substring(first.length());
334+
testCase.query = setStatements + newFirstCommand + query.substring(first.length());
327335
}
328336
}
329337

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvSpecReader.java‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@
1313
import java.util.function.Function;
1414
import java.util.regex.Pattern;
1515

16-
import static org.hamcrest.CoreMatchers.is;
17-
import static org.junit.Assert.assertThat;
18-
1916
public final class CsvSpecReader {
2017

2118
private CsvSpecReader() {}
@@ -25,9 +22,6 @@ public static SpecReader.Parser specParser() {
2522
}
2623

2724
public static class CsvSpecParser implements SpecReader.Parser {
28-
private static final String SCHEMA_PREFIX = "schema::";
29-
30-
private final StringBuilder earlySchema = new StringBuilder();
3125
private final StringBuilder query = new StringBuilder();
3226
private final StringBuilder data = new StringBuilder();
3327
private final List<String> requiredCapabilities = new ArrayList<>();
@@ -39,21 +33,22 @@ private CsvSpecParser() {}
3933
public Object parse(String line) {
4034
// read the query
4135
if (testCase == null) {
42-
if (line.startsWith(SCHEMA_PREFIX)) {
43-
assertThat("Early schema already declared " + earlySchema, earlySchema.length(), is(0));
44-
earlySchema.append(line.substring(SCHEMA_PREFIX.length()).trim());
45-
} else if (line.toLowerCase(Locale.ROOT).startsWith("required_capability:")) {
36+
if (line.toLowerCase(Locale.ROOT).startsWith("required_capability:")) {
4637
requiredCapabilities.add(line.substring("required_capability:".length()).trim());
4738
} else {
48-
if (line.endsWith(";")) {
39+
if (line.endsWith("\\;")) {
40+
// SET statement with escaped ";"
41+
var updatedLine = line.substring(0, line.length() - 2);
42+
query.append(updatedLine);
43+
query.append(";");
44+
query.append("\r\n");
45+
} else if (line.endsWith(";")) {
4946
// pick up the query
5047
testCase = new CsvTestCase();
5148
query.append(line.substring(0, line.length() - 1).trim());
5249
testCase.query = query.toString();
53-
testCase.earlySchema = earlySchema.toString();
5450
testCase.requiredCapabilities = List.copyOf(requiredCapabilities);
5551
requiredCapabilities.clear();
56-
earlySchema.setLength(0);
5752
query.setLength(0);
5853
}
5954
// keep reading the query
@@ -109,7 +104,6 @@ private static Pattern warningRegexToPattern(String regex) {
109104

110105
public static class CsvTestCase {
111106
public String query;
112-
public String earlySchema;
113107
public String expectedResults;
114108
private final List<String> expectedWarnings = new ArrayList<>();
115109
private final List<String> expectedWarningsRegexString = new ArrayList<>();

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,6 @@
8484
import org.elasticsearch.xpack.esql.core.tree.Source;
8585
import org.elasticsearch.xpack.esql.core.type.DataType;
8686
import org.elasticsearch.xpack.esql.core.type.EsField;
87-
import org.elasticsearch.xpack.esql.core.util.DateUtils;
8887
import org.elasticsearch.xpack.esql.core.util.Holder;
8988
import org.elasticsearch.xpack.esql.core.util.StringUtils;
9089
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
@@ -107,7 +106,9 @@
107106
import org.elasticsearch.xpack.esql.inference.InferenceService;
108107
import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
109108
import org.elasticsearch.xpack.esql.parser.QueryParam;
109+
import org.elasticsearch.xpack.esql.plan.EsqlStatement;
110110
import org.elasticsearch.xpack.esql.plan.IndexPattern;
111+
import org.elasticsearch.xpack.esql.plan.QuerySettings;
111112
import org.elasticsearch.xpack.esql.plan.logical.Enrich;
112113
import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
113114
import org.elasticsearch.xpack.esql.plan.logical.Explain;
@@ -563,9 +564,9 @@ private static ThreadPool createMockThreadPool() {
563564

564565
private EsqlTestUtils() {}
565566

566-
public static Configuration configuration(QueryPragmas pragmas, String query) {
567+
public static Configuration configuration(QueryPragmas pragmas, String query, EsqlStatement statement) {
567568
return new Configuration(
568-
DateUtils.UTC,
569+
statement.setting(QuerySettings.TIME_ZONE),
569570
Locale.US,
570571
null,
571572
null,
@@ -582,12 +583,16 @@ public static Configuration configuration(QueryPragmas pragmas, String query) {
582583
);
583584
}
584585

586+
public static Configuration configuration(QueryPragmas pragmas, String query) {
587+
return configuration(pragmas, query, new EsqlStatement(null, List.of()));
588+
}
589+
585590
public static Configuration configuration(QueryPragmas pragmas) {
586591
return configuration(pragmas, StringUtils.EMPTY);
587592
}
588593

589594
public static Configuration configuration(String query) {
590-
return configuration(new QueryPragmas(Settings.EMPTY), query);
595+
return configuration(QueryPragmas.EMPTY, query);
591596
}
592597

593598
public static AnalyzerSettings queryClusterSettings() {

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec‎

Lines changed: 76 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@ emp_no:integer | x:keyword | y:keyword
5555
10061 | 1985-09-17T00:00:00.000Z | 1985-09-17
5656
;
5757

58-
5958
compareToString
6059
from employees | where hire_date < "1985-03-01T00:00:00Z" | keep emp_no, hire_date;
6160
ignoreOrder:true
@@ -1085,6 +1084,33 @@ date:date | year:long
10851084
2022-05-06T00:00:00.000Z | 2022
10861085
;
10871086

1087+
dateExtractSetTimezoneFrom
1088+
required_capability: global_timezone_parameter
1089+
1090+
set time_zone="+07:00"\;
1091+
from employees
1092+
| sort hire_date
1093+
| eval hour = date_extract("hour_of_day", hire_date)
1094+
| keep emp_no, hire_date, hour
1095+
| limit 2;
1096+
1097+
emp_no:integer | hire_date:date | hour:long
1098+
10009 | 1985-02-18T00:00:00.000Z | 7
1099+
10048 | 1985-02-24T00:00:00.000Z | 7
1100+
;
1101+
1102+
dateExtractSetTimezoneRow
1103+
required_capability: global_timezone_parameter
1104+
1105+
set time_zone="+07:00"\;
1106+
ROW hire_date = "2020-02-28T23:00:00.000Z"::date
1107+
| EVAL hour = date_extract("hour_of_day", hire_date)
1108+
| KEEP hour;
1109+
1110+
hour:long
1111+
6
1112+
;
1113+
10881114
docsDateExtractBusinessHours
10891115
// tag::docsDateExtractBusinessHours[]
10901116
FROM sample_data
@@ -1971,6 +1997,30 @@ emp_no:integer | birth_date:date | hire_date:date
19711997
10040 | null | 1993-02-14T00:00:00.000Z | null
19721998
;
19731999

2000+
dayNameSetTimezoneRow
2001+
required_capability: global_timezone_parameter
2002+
2003+
set time_zone="-02:00"\;
2004+
row dt = to_datetime("1953-09-02T00:00:00.000Z")
2005+
| eval weekday = day_name(dt);
2006+
2007+
dt:date | weekday:keyword
2008+
1953-09-02T00:00:00.000Z | Tuesday
2009+
;
2010+
2011+
dayNameSetTimezoneFrom
2012+
required_capability: global_timezone_parameter
2013+
2014+
set time_zone="-02:00"\;
2015+
from employees
2016+
| sort emp_no
2017+
| keep emp_no, hire_date
2018+
| eval day = day_name(hire_date)
2019+
| limit 1;
2020+
2021+
emp_no:integer | hire_date:date | day:keyword
2022+
10001 | 1986-06-26T00:00:00.000Z | Wednesday
2023+
;
19742024

19752025
monthNameRowTest
19762026
required_capability:fn_month_name
@@ -2068,3 +2118,28 @@ from employees
20682118
emp_no:integer | birth_date:date | hire_date:date | monthName:keyword
20692119
10040 | null | 1993-02-14T00:00:00.000Z | null
20702120
;
2121+
2122+
monthNameSetTimezoneRow
2123+
required_capability: global_timezone_parameter
2124+
2125+
set time_zone="Europe/Paris"\;
2126+
row dt = to_datetime("1996-01-31T23:00:00.000Z")
2127+
| eval monthName = MONTH_NAME(dt);
2128+
2129+
dt:date | monthName:keyword
2130+
1996-01-31T23:00:00.000Z | February
2131+
;
2132+
2133+
monthNameSetTimezoneFrom
2134+
required_capability: global_timezone_parameter
2135+
2136+
set time_zone="-10:00"\;
2137+
from employees
2138+
| WHERE emp_no == 10004
2139+
| keep emp_no, hire_date
2140+
| eval monthName = month_name(hire_date)
2141+
| limit 1;
2142+
2143+
emp_no:integer | hire_date:date | monthName:keyword
2144+
10004 | 1986-12-01T00:00:00.000Z | November
2145+
;

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/set.csv-spec‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
set
2+
required_capability: global_timezone_parameter
3+
4+
set time_zone="+02:00"\;
5+
from employees
6+
| sort emp_no
7+
| keep emp_no, hire_date
8+
| eval hour = date_extract("hour_of_day", hire_date)
9+
| limit 1;
10+
11+
emp_no:integer | hire_date:date | hour:long
12+
10001 | 1986-06-26T00:00:00.000Z | 2
13+
;
14+
15+
set with foldable
16+
required_capability: global_timezone_parameter
17+
18+
set time_zone="+02:00"\;
19+
ROW date = "1986-06-26T00:00:00.000Z"::date
20+
| eval hour = date_extract("hour_of_day", date)
21+
;
22+
23+
date:date | hour:long
24+
1986-06-26T00:00:00.000Z | 2
25+
;
26+
27+
last set prevails
28+
required_capability: global_timezone_parameter
29+
30+
set time_zone="+02:00"\;
31+
set time_zone="+05:00"\;
32+
from employees
33+
| sort emp_no
34+
| keep emp_no, hire_date
35+
| eval hour = date_extract("hour_of_day", hire_date)
36+
| limit 1;
37+
38+
emp_no:integer | hire_date:date | hour:long
39+
10001 | 1986-06-26T00:00:00.000Z | 5
40+
;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1548,6 +1548,11 @@ public enum Cap {
15481548
*/
15491549
FIX_FILTER_ORDINALS,
15501550

1551+
/**
1552+
* "time_zone" parameter in request body and in {@code SET "time_zone"="x"}
1553+
*/
1554+
GLOBAL_TIMEZONE_PARAMETER(Build.current().isSnapshot()),
1555+
15511556
/**
15521557
* Optional options argument for DATE_PARSE
15531558
*/

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,11 @@ public LogicalPlan createStatement(String query, QueryParams params, PlanTelemet
115115
return invokeParser(query, params, metrics, EsqlBaseParser::singleStatement, AstBuilder::plan);
116116
}
117117

118+
// testing utility
119+
public EsqlStatement createQuery(String query) {
120+
return createQuery(query, new QueryParams());
121+
}
122+
118123
// testing utility
119124
public EsqlStatement createQuery(String query, QueryParams params) {
120125
return createQuery(query, params, new PlanTelemetry(new EsqlFunctionRegistry()));

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ public long absoluteStartedTimeInMillis() {
202202
/**
203203
* @return Start time of the ESQL query in nanos
204204
*/
205-
public long getQueryStartTimeNanos() {
205+
public long queryStartTimeNanos() {
206206
return queryStartTimeNanos;
207207
}
208208

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@
8080
import org.elasticsearch.xpack.esql.optimizer.LogicalPreOptimizerContext;
8181
import org.elasticsearch.xpack.esql.optimizer.TestLocalPhysicalPlanOptimizer;
8282
import org.elasticsearch.xpack.esql.parser.EsqlParser;
83+
import org.elasticsearch.xpack.esql.plan.EsqlStatement;
8384
import org.elasticsearch.xpack.esql.plan.IndexPattern;
8485
import org.elasticsearch.xpack.esql.plan.logical.Enrich;
8586
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
@@ -191,9 +192,13 @@ public class CsvTests extends ESTestCase {
191192
private final CsvSpecReader.CsvTestCase testCase;
192193
private final String instructions;
193194

194-
private final Configuration configuration = EsqlTestUtils.configuration(
195-
new QueryPragmas(Settings.builder().put("page_size", randomPageSize()).build())
196-
);
195+
/**
196+
* The configuration to be used in the tests.
197+
* <p>
198+
* Initialized in {@link #executePlan}.
199+
* </p>
200+
*/
201+
private Configuration configuration;
197202
private final EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry();
198203
private final EsqlParser parser = new EsqlParser();
199204
private final Mapper mapper = new Mapper();
@@ -551,6 +556,7 @@ private static EnrichPolicy loadEnrichPolicyMapping(String policyFileName) {
551556

552557
private LogicalPlan analyzedPlan(
553558
LogicalPlan parsed,
559+
Configuration configuration,
554560
Map<IndexPattern, CsvTestsDataLoader.MultiIndexTestDataset> datasets,
555561
TransportVersion minimumVersion
556562
) {
@@ -638,11 +644,16 @@ private static TestPhysicalOperationProviders testOperationProviders(
638644
}
639645

640646
private ActualResults executePlan(BigArrays bigArrays) throws Exception {
641-
LogicalPlan parsed = parser.createStatement(testCase.query);
642-
var testDatasets = testDatasets(parsed);
647+
EsqlStatement statement = parser.createQuery(testCase.query);
648+
this.configuration = EsqlTestUtils.configuration(
649+
new QueryPragmas(Settings.builder().put("page_size", randomPageSize()).build()),
650+
testCase.query,
651+
statement
652+
);
653+
var testDatasets = testDatasets(statement.plan());
643654
// Specifically use the newest transport version; the csv tests correspond to a single node cluster on the current version.
644655
TransportVersion minimumVersion = TransportVersion.current();
645-
LogicalPlan analyzed = analyzedPlan(parsed, testDatasets, minimumVersion);
656+
LogicalPlan analyzed = analyzedPlan(statement.plan(), configuration, testDatasets, minimumVersion);
646657

647658
FoldContext foldCtx = FoldContext.small();
648659
EsqlSession session = new EsqlSession(

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@ protected static List<TestCaseSupplier> withNoRowsExpectingNull(List<TestCaseSup
116116
var newData = testCase.getData().stream().map(td -> td.isMultiRow() ? td.withData(List.of()) : td).toList();
117117

118118
return new TestCaseSupplier.TestCase(
119+
testCase.getSource(),
120+
testCase.getConfiguration(),
119121
newData,
120122
testCase.evaluatorToString(),
121123
testCase.expectedType(),

0 commit comments

Comments
 (0)