Skip to content

Commit fc20a26

Browse files
bpinteabpintea
authored
ESQL: allow DATE_PARSE to read the timezones (#118603) (#119119)
This just removes fixing a formatter to a timezone (UTC), allowing `DATE_PARSE` to correctly read timezones. Fixes #117680.
1 parent f28df38 commit fc20a26

File tree

6 files changed

+100
-29
lines changed

6 files changed

+100
-29
lines changed

‎docs/changelog/118603.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 118603
2+
summary: Allow DATE_PARSE to read the timezones
3+
area: ES|QL
4+
type: bug
5+
issues:
6+
- 117680

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -495,6 +495,60 @@ b:datetime
495495
null
496496
;
497497

498+
evalDateParseWithTimezone
499+
required_capability: date_parse_tz
500+
row s = "12/Jul/2022:10:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s);
501+
502+
s:keyword | d:datetime
503+
12/Jul/2022:10:24:10 +0900 | 2022-07-12T01:24:10.000Z
504+
;
505+
506+
evalDateParseWithTimezoneCrossingDayBoundary
507+
required_capability: date_parse_tz
508+
row s = "12/Jul/2022:08:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s);
509+
510+
s:keyword | d:datetime
511+
12/Jul/2022:08:24:10 +0900 | 2022-07-11T23:24:10.000Z
512+
;
513+
514+
evalDateParseWithTimezone2
515+
required_capability: date_parse_tz
516+
row s1 = "12/Jul/2022:10:24:10 +0900", s2 = "2022/12/07 09:24:10 +0800"
517+
| eval d1 = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s1), d2 = date_parse("yyyy/dd/MM HH:mm:ss Z", s2)
518+
| eval eq = d1 == d2
519+
| keep d1, eq
520+
;
521+
522+
d1:datetime | eq:boolean
523+
2022-07-12T01:24:10.000Z | true
524+
;
525+
526+
evalDateParseWithAndWithoutTimezone
527+
required_capability: date_parse_tz
528+
row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss"
529+
| eval no_tz = date_parse(format, s)
530+
| eval with_tz = date_parse(concat(format, " Z"), concat(s, " +0900"))
531+
| keep s, no_tz, with_tz
532+
;
533+
534+
s:keyword | no_tz:datetime | with_tz:datetime
535+
2022/12/07 09:24:10 | 2022-07-12T09:24:10.000Z | 2022-07-12T00:24:10.000Z
536+
;
537+
538+
evalDateParseWithOtherTimezoneSpecifiers
539+
required_capability: date_parse_tz
540+
row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss"
541+
| eval with_tz1 = date_parse(concat(format, " Z"), concat(s, " +0900"))
542+
| eval with_tz2 = date_parse(concat(format, " x"), concat(s, " +09"))
543+
| eval with_tz3 = date_parse(concat(format, " X"), concat(s, " +0900"))
544+
| eval with_tz4 = date_parse(concat(format, " O"), concat(s, " GMT+9"))
545+
| keep s, with_tz*
546+
;
547+
548+
s:keyword | with_tz1:datetime | with_tz2:datetime | with_tz3:datetime | with_tz4:datetime
549+
2022/12/07 09:24:10 | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z
550+
;
551+
498552
evalDateParseDynamic
499553
from employees | where emp_no == 10039 or emp_no == 10040 | sort emp_no
500554
| eval birth_date_string = date_format("yyyy-MM-dd", birth_date)

‎x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java‎

Lines changed: 7 additions & 14 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,11 @@ public enum Cap {
307307
*/
308308
TO_DATE_NANOS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
309309

310+
/**
311+
* DATE_PARSE supports reading timezones
312+
*/
313+
DATE_PARSE_TZ(),
314+
310315
/**
311316
* Support for datetime in least and greatest functions
312317
*/

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,12 @@
2828
import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
2929

3030
import java.io.IOException;
31-
import java.time.ZoneId;
3231
import java.util.List;
3332

3433
import static org.elasticsearch.common.time.DateFormatter.forPattern;
3534
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
3635
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
3736
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
38-
import static org.elasticsearch.xpack.esql.core.util.DateUtils.UTC;
3937
import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact;
4038
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
4139
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong;
@@ -130,13 +128,12 @@ public static long process(BytesRef val, @Fixed DateFormatter formatter) throws
130128
}
131129

132130
@Evaluator(warnExceptions = { IllegalArgumentException.class })
133-
static long process(BytesRef val, BytesRef formatter, @Fixed ZoneId zoneId) throws IllegalArgumentException {
134-
return dateTimeToLong(val.utf8ToString(), toFormatter(formatter, zoneId));
131+
static long process(BytesRef val, BytesRef formatter) throws IllegalArgumentException {
132+
return dateTimeToLong(val.utf8ToString(), toFormatter(formatter));
135133
}
136134

137135
@Override
138136
public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
139-
ZoneId zone = UTC; // TODO session timezone?
140137
ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(field);
141138
if (format == null) {
142139
return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, DEFAULT_DATE_TIME_FORMATTER);
@@ -146,18 +143,18 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
146143
}
147144
if (format.foldable()) {
148145
try {
149-
DateFormatter formatter = toFormatter(format.fold(), zone);
146+
DateFormatter formatter = toFormatter(format.fold());
150147
return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, formatter);
151148
} catch (IllegalArgumentException e) {
152149
throw new InvalidArgumentException(e, "invalid date pattern for [{}]: {}", sourceText(), e.getMessage());
153150
}
154151
}
155152
ExpressionEvaluator.Factory formatEvaluator = toEvaluator.apply(format);
156-
return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator, zone);
153+
return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator);
157154
}
158155

159-
private static DateFormatter toFormatter(Object format, ZoneId zone) {
160-
return forPattern(((BytesRef) format).utf8ToString()).withZone(zone);
156+
private static DateFormatter toFormatter(Object format) {
157+
return forPattern(((BytesRef) format).utf8ToString());
161158
}
162159

163160
@Override

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
2222

2323
import java.util.List;
24+
import java.util.Locale;
2425
import java.util.function.Supplier;
2526

2627
import static org.hamcrest.Matchers.equalTo;
@@ -46,11 +47,26 @@ public static Iterable<Object[]> parameters() {
4647
new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
4748
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
4849
),
49-
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
50+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
5051
DataType.DATETIME,
5152
equalTo(1683244800000L)
5253
)
5354
),
55+
new TestCaseSupplier("Timezoned Case", List.of(DataType.KEYWORD, DataType.KEYWORD), () -> {
56+
long ts_sec = 1657585450L; // 2022-07-12T00:24:10Z
57+
int hours = randomIntBetween(0, 23);
58+
String date = String.format(Locale.ROOT, "12/Jul/2022:%02d:24:10 +0900", hours);
59+
long expected_ts = (ts_sec + (hours - 9) * 3600L) * 1000L;
60+
return new TestCaseSupplier.TestCase(
61+
List.of(
62+
new TestCaseSupplier.TypedData(new BytesRef("dd/MMM/yyyy:HH:mm:ss Z"), DataType.KEYWORD, "first"),
63+
new TestCaseSupplier.TypedData(new BytesRef(date), DataType.KEYWORD, "second")
64+
),
65+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
66+
DataType.DATETIME,
67+
equalTo(expected_ts)
68+
);
69+
}),
5470
new TestCaseSupplier(
5571
"With Text",
5672
List.of(DataType.KEYWORD, DataType.TEXT),
@@ -59,7 +75,7 @@ public static Iterable<Object[]> parameters() {
5975
new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
6076
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
6177
),
62-
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
78+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
6379
DataType.DATETIME,
6480
equalTo(1683244800000L)
6581
)
@@ -72,7 +88,7 @@ public static Iterable<Object[]> parameters() {
7288
new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
7389
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
7490
),
75-
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
91+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
7692
DataType.DATETIME,
7793
equalTo(1683244800000L)
7894
)
@@ -85,7 +101,7 @@ public static Iterable<Object[]> parameters() {
85101
new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
86102
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
87103
),
88-
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
104+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
89105
DataType.DATETIME,
90106
equalTo(1683244800000L)
91107
)
@@ -98,7 +114,7 @@ public static Iterable<Object[]> parameters() {
98114
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
99115

100116
),
101-
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
117+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
102118
DataType.DATETIME,
103119
is(nullValue())
104120
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
@@ -118,7 +134,7 @@ public static Iterable<Object[]> parameters() {
118134
new TestCaseSupplier.TypedData(new BytesRef("not a date"), DataType.KEYWORD, "second")
119135

120136
),
121-
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
137+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
122138
DataType.DATETIME,
123139
is(nullValue())
124140
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")

0 commit comments

Comments
 (0)