Skip to content

Commit 4007ff3

Browse files
bpinteabpintea
authored
ESQL: allow DATE_PARSE to read the timezones (#118603)
This just removes fixing a formatter to a timezone (UTC), allowing `DATE_PARSE` to correctly read timezones. Fixes #117680.
1 parent a2e035e commit 4007ff3

File tree

6 files changed

+100
-29
lines changed

6 files changed

+100
-29
lines changed

‎docs/changelog/118603.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 118603
2+
summary: Allow DATE_PARSE to read the timezones
3+
area: ES|QL
4+
type: bug
5+
issues:
6+
- 117680

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -494,6 +494,60 @@ b:datetime
494494
null
495495
;
496496

497+
evalDateParseWithTimezone
498+
required_capability: date_parse_tz
499+
row s = "12/Jul/2022:10:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s);
500+
501+
s:keyword | d:datetime
502+
12/Jul/2022:10:24:10 +0900 | 2022-07-12T01:24:10.000Z
503+
;
504+
505+
evalDateParseWithTimezoneCrossingDayBoundary
506+
required_capability: date_parse_tz
507+
row s = "12/Jul/2022:08:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s);
508+
509+
s:keyword | d:datetime
510+
12/Jul/2022:08:24:10 +0900 | 2022-07-11T23:24:10.000Z
511+
;
512+
513+
evalDateParseWithTimezone2
514+
required_capability: date_parse_tz
515+
row s1 = "12/Jul/2022:10:24:10 +0900", s2 = "2022/12/07 09:24:10 +0800"
516+
| eval d1 = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s1), d2 = date_parse("yyyy/dd/MM HH:mm:ss Z", s2)
517+
| eval eq = d1 == d2
518+
| keep d1, eq
519+
;
520+
521+
d1:datetime | eq:boolean
522+
2022-07-12T01:24:10.000Z | true
523+
;
524+
525+
evalDateParseWithAndWithoutTimezone
526+
required_capability: date_parse_tz
527+
row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss"
528+
| eval no_tz = date_parse(format, s)
529+
| eval with_tz = date_parse(concat(format, " Z"), concat(s, " +0900"))
530+
| keep s, no_tz, with_tz
531+
;
532+
533+
s:keyword | no_tz:datetime | with_tz:datetime
534+
2022/12/07 09:24:10 | 2022-07-12T09:24:10.000Z | 2022-07-12T00:24:10.000Z
535+
;
536+
537+
evalDateParseWithOtherTimezoneSpecifiers
538+
required_capability: date_parse_tz
539+
row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss"
540+
| eval with_tz1 = date_parse(concat(format, " Z"), concat(s, " +0900"))
541+
| eval with_tz2 = date_parse(concat(format, " x"), concat(s, " +09"))
542+
| eval with_tz3 = date_parse(concat(format, " X"), concat(s, " +0900"))
543+
| eval with_tz4 = date_parse(concat(format, " O"), concat(s, " GMT+9"))
544+
| keep s, with_tz*
545+
;
546+
547+
s:keyword | with_tz1:datetime | with_tz2:datetime | with_tz3:datetime | with_tz4:datetime
548+
2022/12/07 09:24:10 | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z
549+
;
550+
497551
evalDateParseDynamic
498552
from employees | where emp_no == 10039 or emp_no == 10040 | sort emp_no
499553
| eval birth_date_string = date_format("yyyy-MM-dd", birth_date)

‎x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java‎

Lines changed: 7 additions & 14 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -380,6 +380,11 @@ public enum Cap {
380380
*/
381381
DATE_NANOS_AGGREGATIONS(),
382382

383+
/**
384+
* DATE_PARSE supports reading timezones
385+
*/
386+
DATE_PARSE_TZ(),
387+
383388
/**
384389
* Support for datetime in least and greatest functions
385390
*/

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,12 @@
2828
import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
2929

3030
import java.io.IOException;
31-
import java.time.ZoneId;
3231
import java.util.List;
3332

3433
import static org.elasticsearch.common.time.DateFormatter.forPattern;
3534
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
3635
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
3736
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
38-
import static org.elasticsearch.xpack.esql.core.util.DateUtils.UTC;
3937
import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact;
4038
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
4139
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong;
@@ -130,13 +128,12 @@ public static long process(BytesRef val, @Fixed DateFormatter formatter) throws
130128
}
131129

132130
@Evaluator(warnExceptions = { IllegalArgumentException.class })
133-
static long process(BytesRef val, BytesRef formatter, @Fixed ZoneId zoneId) throws IllegalArgumentException {
134-
return dateTimeToLong(val.utf8ToString(), toFormatter(formatter, zoneId));
131+
static long process(BytesRef val, BytesRef formatter) throws IllegalArgumentException {
132+
return dateTimeToLong(val.utf8ToString(), toFormatter(formatter));
135133
}
136134

137135
@Override
138136
public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
139-
ZoneId zone = UTC; // TODO session timezone?
140137
ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(field);
141138
if (format == null) {
142139
return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, DEFAULT_DATE_TIME_FORMATTER);
@@ -146,18 +143,18 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
146143
}
147144
if (format.foldable()) {
148145
try {
149-
DateFormatter formatter = toFormatter(format.fold(), zone);
146+
DateFormatter formatter = toFormatter(format.fold());
150147
return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, formatter);
151148
} catch (IllegalArgumentException e) {
152149
throw new InvalidArgumentException(e, "invalid date pattern for [{}]: {}", sourceText(), e.getMessage());
153150
}
154151
}
155152
ExpressionEvaluator.Factory formatEvaluator = toEvaluator.apply(format);
156-
return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator, zone);
153+
return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator);
157154
}
158155

159-
private static DateFormatter toFormatter(Object format, ZoneId zone) {
160-
return forPattern(((BytesRef) format).utf8ToString()).withZone(zone);
156+
private static DateFormatter toFormatter(Object format) {
157+
return forPattern(((BytesRef) format).utf8ToString());
161158
}
162159

163160
@Override

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
2222

2323
import java.util.List;
24+
import java.util.Locale;
2425
import java.util.function.Supplier;
2526

2627
import static org.hamcrest.Matchers.equalTo;
@@ -46,11 +47,26 @@ public static Iterable<Object[]> parameters() {
4647
new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
4748
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
4849
),
49-
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
50+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
5051
DataType.DATETIME,
5152
equalTo(1683244800000L)
5253
)
5354
),
55+
new TestCaseSupplier("Timezoned Case", List.of(DataType.KEYWORD, DataType.KEYWORD), () -> {
56+
long ts_sec = 1657585450L; // 2022-07-12T00:24:10Z
57+
int hours = randomIntBetween(0, 23);
58+
String date = String.format(Locale.ROOT, "12/Jul/2022:%02d:24:10 +0900", hours);
59+
long expected_ts = (ts_sec + (hours - 9) * 3600L) * 1000L;
60+
return new TestCaseSupplier.TestCase(
61+
List.of(
62+
new TestCaseSupplier.TypedData(new BytesRef("dd/MMM/yyyy:HH:mm:ss Z"), DataType.KEYWORD, "first"),
63+
new TestCaseSupplier.TypedData(new BytesRef(date), DataType.KEYWORD, "second")
64+
),
65+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
66+
DataType.DATETIME,
67+
equalTo(expected_ts)
68+
);
69+
}),
5470
new TestCaseSupplier(
5571
"With Text",
5672
List.of(DataType.KEYWORD, DataType.TEXT),
@@ -59,7 +75,7 @@ public static Iterable<Object[]> parameters() {
5975
new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
6076
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
6177
),
62-
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
78+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
6379
DataType.DATETIME,
6480
equalTo(1683244800000L)
6581
)
@@ -72,7 +88,7 @@ public static Iterable<Object[]> parameters() {
7288
new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
7389
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
7490
),
75-
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
91+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
7692
DataType.DATETIME,
7793
equalTo(1683244800000L)
7894
)
@@ -85,7 +101,7 @@ public static Iterable<Object[]> parameters() {
85101
new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
86102
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
87103
),
88-
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
104+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
89105
DataType.DATETIME,
90106
equalTo(1683244800000L)
91107
)
@@ -98,7 +114,7 @@ public static Iterable<Object[]> parameters() {
98114
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
99115

100116
),
101-
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
117+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
102118
DataType.DATETIME,
103119
is(nullValue())
104120
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
@@ -118,7 +134,7 @@ public static Iterable<Object[]> parameters() {
118134
new TestCaseSupplier.TypedData(new BytesRef("not a date"), DataType.KEYWORD, "second")
119135

120136
),
121-
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
137+
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
122138
DataType.DATETIME,
123139
is(nullValue())
124140
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")

0 commit comments

Comments
 (0)