Skip to content

Commit 42c13b4

Browse files
yuancuyuancu
authored
Support ISO8601-formatted string in PPL (#4246)
* Support parsing ISO 8601 datetime format for timestamp value Signed-off-by: Yuanchun Shen <[email protected]> * Modify tests for ISO 8601 timestamp input Signed-off-by: Yuanchun Shen <[email protected]> * Add support of iso 8601 date string to date and time - add an IT for date time comparison with iso 8601 formatted literal Signed-off-by: Yuanchun Shen <[email protected]> --------- Signed-off-by: Yuanchun Shen <[email protected]>
1 parent 64a8671 commit 42c13b4

File tree

7 files changed

+90
-24
lines changed

7 files changed

+90
-24
lines changed

‎core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import com.google.common.base.Objects;
99
import java.time.Instant;
1010
import java.time.LocalDate;
11+
import java.time.LocalDateTime;
1112
import java.time.LocalTime;
1213
import java.time.ZoneOffset;
1314
import java.time.ZonedDateTime;
@@ -32,7 +33,14 @@ public class ExprDateValue extends AbstractExprValue {
3233
*/
3334
public ExprDateValue(String date) {
3435
try {
35-
this.date = LocalDate.parse(date, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER);
36+
LocalDateTime ldt;
37+
try {
38+
ldt = LocalDateTime.parse(date, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER);
39+
} catch (DateTimeParseException ignored) {
40+
ZonedDateTime zdt = ZonedDateTime.parse(date, DateTimeFormatter.ISO_DATE_TIME);
41+
ldt = zdt.withZoneSameInstant(ZoneOffset.UTC).toLocalDateTime();
42+
}
43+
this.date = ldt.toLocalDate();
3644
} catch (DateTimeParseException e) {
3745
throw new ExpressionEvaluationException(
3846
String.format("date:%s in unsupported format, please use 'yyyy-MM-dd'", date));

‎core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@
1010
import java.time.Instant;
1111
import java.time.LocalDate;
1212
import java.time.LocalTime;
13+
import java.time.OffsetTime;
1314
import java.time.ZoneOffset;
1415
import java.time.ZonedDateTime;
16+
import java.time.format.DateTimeFormatter;
1517
import java.time.format.DateTimeParseException;
1618
import java.util.Objects;
1719
import lombok.RequiredArgsConstructor;
@@ -34,7 +36,23 @@ public class ExprTimeValue extends AbstractExprValue {
3436
*/
3537
public ExprTimeValue(String time) {
3638
try {
37-
this.time = LocalTime.parse(time, DateTimeFormatters.TIME_TIMESTAMP_FORMATTER);
39+
LocalTime lt;
40+
try {
41+
lt = LocalTime.parse(time, DateTimeFormatters.TIME_TIMESTAMP_FORMATTER);
42+
} catch (DateTimeParseException ignore) {
43+
try {
44+
lt =
45+
ZonedDateTime.parse(time, DateTimeFormatter.ISO_DATE_TIME)
46+
.withZoneSameInstant(ZoneOffset.UTC)
47+
.toLocalTime();
48+
} catch (DateTimeParseException ignore2) {
49+
lt =
50+
OffsetTime.parse(time, DateTimeFormatter.ISO_TIME)
51+
.withOffsetSameInstant(ZoneOffset.UTC)
52+
.toLocalTime();
53+
}
54+
}
55+
this.time = lt;
3856
} catch (DateTimeParseException e) {
3957
throw new ExpressionEvaluationException(
4058
String.format("time:%s in unsupported format, please use 'HH:mm:ss[.SSSSSSSSS]'", time));

‎core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
import java.time.LocalDateTime;
1414
import java.time.LocalTime;
1515
import java.time.ZoneOffset;
16+
import java.time.ZonedDateTime;
17+
import java.time.format.DateTimeFormatter;
1618
import java.time.format.DateTimeParseException;
1719
import java.time.temporal.ChronoUnit;
1820
import java.util.Objects;
@@ -31,17 +33,24 @@ public class ExprTimestampValue extends AbstractExprValue {
3133
/**
3234
* Constructor with timestamp string.
3335
*
34-
* @param timestamp a date or timestamp string (does not accept time string)
36+
* @param timestamp a date or timestamp string (does not accept time string). It accepts both ISO
37+
* 8601 format and {@code yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]} format
3538
*/
3639
public ExprTimestampValue(String timestamp) {
3740
try {
38-
this.timestamp =
39-
LocalDateTime.parse(timestamp, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER)
40-
.toInstant(ZoneOffset.UTC);
41+
LocalDateTime ldt;
42+
try {
43+
ldt = LocalDateTime.parse(timestamp, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER);
44+
} catch (DateTimeParseException ignored) {
45+
ZonedDateTime zdt = ZonedDateTime.parse(timestamp, DateTimeFormatter.ISO_DATE_TIME);
46+
ldt = zdt.withZoneSameInstant(ZoneOffset.UTC).toLocalDateTime();
47+
}
48+
this.timestamp = ldt.toInstant(ZoneOffset.UTC);
4149
} catch (DateTimeParseException e) {
4250
throw new ExpressionEvaluationException(
4351
String.format(
44-
"timestamp:%s in unsupported format, please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'",
52+
"timestamp:%s in unsupported format, please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]' or"
53+
+ " ISO 8601 format",
4554
timestamp));
4655
}
4756
}

‎core/src/main/java/org/opensearch/sql/executor/QueryService.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,11 @@ public RelNode optimize(RelNode plan, CalcitePlanContext context) {
265265

266266
private boolean isCalciteFallbackAllowed() {
267267
if (settings != null) {
268-
return settings.getSettingValue(Settings.Key.CALCITE_FALLBACK_ALLOWED);
268+
Boolean fallback_allowed = settings.getSettingValue(Settings.Key.CALCITE_FALLBACK_ALLOWED);
269+
if (fallback_allowed == null) {
270+
return false;
271+
}
272+
return fallback_allowed;
269273
} else {
270274
return true;
271275
}

‎core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import java.time.LocalTime;
1717
import java.time.ZoneOffset;
1818
import java.time.ZonedDateTime;
19+
import java.time.format.DateTimeFormatter;
1920
import org.junit.jupiter.api.Test;
2021
import org.opensearch.sql.exception.ExpressionEvaluationException;
2122
import org.opensearch.sql.expression.function.FunctionProperties;
@@ -112,15 +113,21 @@ public void timeInUnsupportedFormat() {
112113
}
113114

114115
@Test
115-
public void timestampInUnsupportedFormat() {
116-
Throwable exception =
117-
assertThrows(
118-
ExpressionEvaluationException.class,
119-
() -> new ExprTimestampValue("2020-07-07T01:01:01Z"));
116+
public void timestampInISO8601Format() {
117+
ExprTimestampValue timestampValue = new ExprTimestampValue("2020-07-07T01:01:01Z");
120118
assertEquals(
121-
"timestamp:2020-07-07T01:01:01Z in unsupported format, "
122-
+ "please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'",
123-
exception.getMessage());
119+
LocalDateTime.parse("2020-07-07T01:01:01Z", DateTimeFormatter.ISO_DATE_TIME)
120+
.toInstant(ZoneOffset.UTC),
121+
timestampValue.timestampValue());
122+
}
123+
124+
@Test
125+
public void timestampInISO8601FormatWithTimeZone() {
126+
ExprTimestampValue timestampValue = new ExprTimestampValue("2020-07-07T01:01:01-01:00");
127+
assertEquals(
128+
LocalDateTime.parse("2020-07-07T02:01:01Z", DateTimeFormatter.ISO_DATE_TIME)
129+
.toInstant(ZoneOffset.UTC),
130+
timestampValue.timestampValue());
124131
}
125132

126133
@Test
@@ -134,13 +141,11 @@ public void stringTimestampValue() {
134141
assertEquals(LocalTime.parse("19:44:00"), stringValue.timeValue());
135142
assertEquals("\"2020-08-17 19:44:00\"", stringValue.toString());
136143

137-
Throwable exception =
138-
assertThrows(
139-
ExpressionEvaluationException.class,
140-
() -> new ExprStringValue("2020-07-07T01:01:01Z").timestampValue());
144+
ExprValue stringValueWithIsoTimestamp = new ExprStringValue("2020-07-07T01:01:01Z");
141145
assertEquals(
142-
"date:2020-07-07T01:01:01Z in unsupported format, " + "please use 'yyyy-MM-dd'",
143-
exception.getMessage());
146+
LocalDateTime.parse("2020-07-07T01:01:01Z", DateTimeFormatter.ISO_DATE_TIME)
147+
.toInstant(ZoneOffset.UTC),
148+
stringValueWithIsoTimestamp.timestampValue());
144149
}
145150

146151
@Test
@@ -221,7 +226,7 @@ public void timestampOverMaxNanoPrecision() {
221226
() -> new ExprTimestampValue("2020-07-07 01:01:01.1234567890"));
222227
assertEquals(
223228
"timestamp:2020-07-07 01:01:01.1234567890 in unsupported format, please use "
224-
+ "'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'",
229+
+ "'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]' or ISO 8601 format",
225230
exception.getMessage());
226231
}
227232

‎core/src/test/java/org/opensearch/sql/expression/datetime/TimestampTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,8 @@ public void timestamp_one_arg_string_invalid_format(String value, String testNam
6464
() -> DSL.timestamp(functionProperties, DSL.literal(value)).valueOf());
6565
assertEquals(
6666
String.format(
67-
"timestamp:%s in unsupported format, please " + "use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'",
67+
"timestamp:%s in unsupported format, please "
68+
+ "use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]' or ISO 8601 format",
6869
value),
6970
exception.getMessage());
7071
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -707,6 +707,27 @@ public void testComparisonBetweenDateAndTimestamp() throws IOException {
707707
verifyDataRows(actual, rows(2));
708708
}
709709

710+
@Test
711+
public void testComparisonWithIso8601DateLiteral() {
712+
Object[][] tests = {
713+
{"date_optional_time = '1984-04-12T09:07:42.000Z'", 2},
714+
{"date_time = '1984-04-12T07:07:42-02:00'", 2},
715+
{"date = '1984-04-12'", 2},
716+
{"date = '1984-04-12T09:07:42.000Z'", 2},
717+
{"basic_t_time = '1984-04-12T09:07:42.000Z'", 2},
718+
{"basic_t_time = '10:07:42.000+01:00'", 2}
719+
};
720+
for (Object[] pair : tests) {
721+
String query = (String) pair[0];
722+
int result = (int) pair[1];
723+
JSONObject actual =
724+
executeQuery(
725+
String.format(
726+
"source=%s | where %s | stats COUNT() AS cnt", TEST_INDEX_DATE_FORMATS, query));
727+
verifyDataRows(actual, rows(result));
728+
}
729+
}
730+
710731
@Test
711732
public void testAddSubTime() throws IOException {
712733
JSONObject actual =

0 commit comments

Comments
 (0)