Skip to content

Commit ec5d429

Browse files
yuancuyuancu
authored
Support ISO8601-formatted string in PPL (opensearch-project#4246) (opensearch-project#4685)
* Support parsing ISO 8601 datetime format for timestamp value * Modify tests for ISO 8601 timestamp input * Add support of iso 8601 date string to date and time - add an IT for date time comparison with iso 8601 formatted literal --------- (cherry picked from commit 42c13b4) Signed-off-by: Yuanchun Shen <[email protected]>
1 parent 8a0e9c2 commit ec5d429

File tree

6 files changed

+87
-24
lines changed

6 files changed

+87
-24
lines changed

‎core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import java.time.LocalDate;
1313
import java.time.LocalDateTime;
1414
import java.time.LocalTime;
15+
import java.time.ZoneOffset;
1516
import java.time.ZonedDateTime;
1617
import java.time.format.DateTimeFormatter;
1718
import java.time.format.DateTimeParseException;
@@ -34,7 +35,14 @@ public class ExprDateValue extends AbstractExprValue {
3435
*/
3536
public ExprDateValue(String date) {
3637
try {
37-
this.date = LocalDate.parse(date, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER);
38+
LocalDateTime ldt;
39+
try {
40+
ldt = LocalDateTime.parse(date, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER);
41+
} catch (DateTimeParseException ignored) {
42+
ZonedDateTime zdt = ZonedDateTime.parse(date, DateTimeFormatter.ISO_DATE_TIME);
43+
ldt = zdt.withZoneSameInstant(ZoneOffset.UTC).toLocalDateTime();
44+
}
45+
this.date = ldt.toLocalDate();
3846
} catch (DateTimeParseException e) {
3947
throw new ExpressionEvaluationException(
4048
String.format("date:%s in unsupported format, please use 'yyyy-MM-dd'", date));

‎core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,10 @@
1212
import java.time.LocalDate;
1313
import java.time.LocalDateTime;
1414
import java.time.LocalTime;
15+
import java.time.OffsetTime;
16+
import java.time.ZoneOffset;
1517
import java.time.ZonedDateTime;
18+
import java.time.format.DateTimeFormatter;
1619
import java.time.format.DateTimeParseException;
1720
import java.util.Objects;
1821
import lombok.RequiredArgsConstructor;
@@ -35,7 +38,23 @@ public class ExprTimeValue extends AbstractExprValue {
3538
*/
3639
public ExprTimeValue(String time) {
3740
try {
38-
this.time = LocalTime.parse(time, DateTimeFormatters.TIME_TIMESTAMP_FORMATTER);
41+
LocalTime lt;
42+
try {
43+
lt = LocalTime.parse(time, DateTimeFormatters.TIME_TIMESTAMP_FORMATTER);
44+
} catch (DateTimeParseException ignore) {
45+
try {
46+
lt =
47+
ZonedDateTime.parse(time, DateTimeFormatter.ISO_DATE_TIME)
48+
.withZoneSameInstant(ZoneOffset.UTC)
49+
.toLocalTime();
50+
} catch (DateTimeParseException ignore2) {
51+
lt =
52+
OffsetTime.parse(time, DateTimeFormatter.ISO_TIME)
53+
.withOffsetSameInstant(ZoneOffset.UTC)
54+
.toLocalTime();
55+
}
56+
}
57+
this.time = lt;
3958
} catch (DateTimeParseException e) {
4059
throw new ExpressionEvaluationException(
4160
String.format("time:%s in unsupported format, please use 'HH:mm:ss[.SSSSSSSSS]'", time));

‎core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
import java.time.LocalDateTime;
1515
import java.time.LocalTime;
1616
import java.time.ZoneOffset;
17+
import java.time.ZonedDateTime;
18+
import java.time.format.DateTimeFormatter;
1719
import java.time.format.DateTimeParseException;
1820
import java.time.temporal.ChronoUnit;
1921
import java.util.Objects;
@@ -32,17 +34,24 @@ public class ExprTimestampValue extends AbstractExprValue {
3234
/**
3335
* Constructor with timestamp string.
3436
*
35-
* @param timestamp a date or timestamp string (does not accept time string)
37+
* @param timestamp a date or timestamp string (does not accept time string). It accepts both ISO
38+
* 8601 format and {@code yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]} format
3639
*/
3740
public ExprTimestampValue(String timestamp) {
3841
try {
39-
this.timestamp =
40-
LocalDateTime.parse(timestamp, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER)
41-
.toInstant(ZoneOffset.UTC);
42+
LocalDateTime ldt;
43+
try {
44+
ldt = LocalDateTime.parse(timestamp, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER);
45+
} catch (DateTimeParseException ignored) {
46+
ZonedDateTime zdt = ZonedDateTime.parse(timestamp, DateTimeFormatter.ISO_DATE_TIME);
47+
ldt = zdt.withZoneSameInstant(ZoneOffset.UTC).toLocalDateTime();
48+
}
49+
this.timestamp = ldt.toInstant(ZoneOffset.UTC);
4250
} catch (DateTimeParseException e) {
4351
throw new ExpressionEvaluationException(
4452
String.format(
45-
"timestamp:%s in unsupported format, please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'",
53+
"timestamp:%s in unsupported format, please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]' or"
54+
+ " ISO 8601 format",
4655
timestamp));
4756
}
4857
}

‎core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java‎

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@
1515
import java.time.LocalDate;
1616
import java.time.LocalDateTime;
1717
import java.time.LocalTime;
18+
import java.time.ZoneOffset;
1819
import java.time.ZonedDateTime;
20+
import java.time.format.DateTimeFormatter;
1921
import org.junit.jupiter.api.Test;
2022
import org.opensearch.sql.exception.ExpressionEvaluationException;
2123
import org.opensearch.sql.exception.SemanticCheckException;
@@ -126,15 +128,21 @@ public void timeInUnsupportedFormat() {
126128
}
127129

128130
@Test
129-
public void timestampInUnsupportedFormat() {
130-
Throwable exception =
131-
assertThrows(
132-
ExpressionEvaluationException.class,
133-
() -> new ExprTimestampValue("2020-07-07T01:01:01Z"));
131+
public void timestampInISO8601Format() {
132+
ExprTimestampValue timestampValue = new ExprTimestampValue("2020-07-07T01:01:01Z");
134133
assertEquals(
135-
"timestamp:2020-07-07T01:01:01Z in unsupported format, "
136-
+ "please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'",
137-
exception.getMessage());
134+
LocalDateTime.parse("2020-07-07T01:01:01Z", DateTimeFormatter.ISO_DATE_TIME)
135+
.toInstant(ZoneOffset.UTC),
136+
timestampValue.timestampValue());
137+
}
138+
139+
@Test
140+
public void timestampInISO8601FormatWithTimeZone() {
141+
ExprTimestampValue timestampValue = new ExprTimestampValue("2020-07-07T01:01:01-01:00");
142+
assertEquals(
143+
LocalDateTime.parse("2020-07-07T02:01:01Z", DateTimeFormatter.ISO_DATE_TIME)
144+
.toInstant(ZoneOffset.UTC),
145+
timestampValue.timestampValue());
138146
}
139147

140148
@Test
@@ -157,14 +165,11 @@ public void stringDateTimeValue() {
157165
assertEquals(LocalTime.parse("19:44:00"), stringValue.timeValue());
158166
assertEquals("\"2020-08-17 19:44:00\"", stringValue.toString());
159167

160-
Throwable exception =
161-
assertThrows(
162-
ExpressionEvaluationException.class,
163-
() -> new ExprStringValue("2020-07-07T01:01:01Z").datetimeValue());
168+
ExprValue stringValueWithIsoTimestamp = new ExprStringValue("2020-07-07T01:01:01Z");
164169
assertEquals(
165-
"datetime:2020-07-07T01:01:01Z in unsupported format, "
166-
+ "please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'",
167-
exception.getMessage());
170+
LocalDateTime.parse("2020-07-07T01:01:01Z", DateTimeFormatter.ISO_DATE_TIME)
171+
.toInstant(ZoneOffset.UTC),
172+
stringValueWithIsoTimestamp.timestampValue());
168173
}
169174

170175
@Test
@@ -263,7 +268,7 @@ public void timestampOverMaxNanoPrecision() {
263268
() -> new ExprTimestampValue("2020-07-07 01:01:01.1234567890"));
264269
assertEquals(
265270
"timestamp:2020-07-07 01:01:01.1234567890 in unsupported format, please use "
266-
+ "'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'",
271+
+ "'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]' or ISO 8601 format",
267272
exception.getMessage());
268273
}
269274

‎core/src/test/java/org/opensearch/sql/expression/datetime/TimestampTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ public void timestamp_one_arg_string_invalid_format(String value, String testNam
6262
() -> DSL.timestamp(functionProperties, DSL.literal(value)).valueOf());
6363
assertEquals(
6464
String.format(
65-
"timestamp:%s in unsupported format, please " + "use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'",
65+
"timestamp:%s in unsupported format, please "
66+
+ "use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]' or ISO 8601 format",
6667
value),
6768
exception.getMessage());
6869
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -708,6 +708,27 @@ public void testComparisonBetweenDateAndTimestamp() throws IOException {
708708
verifyDataRows(actual, rows(2));
709709
}
710710

711+
@Test
712+
public void testComparisonWithIso8601DateLiteral() {
713+
Object[][] tests = {
714+
{"date_optional_time = '1984-04-12T09:07:42.000Z'", 2},
715+
{"date_time = '1984-04-12T07:07:42-02:00'", 2},
716+
{"date = '1984-04-12'", 2},
717+
{"date = '1984-04-12T09:07:42.000Z'", 2},
718+
{"basic_t_time = '1984-04-12T09:07:42.000Z'", 2},
719+
{"basic_t_time = '10:07:42.000+01:00'", 2}
720+
};
721+
for (Object[] pair : tests) {
722+
String query = (String) pair[0];
723+
int result = (int) pair[1];
724+
JSONObject actual =
725+
executeQuery(
726+
String.format(
727+
"source=%s | where %s | stats COUNT() AS cnt", TEST_INDEX_DATE_FORMATS, query));
728+
verifyDataRows(actual, rows(result));
729+
}
730+
}
731+
711732
@Test
712733
public void testAddSubTime() throws IOException {
713734
JSONObject actual =

0 commit comments

Comments
 (0)