Support millisecond span (opensearch-project#4672)

* Support millisecond span

Signed-off-by: Yuanchun Shen <[email protected]>

* Update per funciton tests

Signed-off-by: Yuanchun Shen <[email protected]>

---------

Signed-off-by: Yuanchun Shen <[email protected]>

Author: yuancu

core/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java

@@ -14,8 +14,8 @@
 @RequiredArgsConstructor
 public enum IntervalUnit {
   UNKNOWN,
-
   MICROSECOND,
+  MILLISECOND,
   SECOND,
   MINUTE,
   HOUR,

core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java

@@ -15,6 +15,8 @@
 public enum SpanUnit {
   UNKNOWN("unknown"),
   NONE(""),
+  MICROSECOND("us"),
+  US("us"),
   MILLISECOND("ms"),
   MS("ms"),
   SECONDS("s"),

core/src/main/java/org/opensearch/sql/ast/tree/Timechart.java

@@ -10,7 +10,7 @@
 import static org.opensearch.sql.ast.dsl.AstDSL.eval;
 import static org.opensearch.sql.ast.dsl.AstDSL.function;
 import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral;
-import static org.opensearch.sql.ast.expression.IntervalUnit.SECOND;
+import static org.opensearch.sql.ast.expression.IntervalUnit.MILLISECOND;
 import static org.opensearch.sql.ast.tree.Timechart.PerFunctionRateExprBuilder.sum;
 import static org.opensearch.sql.ast.tree.Timechart.PerFunctionRateExprBuilder.timestampadd;
 import static org.opensearch.sql.ast.tree.Timechart.PerFunctionRateExprBuilder.timestampdiff;
@@ -112,11 +112,13 @@ private UnresolvedPlan transformPerFunction() {
     Span span = (Span) this.binExpression;
     Field spanStartTime = AstDSL.field(IMPLICIT_FIELD_TIMESTAMP);
     Function spanEndTime = timestampadd(span.getUnit(), span.getValue(), spanStartTime);
-    Function spanSeconds = timestampdiff(SECOND, spanStartTime, spanEndTime);
-
+    Function spanMillis = timestampdiff(MILLISECOND, spanStartTime, spanEndTime);
+    final int SECOND_IN_MILLISECOND = 1000;
     return eval(
         timechart(AstDSL.alias(perFunc.aggName, sum(perFunc.aggArg))),
-        let(perFunc.aggName).multiply(perFunc.seconds).dividedBy(spanSeconds));
+        let(perFunc.aggName)
+            .multiply(perFunc.seconds * SECOND_IN_MILLISECOND)
+            .dividedBy(spanMillis));
   }
 
   private Timechart timechart(UnresolvedExpression newAggregateFunction) {

core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java

@@ -54,38 +54,59 @@ public RelDataType commonType(RexNode... nodes) {
   public SqlIntervalQualifier createIntervalUntil(SpanUnit unit) {
     TimeUnit timeUnit;
     switch (unit) {
+      case MICROSECOND:
+      case US:
+        timeUnit = TimeUnit.MICROSECOND;
+        break;
       case MILLISECOND:
       case MS:
         timeUnit = TimeUnit.MILLISECOND;
         break;
+      case SECONDS:
       case SECOND:
+      case SECS:
+      case SEC:
       case S:
         timeUnit = TimeUnit.SECOND;
         break;
+      case MINUTES:
       case MINUTE:
+      case MINS:
+      case MIN:
       case m:
         timeUnit = TimeUnit.MINUTE;
         break;
+      case HOURS:
       case HOUR:
+      case HRS:
+      case HR:
       case H:
         timeUnit = TimeUnit.HOUR;
         break;
+      case DAYS:
       case DAY:
       case D:
         timeUnit = TimeUnit.DAY;
         break;
+      case WEEKS:
       case WEEK:
       case W:
         timeUnit = TimeUnit.WEEK;
         break;
+      case MONTHS:
       case MONTH:
+      case MON:
       case M:
         timeUnit = TimeUnit.MONTH;
         break;
+      case QUARTERS:
       case QUARTER:
+      case QTRS:
+      case QTR:
       case Q:
         timeUnit = TimeUnit.QUARTER;
         break;
+      case YEARS:
       case YEAR:
       case Y:
         timeUnit = TimeUnit.YEAR;

core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java

@@ -68,7 +68,8 @@ public interface PlanUtils {
 
   static SpanUnit intervalUnitToSpanUnit(IntervalUnit unit) {
     return switch (unit) {
-      case MICROSECOND -> SpanUnit.MILLISECOND;
+      case MICROSECOND -> SpanUnit.MICROSECOND;
+      case MILLISECOND -> SpanUnit.MILLISECOND;
       case SECOND -> SpanUnit.SECOND;
       case MINUTE -> SpanUnit.MINUTE;
       case HOUR -> SpanUnit.HOUR;
@@ -84,9 +85,12 @@ static SpanUnit intervalUnitToSpanUnit(IntervalUnit unit) {
 
   static IntervalUnit spanUnitToIntervalUnit(SpanUnit unit) {
     switch (unit) {
+      case MICROSECOND:
+      case US:
+        return IntervalUnit.MICROSECOND;
       case MILLISECOND:
       case MS:
-        return IntervalUnit.MICROSECOND;
+        return IntervalUnit.MILLISECOND;
       case SECOND:
       case SECONDS:
       case SEC:

core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java

@@ -8,6 +8,7 @@
 import static java.time.temporal.ChronoUnit.DAYS;
 import static java.time.temporal.ChronoUnit.HOURS;
 import static java.time.temporal.ChronoUnit.MICROS;
+import static java.time.temporal.ChronoUnit.MILLIS;
 import static java.time.temporal.ChronoUnit.MINUTES;
 import static java.time.temporal.ChronoUnit.MONTHS;
 import static java.time.temporal.ChronoUnit.SECONDS;
@@ -1896,6 +1897,9 @@ public static ExprValue exprTimestampAdd(
       case "MICROSECOND":
         temporalUnit = MICROS;
         break;
+      case "MILLISECOND":
+        temporalUnit = MILLIS;
+        break;
       case "SECOND":
         temporalUnit = SECONDS;
         break;
@@ -1935,10 +1939,13 @@ public static ExprValue exprTimestampAddForTimeType(
 
   private ExprValue getTimeDifference(String part, LocalDateTime startTime, LocalDateTime endTime) {
     long returnVal;
-    switch (part) {
+    switch (part.toUpperCase(Locale.ROOT)) {
       case "MICROSECOND":
         returnVal = MICROS.between(startTime, endTime);
         break;
+      case "MILLISECOND":
+        returnVal = MILLIS.between(startTime, endTime);
+        break;
       case "SECOND":
         returnVal = SECONDS.between(startTime, endTime);
         break;

core/src/test/java/org/opensearch/sql/ast/tree/TimechartTest.java

@@ -54,9 +54,9 @@ void should_transform_per_second_for_different_spans(
                 let(
                     "per_second(bytes)",
                     divide(
-                        multiply("per_second(bytes)", 1.0),
+                        multiply("per_second(bytes)", 1000.0),
                         timestampdiff(
-                            "SECOND",
+                            "MILLISECOND",
                             "@timestamp",
                             timestampadd(expectedIntervalUnit, spanValue, "@timestamp")))),
                 timechart(span(spanValue, spanUnit), alias("per_second(bytes)", sum("bytes")))));
@@ -73,9 +73,9 @@ void should_transform_per_minute_for_different_spans(
                 let(
                     "per_minute(bytes)",
                     divide(
-                        multiply("per_minute(bytes)", 60.0),
+                        multiply("per_minute(bytes)", 60000.0),
                         timestampdiff(
-                            "SECOND",
+                            "MILLISECOND",
                             "@timestamp",
                             timestampadd(expectedIntervalUnit, spanValue, "@timestamp")))),
                 timechart(span(spanValue, spanUnit), alias("per_minute(bytes)", sum("bytes")))));
@@ -92,9 +92,9 @@ void should_transform_per_hour_for_different_spans(
                 let(
                     "per_hour(bytes)",
                     divide(
-                        multiply("per_hour(bytes)", 3600.0),
+                        multiply("per_hour(bytes)", 3600000.0),
                         timestampdiff(
-                            "SECOND",
+                            "MILLISECOND",
                             "@timestamp",
                             timestampadd(expectedIntervalUnit, spanValue, "@timestamp")))),
                 timechart(span(spanValue, spanUnit), alias("per_hour(bytes)", sum("bytes")))));
@@ -111,9 +111,9 @@ void should_transform_per_day_for_different_spans(
                 let(
                     "per_day(bytes)",
                     divide(
-                        multiply("per_day(bytes)", 86400.0),
+                        multiply("per_day(bytes)", 8.64E7),
                         timestampdiff(
-                            "SECOND",
+                            "MILLISECOND",
                             "@timestamp",
                             timestampadd(expectedIntervalUnit, spanValue, "@timestamp")))),
                 timechart(span(spanValue, spanUnit), alias("per_day(bytes)", sum("bytes")))));
@@ -149,9 +149,9 @@ void should_preserve_all_fields_during_per_function_transformation() {
                 let(
                     "per_second(bytes)",
                     divide(
-                        multiply("per_second(bytes)", 1.0),
+                        multiply("per_second(bytes)", 1000.0),
                         timestampdiff(
-                            "SECOND", "@timestamp", timestampadd("MINUTE", 5, "@timestamp")))),
+                            "MILLISECOND", "@timestamp", timestampadd("MINUTE", 5, "@timestamp")))),
                 expected));
   }
 

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java

@@ -444,8 +444,8 @@ public void testExplainTimechartPerSecond() throws IOException {
     var result = explainQueryToString("source=events | timechart span=2m per_second(cpu_usage)");
     assertTrue(
         result.contains(
-            "per_second(cpu_usage)=[DIVIDE(*($1, 1.0E0), "
-                + "TIMESTAMPDIFF('SECOND':VARCHAR, $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]"));
+            "per_second(cpu_usage)=[DIVIDE(*($1, 1000.0E0), TIMESTAMPDIFF('MILLISECOND':VARCHAR,"
+                + " $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]"));
     assertTrue(result.contains("per_second(cpu_usage)=[SUM($0)]"));
   }
 
@@ -454,8 +454,8 @@ public void testExplainTimechartPerMinute() throws IOException {
     var result = explainQueryToString("source=events | timechart span=2m per_minute(cpu_usage)");
     assertTrue(
         result.contains(
-            "per_minute(cpu_usage)=[DIVIDE(*($1, 60.0E0), "
-                + "TIMESTAMPDIFF('SECOND':VARCHAR, $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]"));
+            "per_minute(cpu_usage)=[DIVIDE(*($1, 60000.0E0), TIMESTAMPDIFF('MILLISECOND':VARCHAR,"
+                + " $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]"));
     assertTrue(result.contains("per_minute(cpu_usage)=[SUM($0)]"));
   }
 
@@ -464,8 +464,8 @@ public void testExplainTimechartPerHour() throws IOException {
     var result = explainQueryToString("source=events | timechart span=2m per_hour(cpu_usage)");
     assertTrue(
         result.contains(
-            "per_hour(cpu_usage)=[DIVIDE(*($1, 3600.0E0), "
-                + "TIMESTAMPDIFF('SECOND':VARCHAR, $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]"));
+            "per_hour(cpu_usage)=[DIVIDE(*($1, 3600000.0E0), TIMESTAMPDIFF('MILLISECOND':VARCHAR,"
+                + " $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]"));
     assertTrue(result.contains("per_hour(cpu_usage)=[SUM($0)]"));
   }
 
@@ -474,8 +474,8 @@ public void testExplainTimechartPerDay() throws IOException {
     var result = explainQueryToString("source=events | timechart span=2m per_day(cpu_usage)");
     assertTrue(
         result.contains(
-            "per_day(cpu_usage)=[DIVIDE(*($1, 86400.0E0), "
-                + "TIMESTAMPDIFF('SECOND':VARCHAR, $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]"));
+            "per_day(cpu_usage)=[DIVIDE(*($1, 8.64E7), TIMESTAMPDIFF('MILLISECOND':VARCHAR, $0,"
+                + " TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]"));
     assertTrue(result.contains("per_day(cpu_usage)=[SUM($0)]"));
   }
 

integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4550.yml

@@ -0,0 +1,93 @@
+setup:
+  - do:
+      indices.create:
+        index: test_data_2023
+        body:
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              "packets":
+                type: integer
+  - do:
+      bulk:
+        index: test_data_2023
+        refresh: true
+        body:
+          - '{"index":{}}'
+          - '{"@timestamp":"2023-10-08T10:00:00.000Z","packets":10}'
+          - '{"index":{}}'
+          - '{"@timestamp":"2023-10-08T10:00:00.500Z","packets":15}'
+          - '{"index":{}}'
+          - '{"@timestamp":"2023-10-08T10:00:01.000Z","packets":20}'
+          - '{"index":{}}'
+          - '{"@timestamp":"2023-10-08T10:00:01.500Z","packets":25}'
+          - '{"index":{}}'
+          - '{"@timestamp":"2023-10-08T10:00:02.000Z","packets":30}'
+
+---
+"timechart with millisecond span":
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: source=test_data_2023 | timechart span=500ms count()
+
+  - match: { total: 5 }
+  - match: { "schema": [ { "name": "@timestamp", "type": "timestamp" }, { "name": "count", "type": "bigint" }] }
+  - match: {"datarows": [["2023-10-08 10:00:00", 1], ["2023-10-08 10:00:00.5", 1], ["2023-10-08 10:00:01", 1], ["2023-10-08 10:00:01.5", 1], ["2023-10-08 10:00:02", 1]]}
+
+---
+"timechart with millisecond span and per_second function":
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: source=test_data_2023 | timechart span=1000ms per_second(packets)
+
+  - match: { total: 3 }
+  - match: { "schema": [ { "name": "@timestamp", "type": "timestamp" }, { "name": "per_second(packets)", "type": "double" }] }
+  - match: {"datarows": [["2023-10-08 10:00:00", 25.0], ["2023-10-08 10:00:01", 45.0], ["2023-10-08 10:00:02", 30.0]]}
+
+---
+"timechart with milliseconds":
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: source=test_data_2023 | timechart span=250milliseconds count()
+
+  - match: { total: 5 }
+  - match: { "schema": [ { "name": "@timestamp", "type": "timestamp" }, { "name": "count", "type": "bigint" }] }
+
+---
+"timechart with second span for comparison":
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: source=test_data_2023 | timechart span=1s count()
+
+  - match: { total: 3 }
+  - match: { "schema": [ { "name": "@timestamp", "type": "timestamp" }, { "name": "count", "type": "bigint" }] }
+  - match: {"datarows": [["2023-10-08 10:00:00", 2], ["2023-10-08 10:00:01", 2], ["2023-10-08 10:00:02", 1]]}

ppl/src/main/antlr/OpenSearchPPLLexer.g4

@@ -169,6 +169,7 @@ HOUR_MINUTE:                        'HOUR_MINUTE';
 HOUR_OF_DAY:                        'HOUR_OF_DAY';
 HOUR_SECOND:                        'HOUR_SECOND';
 INTERVAL:                           'INTERVAL';
+MILLISECOND:                        'MILLISECOND';
 MICROSECOND:                        'MICROSECOND';
 MINUTE:                             'MINUTE';
 MINUTE_MICROSECOND:                 'MINUTE_MICROSECOND';
@@ -502,7 +503,8 @@ ALIGNTIME:                          'ALIGNTIME';
 PERCENTILE_SHORTCUT:                PERC(INTEGER_LITERAL | DECIMAL_LITERAL) | 'P'(INTEGER_LITERAL | DECIMAL_LITERAL);
 
 SPANLENGTH: [0-9]+ (
-    'US'|'MS'|'CS'|'DS'
+    'US' |'CS'|'DS'
+    |'MS'|'MILLISECOND'|'MILLISECONDS'
     |'S'|'SEC'|'SECS'|'SECOND'|'SECONDS'
     |'MIN'|'MINS'|'MINUTE'|'MINUTES'
     |'H'|'HR'|'HRS'|'HOUR'|'HOURS'