Merge remote-tracking branch 'origin/main' into fix-divide

Author: yuancu

core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java

@@ -1332,7 +1332,7 @@ public static ExprValue exprDateDiff(
    * @return ExprValue of date type.
    */
   public static ExprValue exprDateTime(ExprValue timestamp, ExprValue timeZone) {
-    String defaultTimeZone = TimeZone.getDefault().getID();
+    String defaultTimeZone = TimeZone.getDefault().toZoneId().toString();
 
     try {
       LocalDateTime ldtFormatted =

ppl/src/main/antlr/OpenSearchPPLLexer.g4

@@ -7,7 +7,7 @@
 lexer grammar OpenSearchPPLLexer;
 
 channels { WHITESPACE, ERRORCHANNEL }
-
+options { caseInsensitive = true; }
 
 // COMMAND KEYWORDS
 SEARCH:                             'SEARCH';
@@ -36,7 +36,6 @@ AD:                                 'AD';
 ML:                                 'ML';
 FILLNULL:                           'FILLNULL';
 TRENDLINE:                          'TRENDLINE';
-PATTERN_METHOD:                     'PATTERN_METHOD';
 SIMPLE_PATTERN:                     'SIMPLE_PATTERN';
 BRAIN:                              'BRAIN';
 VARIABLE_COUNT_THRESHOLD:           'VARIABLE_COUNT_THRESHOLD';

ppl/src/main/antlr/OpenSearchPPLParser.g4

@@ -983,6 +983,7 @@ keywordsCanBeId
    | multiFieldRelevanceFunctionName
    | commandName
    | comparisonOperator
+   | patternMethod
    // commands assist keywords
    | IN
    | BETWEEN
@@ -994,6 +995,9 @@ keywordsCanBeId
    | FROM
    | PATTERN
    | NEW_FIELD
+   | VARIABLE_COUNT_THRESHOLD
+   | FREQUENCY_THRESHOLD_PERCENTAGE
+   | WITH
    | REGEX
    | PUNCT
    | USING
@@ -1048,10 +1052,6 @@ keywordsCanBeId
    | LAST
    | LIST
    | VALUES
-   | EARLIEST
-   | EARLIEST_TIME
-   | LATEST
-   | LATEST_TIME
    | PER_DAY
    | PER_HOUR
    | PER_MINUTE
@@ -1069,4 +1069,6 @@ keywordsCanBeId
    | FULL
    | SEMI
    | ANTI
+   | LEFT_HINT
+   | RIGHT_HINT
    ;

protocol/src/main/java/org/opensearch/sql/protocol/response/format/FlatResponseBase.java

@@ -84,7 +84,10 @@ protected List<List<String>> formatData(List<List<String>> lines) {
 
   protected String quoteIfRequired(String separator, String cell) {
     final String quote = "\"";
-    if (cell.contains(separator) || cell.contains(quote)) {
+    if (cell.contains(separator)
+        || cell.contains(quote)
+        || cell.contains("\r")
+        || cell.contains("\n")) {
       return quote + cell.replaceAll(quote, quote + quote) + quote;
     } else {
       return cell;

protocol/src/test/java/org/opensearch/sql/protocol/response/format/CsvResponseFormatterTest.java

@@ -110,9 +110,11 @@ void quoteIfRequired() {
             schema,
             Arrays.asList(
                 tupleValue(ImmutableMap.of("na,me", "John,Smith", ",,age", "30,,,")),
+                tupleValue(ImmutableMap.of("na,me", "Line\nBreak", ",,age", "28,,,")),
                 tupleValue(ImmutableMap.of("na,me", "\"Janice Jones", ",,age", "26\""))));
     String expected =
-        "\"na,me\",\",,age\"%n\"John,Smith\",\"30,,,\"%n\"\"\"Janice Jones\",\"26\"\"\"";
+        "\"na,me\",\",,age\"%n\"John,Smith\",\"30,,,\"%n\"Line\nBreak\",\"28,,,\"%n"
+            + "\"\"\"Janice Jones\",\"26\"\"\"";
     assertEquals(format(expected), formatter.format(response));
   }
 

protocol/src/test/java/org/opensearch/sql/protocol/response/format/RawResponseFormatterTest.java

@@ -135,6 +135,42 @@ void quoteIfRequired() {
     assertEquals(format(expectedPretty), getRawFormatterPretty().format(response));
   }
 
+  @Test
+  void quoteIfRequiredCrlf() {
+    ExecutionEngine.Schema schema =
+        new ExecutionEngine.Schema(
+            ImmutableList.of(
+                new ExecutionEngine.Schema.Column("field1", "field1", STRING),
+                new ExecutionEngine.Schema.Column("num", "num", INTEGER)));
+    QueryResult response =
+        new QueryResult(
+            schema,
+            Arrays.asList(
+                tupleValue(ImmutableMap.of("field1", "abc", "num", "5")),
+                tupleValue(ImmutableMap.of("field1", "def\r", "num", "\n6")),
+                tupleValue(ImmutableMap.of("field1", "gh\ri", "num", "7\n")),
+                tupleValue(ImmutableMap.of("field1", "jk\r\nl", "num", "\n\r8")),
+                tupleValue(ImmutableMap.of("field1", "\r\nmno\r\n", "num", "9\n\r0"))));
+    String expected =
+        "field1|num%n"
+            + "abc|5%n"
+            + "\"def\r\"|\"\n6\"%n"
+            + "\"gh\ri\"|\"7\n\"%n"
+            + "\"jk\r\nl\"|\"\n\r8\"%n"
+            + "\"\r\nmno\r\n\"|\"9\n\r0\"";
+    assertEquals(format(expected), getRawFormatter().format(response));
+    // Pretty formatting raw or CSV data with embedded newlines and carriage
+    // returns will still look awful on output.
+    String expectedPretty =
+        "field1   |num   %n"
+            + "abc      |5     %n"
+            + "\"def\r\"   |\"\n6\"  %n"
+            + "\"gh\ri\"   |\"7\n\"  %n"
+            + "\"jk\r\nl\"  |\"\n\r8\" %n"
+            + "\"\r\nmno\r\n\"|\"9\n\r0\"";
+    assertEquals(format(expectedPretty), getRawFormatterPretty().format(response));
+  }
+
   @Test
   void formatError() {
     Throwable t = new RuntimeException("This is an exception");