Merge branch 'main' into feature/headtop-limit

Signed-off-by: Simeon Widdis <[email protected]>

Author: Swiddis

.github/workflows/integ-tests-with-security.yml

@@ -9,6 +9,7 @@ on:
     paths:
       - 'integ-test/**'
       - '.github/workflows/integ-tests-with-security.yml'
+  merge_group:
 
 jobs:
   Get-CI-Image-Tag:

.github/workflows/sql-test-and-build-workflow.yml

@@ -16,6 +16,7 @@ on:
       - '**/*.jar'
       - '**/*.pom'
       - '.github/workflows/sql-test-and-build-workflow.yml'
+  merge_group:
 
 jobs:
   Get-CI-Image-Tag:

.pre-commit-config.yaml

@@ -1,8 +1,13 @@
 repos:
   - repo: local
     hooks:
+      - id: spotless-format
+        name: Spotless Format
+        entry: bash -c './gradlew spotlessApply && git add -u'
+        language: system
+        pass_filenames: false
       - id: spotless-check
-        name: Spotless Check
-        entry: ./gradlew spotlessCheck
+        name: Spotless Post-format Check
+        entry: bash -c './gradlew spotlessCheck'
         language: system
         pass_filenames: false

DEVELOPER_GUIDE.rst

@@ -173,6 +173,7 @@ Here are other files and sub-folders that you are likely to touch:
 - ``build.gradle``: Gradle build script.
 - ``docs``: documentation for developers and reference manual for users.
 - ``doc-test``: code that run .rst docs in ``docs`` folder by Python doctest library.
+- ``language-grammar``: centralized package for ANTLR grammar files. See `Language Grammar Package`_ for details.
 
 Note that other related project code has already merged into this single repository together:
 
@@ -441,3 +442,29 @@ with an appropriate label `backport <backport-branch-name>` is merged to main wi
 PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the
 backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR
 to the `1.x` branch.
+
+Language Grammar Package
+========================
+
+The ``language-grammar`` package serves as a centralized repository for all ANTLR grammar files used throughout the OpenSearch SQL project. This package contains the definitive versions of grammar files for:
+
+- SQL parsing (``OpenSearchSQLParser.g4``, ``OpenSearchSQLLexer.g4``)
+- PPL parsing (``OpenSearchPPLParser.g4``, ``OpenSearchPPLLexer.g4``)
+- Legacy SQL parsing (``OpenSearchLegacySqlParser.g4``, ``OpenSearchLegacySqlLexer.g4``)
+- Spark SQL extensions (``SparkSqlBase.g4``, ``FlintSparkSqlExtensions.g4``, ``SqlBaseParser.g4``, ``SqlBaseLexer.g4``)
+
+Purpose
+-------
+
+The language-grammar package enables sharing of grammar files between the main SQL repository and the Spark repository, ensuring consistency and reducing duplication. Once updated, the package automatically triggers CI to upload the new version to Maven Central for consumption by other projects.
+
+Updating Grammar Files
+----------------------
+
+When grammar files are modified in their respective modules (``sql/``, ``ppl/``, ``legacy/``, ``async-query-core/``), they must be manually copied to the ``language-grammar/src/main/antlr4/`` directory.
+
+**Workflow:**
+
+1. Modify grammar files in their source locations (e.g., ``sql/src/main/antlr/``)
+2. Copy updated files to ``language-grammar/src/main/antlr4/``
+3. Commit changes to trigger automatic Maven publication via CI

async-query-core/src/main/java/org/opensearch/sql/spark/rest/model/LangType.java

@@ -8,7 +8,8 @@
 /** Language type accepted in async query apis. */
 public enum LangType {
   SQL("sql"),
-  PPL("ppl");
+  PPL("ppl"),
+  PROMQL("promql");
   private final String text;
 
   LangType(String text) {

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -86,6 +86,7 @@
 import org.opensearch.sql.ast.tree.Rename;
 import org.opensearch.sql.ast.tree.Reverse;
 import org.opensearch.sql.ast.tree.Rex;
+import org.opensearch.sql.ast.tree.SPath;
 import org.opensearch.sql.ast.tree.Search;
 import org.opensearch.sql.ast.tree.Sort;
 import org.opensearch.sql.ast.tree.Sort.SortOption;
@@ -95,6 +96,7 @@
 import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.ast.tree.Values;
+import org.opensearch.sql.ast.tree.Window;
 import org.opensearch.sql.common.antlr.SyntaxCheckException;
 import org.opensearch.sql.data.model.ExprMissingValue;
 import org.opensearch.sql.data.type.ExprCoreType;
@@ -755,11 +757,21 @@ public LogicalPlan visitReverse(Reverse node, AnalysisContext context) {
     throw getOnlyForCalciteException("Reverse");
   }
 
+  @Override
+  public LogicalPlan visitSpath(SPath node, AnalysisContext context) {
+    throw getOnlyForCalciteException("Spath");
+  }
+
   @Override
   public LogicalPlan visitTimechart(Timechart node, AnalysisContext context) {
     throw getOnlyForCalciteException("Timechart");
   }
 
+  @Override
+  public LogicalPlan visitWindow(Window node, AnalysisContext context) {
+    throw getOnlyForCalciteException("Window");
+  }
+
   @Override
   public LogicalPlan visitRegex(Regex node, AnalysisContext context) {
     throw getOnlyForCalciteException("Regex");

core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java

@@ -38,6 +38,7 @@
 import org.opensearch.sql.ast.expression.HighlightFunction;
 import org.opensearch.sql.ast.expression.In;
 import org.opensearch.sql.ast.expression.Interval;
+import org.opensearch.sql.ast.expression.LambdaFunction;
 import org.opensearch.sql.ast.expression.Literal;
 import org.opensearch.sql.ast.expression.Not;
 import org.opensearch.sql.ast.expression.Or;
@@ -479,6 +480,11 @@ public Expression visitInSubquery(InSubquery node, AnalysisContext context) {
     throw getOnlyForCalciteException("Subsearch");
   }
 
+  @Override
+  public Expression visitLambdaFunction(LambdaFunction node, AnalysisContext context) {
+    throw getOnlyForCalciteException("Lambda function");
+  }
+
   /**
    * If QualifiedName is actually a reserved metadata field, return the expr type associated with
    * the metadata field.

core/src/main/java/org/opensearch/sql/ast/expression/Literal.java

@@ -7,6 +7,7 @@
 
 import com.google.common.collect.ImmutableList;
 import java.math.BigDecimal;
+import java.text.DecimalFormat;
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
@@ -25,7 +26,21 @@ public class Literal extends UnresolvedExpression {
 
   public Literal(Object value, DataType dataType) {
     if (dataType == DataType.DECIMAL && value instanceof Double) {
-      this.value = BigDecimal.valueOf((Double) value);
+      // This branch is only used in testing for backward compatibility:
+      // We accept decimal literal by Literal(double, DataType.DECIMAL).
+      // Some double values such as 0.0001 will be converted to string "1.0E-4" and finally
+      // generate decimal 0.00010. So here we parse a decimal text to Double then convert it
+      // to BigDecimal as well.
+      // In v2, a decimal literal will be converted back to double in resolving expression
+      // via ExprDoubleValue.
+      // In v3, a decimal literal will be kept in Calcite RexNode and converted back to double
+      // in runtime.
+      DecimalFormat df = new DecimalFormat();
+      df.setMaximumFractionDigits(38);
+      df.setMinimumFractionDigits(1);
+      df.setGroupingUsed(false);
+      String plain = df.format(value);
+      this.value = new BigDecimal(plain);
     } else {
       this.value = value;
     }

core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java

@@ -42,6 +42,11 @@ public enum SpanUnit {
     SPAN_UNITS = builder.add(SpanUnit.values()).build();
   }
 
+  /** Util method to check if the unit is time unit. */
+  public static boolean isTimeUnit(SpanUnit unit) {
+    return unit != UNKNOWN && unit != NONE;
+  }
+
   /** Util method to get span unit given the unit name. */
   public static SpanUnit of(String unit) {
     switch (unit) {