Bump Calcite to 1.41.0 (#4714)

* Bump Calcite to 1.41.0

Signed-off-by: Lantao Jin <[email protected]>

* Workaround the Calcite codegen bug

Signed-off-by: Lantao Jin <[email protected]>

* upgrade calcite-testkit

Signed-off-by: Lantao Jin <[email protected]>

---------

Signed-off-by: Lantao Jin <[email protected]>

Author: LantaoJin

api/build.gradle

@@ -15,7 +15,7 @@ dependencies {
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
     testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}"
     testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}"
-    testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.38.0'
+    testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.41.0'
 }
 
 spotless {

core/build.gradle

@@ -58,19 +58,19 @@ dependencies {
     api group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
     api group: 'com.tdunning', name: 't-digest', version: '3.3'
     api "net.minidev:json-smart:${versions.json_smart}"
-    api('org.apache.calcite:calcite-core:1.38.0') {
+    api('org.apache.calcite:calcite-core:1.41.0') {
         exclude group: 'net.minidev', module: 'json-smart'
-        exclude group: 'commons-lang', module: 'commons-lang'
     }
-    api 'org.apache.calcite:calcite-linq4j:1.38.0'
+    api 'org.apache.calcite:calcite-linq4j:1.41.0'
     api project(':common')
     implementation "com.github.seancfoley:ipaddress:5.4.2"
     implementation "com.jayway.jsonpath:json-path:2.9.0"
 
     annotationProcessor('org.immutables:value:2.8.8')
-    compileOnly('org.immutables:value-annotations:2.8.8')
+    compileOnly 'org.immutables:value-annotations:2.8.8'
+    compileOnlyApi 'com.google.code.findbugs:jsr305:3.0.2'
 
-    testImplementation('org.junit.jupiter:junit-jupiter:5.9.3')
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3'
     testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}"
     testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}"
     testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${mockito_version}"

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -39,6 +39,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import org.apache.calcite.adapter.enumerable.RexToLixTranslator;
 import org.apache.calcite.plan.RelOptTable;
 import org.apache.calcite.plan.ViewExpanders;
 import org.apache.calcite.rel.RelNode;
@@ -60,6 +61,8 @@
 import org.apache.calcite.rex.RexWindowBounds;
 import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.ArraySqlType;
+import org.apache.calcite.sql.type.MapSqlType;
 import org.apache.calcite.sql.type.SqlTypeFamily;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.tools.RelBuilder;
@@ -906,7 +909,7 @@ private boolean isCountField(RexCall call) {
 
   /**
    * Resolve the aggregation with trimming unused fields to avoid bugs in {@link
-   * org.apache.calcite.sql2rel.RelDecorrelator#decorrelateRel(Aggregate, boolean)}
+   * org.apache.calcite.sql2rel.RelDecorrelator#decorrelateRel(Aggregate, boolean, boolean)}
    *
    * @param groupExprList group by expression list
    * @param aggExprList aggregate expression list
@@ -2614,6 +2617,21 @@ private void buildParseRelNode(Parse node, CalcitePlanContext context) {
     projectPlusOverriding(newFields, groupCandidates, context);
   }
 
+  /**
+   * CALCITE-6981 introduced a stricter type checking for Array type in {@link RexToLixTranslator}.
+   * We defined a MAP(VARCHAR, ANY) in {@link UserDefinedFunctionUtils#nullablePatternAggList}, when
+   * we convert the value type to ArraySqlType, it will check the source data type by {@link
+   * RelDataType#getComponentType()} which will return null due to the source type is ANY.
+   */
+  private RexNode explicitMapType(
+      CalcitePlanContext context, RexNode origin, SqlTypeName targetType) {
+    MapSqlType originalMapType = (MapSqlType) origin.getType();
+    ArraySqlType newValueType =
+        new ArraySqlType(context.rexBuilder.getTypeFactory().createSqlType(targetType), true);
+    MapSqlType newMapType = new MapSqlType(originalMapType.getKeyType(), newValueType, true);
+    return new RexInputRef(((RexInputRef) origin).getIndex(), newMapType);
+  }
+
   private void flattenParsedPattern(
       String originalPatternResultAlias,
       RexNode parsedNode,
@@ -2674,7 +2692,7 @@ private void flattenParsedPattern(
               PPLFuncImpTable.INSTANCE.resolve(
                   context.rexBuilder,
                   BuiltinFunctionName.INTERNAL_ITEM,
-                  parsedNode,
+                  explicitMapType(context, parsedNode, SqlTypeName.VARCHAR),
                   context.rexBuilder.makeLiteral(PatternUtils.SAMPLE_LOGS)),
               true,
               true);

core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java

@@ -178,7 +178,13 @@ public static List<Expression> convertToExprValues(
         types.stream().map(OpenSearchTypeFactory::convertRelDataTypeToExprType).toList();
     List<Expression> exprValues = new ArrayList<>();
     for (int i = 0; i < operands.size(); i++) {
-      Expression operand = Expressions.convert_(operands.get(i), Object.class);
+      // TODO a workaround of Apache Calcite bug in 1.41.0:
+      // If you call Expressions.convert_(expr, Number.class) or
+      // Expressions.convert_(expr, Object.class),
+      // you must change to Expressions.convert_(Expressions.box(expr), Number.class/Object.class).
+      // Because the codegen in Janino.UnitCompiler, "(Object) -1" will be mistakenly treated to
+      // "Object subtracting one" instead of "type casting on native one".
+      Expression operand = Expressions.convert_(Expressions.box(operands.get(i)), Object.class);
       exprValues.add(
           i,
           Expressions.call(

core/src/main/java/org/opensearch/sql/executor/QueryService.java

@@ -156,7 +156,7 @@ public void explainWithCalcite(
             } else {
               if (t instanceof Error) {
                 // Calcite may throw AssertError during query execution.
-                listener.onFailure(new CalciteUnsupportedException(t.getMessage()));
+                listener.onFailure(new CalciteUnsupportedException(t.getMessage(), t));
               } else {
                 listener.onFailure((Exception) t);
               }

core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java

@@ -94,7 +94,6 @@
 import org.opensearch.sql.expression.function.udf.ip.CidrMatchFunction;
 import org.opensearch.sql.expression.function.udf.ip.CompareIpFunction;
 import org.opensearch.sql.expression.function.udf.ip.IPFunction;
-import org.opensearch.sql.expression.function.udf.math.CRC32Function;
 import org.opensearch.sql.expression.function.udf.math.ConvFunction;
 import org.opensearch.sql.expression.function.udf.math.DivideFunction;
 import org.opensearch.sql.expression.function.udf.math.EulerFunction;
@@ -128,7 +127,6 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable {
   public static final SqlOperator E = new EulerFunction().toUDF("E");
   public static final SqlOperator CONV = new ConvFunction().toUDF("CONVERT");
   public static final SqlOperator MOD = new ModFunction().toUDF("MOD");
-  public static final SqlOperator CRC32 = new CRC32Function().toUDF("CRC32");
   public static final SqlOperator DIVIDE = new DivideFunction().toUDF("DIVIDE");
   public static final SqlOperator SHA2 = CryptographicFunction.sha2().toUDF("SHA2");
   public static final SqlOperator CIDRMATCH = new CidrMatchFunction().toUDF("CIDRMATCH");

core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java

@@ -832,6 +832,7 @@ void populate() {
       registerOperator(LOG2, SqlLibraryOperators.LOG2);
       registerOperator(MD5, SqlLibraryOperators.MD5);
       registerOperator(SHA1, SqlLibraryOperators.SHA1);
+      registerOperator(CRC32, SqlLibraryOperators.CRC32);
       registerOperator(INTERNAL_REGEXP_REPLACE_3, SqlLibraryOperators.REGEXP_REPLACE_3);
       registerOperator(INTERNAL_REGEXP_REPLACE_PG_4, SqlLibraryOperators.REGEXP_REPLACE_PG_4);
       registerOperator(INTERNAL_REGEXP_REPLACE_5, SqlLibraryOperators.REGEXP_REPLACE_5);
@@ -852,7 +853,6 @@ void populate() {
       registerOperator(MOD, PPLBuiltinOperators.MOD);
       registerOperator(MODULUS, PPLBuiltinOperators.MOD);
       registerOperator(MODULUSFUNCTION, PPLBuiltinOperators.MOD);
-      registerOperator(CRC32, PPLBuiltinOperators.CRC32);
       registerDivideFunction(DIVIDE);
       registerDivideFunction(DIVIDEFUNCTION);
       registerOperator(SHA2, PPLBuiltinOperators.SHA2);

core/src/main/java/org/opensearch/sql/expression/function/udf/binning/MinspanBucketFunction.java

@@ -64,10 +64,10 @@ public Expression implement(
       return Expressions.call(
           MinspanBucketImplementor.class,
           "calculateMinspanBucket",
-          Expressions.convert_(fieldValue, Number.class),
-          Expressions.convert_(minSpan, Number.class),
-          Expressions.convert_(dataRange, Number.class),
-          Expressions.convert_(maxValue, Number.class));
+          Expressions.convert_(Expressions.box(fieldValue), Number.class),
+          Expressions.convert_(Expressions.box(minSpan), Number.class),
+          Expressions.convert_(Expressions.box(dataRange), Number.class),
+          Expressions.convert_(Expressions.box(maxValue), Number.class));
     }
 
     /** Minspan bucket calculation. */

core/src/main/java/org/opensearch/sql/expression/function/udf/binning/RangeBucketFunction.java

@@ -69,11 +69,11 @@ public Expression implement(
       return Expressions.call(
           RangeBucketImplementor.class,
           "calculateRangeBucket",
-          Expressions.convert_(fieldValue, Number.class),
-          Expressions.convert_(dataMin, Number.class),
-          Expressions.convert_(dataMax, Number.class),
-          Expressions.convert_(startParam, Number.class),
-          Expressions.convert_(endParam, Number.class));
+          Expressions.convert_(Expressions.box(fieldValue), Number.class),
+          Expressions.convert_(Expressions.box(dataMin), Number.class),
+          Expressions.convert_(Expressions.box(dataMax), Number.class),
+          Expressions.convert_(Expressions.box(startParam), Number.class),
+          Expressions.convert_(Expressions.box(endParam), Number.class));
     }
 
     /** Range bucket calculation with expansion algorithm and magnitude-based width. */

core/src/main/java/org/opensearch/sql/expression/function/udf/binning/SpanBucketFunction.java

@@ -60,8 +60,8 @@ public Expression implement(
       return Expressions.call(
           SpanBucketImplementor.class,
           "calculateSpanBucket",
-          Expressions.convert_(fieldValue, Number.class),
-          Expressions.convert_(spanValue, Number.class));
+          Expressions.convert_(Expressions.box(fieldValue), Number.class),
+          Expressions.convert_(Expressions.box(spanValue), Number.class));
     }
 
     /** Span bucket calculation. */