[DRAFT] Update Calcite to 1.40 Take 2

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
     name: Main Build
     runs-on: ubuntu-latest
     if: github.repository == 'apache/drill'
-    timeout-minutes: 150
+    timeout-minutes: 180
     strategy:
       matrix:
         # Java versions to run unit tests

exec/java-exec/src/main/codegen/data/Parser.tdd

@@ -960,6 +960,10 @@
   includeAdditionalDeclarations: false,
   includeParsingStringLiteralAsArrayLiteral: false,
 
-  identifierMaxLength: 1024
+  identifierMaxLength: 1024,
+
+  # Configuration for Calcite 1.40 parser template
+  setOptionParserMethod: "DrillSqlSetOption",
+  truncateStatementParserMethods: []
 
 }

exec/java-exec/src/main/codegen/includes/parserImpls.ftl

@@ -988,3 +988,4 @@ SqlNode SqlDropAllAliases() :
               .build();
    }
 }
+

exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions1.java

@@ -93,7 +93,7 @@ public void add() {
         outputScale.value = in.scale;
       }
       org.apache.drill.exec.util.DecimalUtility.checkValueOverflow((java.math.BigDecimal) value.obj,
-          org.apache.drill.exec.planner.types.DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM.getMaxNumericPrecision(), outputScale.value);
+          org.apache.drill.exec.planner.types.DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM.getMaxPrecision(org.apache.calcite.sql.type.SqlTypeName.DECIMAL), outputScale.value);
       <#if type.inputType?starts_with("Nullable")>
       } // end of sout block
       </#if>
@@ -106,7 +106,7 @@ public void output() {
         out.start  = 0;
         out.scale = outputScale.value;
         out.precision =
-            org.apache.drill.exec.planner.types.DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM.getMaxNumericPrecision();
+            org.apache.drill.exec.planner.types.DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM.getMaxPrecision(org.apache.calcite.sql.type.SqlTypeName.DECIMAL);
         value.obj = ((java.math.BigDecimal) value.obj).setScale(out.scale, java.math.BigDecimal.ROUND_HALF_UP);
         byte[] bytes = ((java.math.BigDecimal) value.obj).unscaledValue().toByteArray();
         int len = bytes.length;

exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions2.java

@@ -108,7 +108,7 @@ public void output() {
       out.scale = Math.max(outputScale.value, 6);
       java.math.BigDecimal average = ((java.math.BigDecimal) value.obj)
             .divide(java.math.BigDecimal.valueOf(count.value), out.scale, java.math.BigDecimal.ROUND_HALF_UP);
-      out.precision = org.apache.drill.exec.planner.types.DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM.getMaxNumericPrecision();
+      out.precision = org.apache.drill.exec.planner.types.DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM.getMaxPrecision(org.apache.calcite.sql.type.SqlTypeName.DECIMAL);
       byte[] bytes = average.unscaledValue().toByteArray();
       int len = bytes.length;
       out.buffer = buffer = buffer.reallocIfNeeded(len);

exec/java-exec/src/main/codegen/templates/Decimal/DecimalAggrTypeFunctions3.java

@@ -102,7 +102,7 @@ public void add() {
           .add(input.subtract(temp)
                   .divide(java.math.BigDecimal.valueOf(count.value),
                       new java.math.MathContext(
-                          org.apache.drill.exec.planner.types.DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM.getMaxNumericPrecision(),
+                          org.apache.drill.exec.planner.types.DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM.getMaxPrecision(org.apache.calcite.sql.type.SqlTypeName.DECIMAL),
                           java.math.RoundingMode.HALF_UP)));
       dev.obj = ((java.math.BigDecimal) dev.obj)
           .add(input.subtract(temp).multiply(input.subtract(((java.math.BigDecimal) avg.obj))));
@@ -154,7 +154,7 @@ public void output() {
           out.scale = scale.value;
           result = result.setScale(out.scale, java.math.RoundingMode.HALF_UP);
           out.start  = 0;
-          out.precision = org.apache.drill.exec.planner.types.DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM.getMaxNumericPrecision();
+          out.precision = org.apache.drill.exec.planner.types.DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM.getMaxPrecision(org.apache.calcite.sql.type.SqlTypeName.DECIMAL);
           org.apache.drill.exec.util.DecimalUtility.checkValueOverflow(result, out.precision, out.scale);
           byte[] bytes = result.unscaledValue().toByteArray();
           int len = bytes.length;

exec/java-exec/src/main/codegen/templates/ParquetTypeHelper.java

@@ -22,6 +22,7 @@
 
 package org.apache.drill.exec.store.parquet;
 
+import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MinorType;
 import org.apache.drill.exec.planner.types.DrillRelDataTypeSystem;
@@ -168,7 +169,7 @@ public static int getMaxPrecisionForPrimitiveType(PrimitiveTypeName type) {
       case INT64:
         return 18;
       case FIXED_LEN_BYTE_ARRAY:
-        return DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM.getMaxNumericPrecision();
+        return DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM.getMaxPrecision(SqlTypeName.DECIMAL);
       default:
         throw new UnsupportedOperationException(String.format(
           "Specified PrimitiveTypeName %s cannot be used to determine max precision",

exec/java-exec/src/main/codegen/templates/Parser.jj

@@ -777,7 +777,7 @@ void LimitClause(Span s, SqlNode[] offsetFetch) :
         <COMMA> offsetFetch[1] = UnsignedNumericLiteralOrParam() {
             if (!this.conformance.isLimitStartCountAllowed()) {
                 throw SqlUtil.newContextException(s.end(this),
-                    RESOURCE.limitStartCountNotAllowed());
+                    RESOURCE.limitStartCountOrAllNotAllowed("LIMIT start, count"));
             }
         }
     |
@@ -6128,7 +6128,7 @@ SqlNode BuiltinFunctionCall() :
                 <FROM> {
                     if (null == flag && null == trimChars) {
                         throw SqlUtil.newContextException(getPos(),
-                            RESOURCE.illegalFromEmpty());
+                            RESOURCE.validationError("Invalid TRIM syntax"));
                     }
                 }
             |

exec/java-exec/src/main/java/org/apache/calcite/jdbc/DynamicRootSchema.java

@@ -21,6 +21,9 @@
 import org.apache.calcite.linq4j.tree.Expression;
 import org.apache.calcite.linq4j.tree.Expressions;
 import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.lookup.Lookup;
+import org.apache.calcite.schema.lookup.LikePattern;
+import org.apache.calcite.schema.lookup.Named;
 import org.apache.calcite.util.BuiltInMethod;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.common.exceptions.UserExceptionUtils;
@@ -41,6 +44,7 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -67,14 +71,58 @@ public class DynamicRootSchema extends DynamicSchema {
   }
 
   @Override
-  protected CalciteSchema getImplicitSubSchema(String schemaName,
-                                               boolean caseSensitive) {
-    String actualSchemaName = aliasRegistryProvider.getStorageAliasesRegistry()
-      .getUserAliases(schemaConfig.getUserName()).get(SchemaPath.getSimplePath(schemaName).toExpr());
-    return getSchema(actualSchemaName != null
-        ? SchemaPath.parseFromString(actualSchemaName).getRootSegmentPath()
-        : schemaName,
-      caseSensitive);
+  public Lookup<CalciteSchema> subSchemas() {
+    Lookup<CalciteSchema> baseLookup = super.subSchemas();
+    return new StoragePluginLookup(baseLookup);
+  }
+
+  private class StoragePluginLookup implements Lookup<CalciteSchema> {
+    private final Lookup<CalciteSchema> baseLookup;
+
+    StoragePluginLookup(Lookup<CalciteSchema> baseLookup) {
+      this.baseLookup = baseLookup;
+    }
+
+    @Override
+    public CalciteSchema get(String schemaName) {
+      // First check the base lookup for existing schemas
+      CalciteSchema existing = baseLookup.get(schemaName);
+      if (existing != null) {
+        return existing;
+      }
+
+      // Then try to resolve through storage plugin
+      String actualSchemaName = aliasRegistryProvider.getStorageAliasesRegistry()
+        .getUserAliases(schemaConfig.getUserName()).get(SchemaPath.getSimplePath(schemaName).toExpr());
+      return getSchema(actualSchemaName != null
+          ? SchemaPath.parseFromString(actualSchemaName).getRootSegmentPath()
+          : schemaName,
+        false);
+    }
+
+    @Override
+    public Named<CalciteSchema> getIgnoreCase(String schemaName) {
+      Named<CalciteSchema> existing = baseLookup.getIgnoreCase(schemaName);
+      if (existing != null) {
+        return existing;
+      }
+
+      CalciteSchema schema = get(schemaName);
+      return schema != null ? new Named<>(schemaName, schema) : null;
+    }
+
+    @Override
+    public Set<String> getNames(LikePattern pattern) {
+      Set<String> names = new HashSet<>(baseLookup.getNames(pattern));
+      // Add storage plugin names that match the pattern
+      Set<String> pluginNames = storages.availablePlugins();
+      for (String pluginName : pluginNames) {
+        if (pattern.matcher().apply(pluginName)) {
+          names.add(pluginName);
+        }
+      }
+      return names;
+    }
   }
 
   private CalciteSchema getSchema(String schemaName, boolean caseSensitive) {
@@ -230,10 +278,10 @@ private void loadSchemaFactory(String schemaName, boolean caseSensitive) {
   }
 
   @Override
-  protected TableEntry getImplicitTable(String tableName, boolean caseSensitive) {
+  protected TableEntry getImplicitTableBasedOnNullaryFunction(String tableName, boolean caseSensitive) {
     return Optional.ofNullable(getTemporaryTable(tableName, caseSensitive))
       .<TableEntry>map(table -> new TableEntryImpl(this, tableName, table.getTable(), table.sqls))
-      .orElse(super.getImplicitTable(tableName, true));
+      .orElse(super.getImplicitTableBasedOnNullaryFunction(tableName, true));
   }
 
   private TableEntry getTemporaryTable(String tableName, boolean caseSensitive) {
@@ -249,7 +297,7 @@ private TableEntry getTemporaryTable(String tableName, boolean caseSensitive) {
     }
 
     while (!pathSegment.isLastPath()) {
-      currentSchema = currentSchema.getImplicitSubSchema(pathSegment.getPath(), caseSensitive);
+      currentSchema = currentSchema.getSubSchema(pathSegment.getPath(), caseSensitive);
       pathSegment = pathSegment.getChild().getNameSegment();
     }
 

exec/java-exec/src/main/java/org/apache/calcite/jdbc/DynamicSchema.java

@@ -19,12 +19,18 @@
 
 import org.apache.calcite.schema.Schema;
 import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.Table;
+import org.apache.calcite.schema.lookup.Lookup;
+import org.apache.calcite.schema.lookup.LikePattern;
+import org.apache.calcite.schema.lookup.Named;
 import org.apache.drill.common.AutoCloseables;
 import org.apache.drill.exec.alias.AliasRegistryProvider;
 import org.apache.drill.exec.store.AbstractSchema;
 import org.apache.drill.exec.store.SchemaConfig;
 import org.apache.drill.exec.store.StoragePluginRegistry;
 
+import java.util.Set;
+import com.google.common.collect.ImmutableList;
 
 /**
  * Unlike SimpleCalciteSchema, DynamicSchema could have an empty or partial schemaMap, but it could maintain a map of
@@ -37,13 +43,8 @@ public DynamicSchema(CalciteSchema parent, Schema schema, String name) {
   }
 
   @Override
-  protected CalciteSchema getImplicitSubSchema(String schemaName,
-                                               boolean caseSensitive) {
-    Schema s = schema.getSubSchema(schemaName);
-    if (s != null) {
-      return new DynamicSchema(this, s, schemaName);
-    }
-    return getSubSchemaMap().get(schemaName);
+  protected CalciteSchema createSubSchema(Schema schema, String name) {
+    return new DynamicSchema(this, schema, name);
   }
 
   public static SchemaPlus createRootSchema(StoragePluginRegistry storages,
@@ -61,8 +62,58 @@ public CalciteSchema add(String name, Schema schema) {
   }
 
   @Override
-  protected TableEntry getImplicitTable(String tableName, boolean caseSensitive) {
-    return super.getImplicitTable(tableName, true);
+  public Lookup<TableEntry> tables() {
+    Lookup<TableEntry> baseLookup = super.tables();
+    return new ImplicitTableLookup(baseLookup);
+  }
+
+  private class ImplicitTableLookup implements Lookup<TableEntry> {
+    private final Lookup<TableEntry> baseLookup;
+
+    ImplicitTableLookup(Lookup<TableEntry> baseLookup) {
+      this.baseLookup = baseLookup;
+    }
+
+    @Override
+    public TableEntry get(String tableName) {
+      // First check the base lookup for existing tables
+      TableEntry existing = baseLookup.get(tableName);
+      if (existing != null) {
+        return existing;
+      }
+
+      // Then try to get implicit table
+      return getImplicitTableBasedOnNullaryFunction(tableName, false);
+    }
+
+    @Override
+    public Named<TableEntry> getIgnoreCase(String tableName) {
+      Named<TableEntry> existing = baseLookup.getIgnoreCase(tableName);
+      if (existing != null) {
+        return existing;
+      }
+
+      TableEntry table = getImplicitTableBasedOnNullaryFunction(tableName, true);
+      return table != null ? new Named<>(tableName, table) : null;
+    }
+
+    @Override
+    public Set<String> getNames(LikePattern pattern) {
+      // For now, just return the base lookup names
+      // Implicit tables are typically discovered on-demand
+      return baseLookup.getNames(pattern);
+    }
+  }
+
+  @Override
+  protected TableEntry getImplicitTableBasedOnNullaryFunction(String tableName, boolean caseSensitive) {
+    // Delegate to the underlying schema's getTable method
+    Table table = schema.getTable(tableName);
+    if (table != null) {
+      // Create a TableEntry for the table
+      return new CalciteSchema.TableEntryImpl(this, tableName, table, ImmutableList.of());
+    }
+    return null;
   }
 
   @Override

exec/java-exec/src/main/java/org/apache/calcite/sql/fun/SqlRandFunction.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.sql.fun;
+
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+
+/**
+ * Compatibility class for Apache Drill's integration with Calcite 1.40.
+ * This class provides the SqlRandFunction that was removed from Calcite 1.40
+ * but is still referenced by Drill's codebase during runtime.
+ *
+ * In Calcite 1.40, the RAND function implementation was moved to
+ * org.apache.calcite.runtime.RandomFunction, but some internal
+ * mechanisms still expect this SQL function class to exist.
+ */
+public class SqlRandFunction extends SqlFunction {
+
+  public SqlRandFunction() {
+    super("RAND",
+        SqlKind.OTHER_FUNCTION,
+        ReturnTypes.DOUBLE,
+        null,
+        OperandTypes.or(OperandTypes.NILADIC, OperandTypes.NUMERIC),
+        SqlFunctionCategory.NUMERIC);
+  }
+}