Introduce getSqlInvokedFunctions SPI and BuiltInPluginFunctionNamespa…

.github/workflows/release-notes-check.yml

@@ -10,6 +10,9 @@ env:
 jobs:
   check_release_note:
     runs-on: ubuntu-latest
+    concurrency:
+      group: ${{ github.workflow }}-check-release-note-${{ github.event.pull_request.number }}
+      cancel-in-progress: true
     steps:
       - name: Checkout
         uses: actions/checkout@v4

README.md

@@ -40,11 +40,16 @@ After opening the project in IntelliJ, double check that the Java SDK is properl
 * Open the File menu and select Project Structure
 * In the SDKs section, ensure that a distribution of JDK 17 is selected (create one if none exist)
 * In the Project section, ensure the Project language level is set to at least 8.0.
+* When using JDK 17, an [IntelliJ bug](https://youtrack.jetbrains.com/issue/IDEA-201168) requires you
+  to disable the `Use '--release' option for cross-compilation (Java 9 and later)` setting in
+  `Settings > Build, Execution, Deployment > Compiler > Java Compiler`. If this option remains enabled,
+  you may encounter errors such as: `package sun.misc does not exist` because IntelliJ fails to resolve
+  certain internal JDK classes.
 
 Presto comes with sample configuration that should work out-of-the-box for development. Use the following options to create a run configuration:
 
 * Main Class: `com.facebook.presto.server.PrestoServer`
-* VM Options: `-ea -XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent -Xmx2G -Dconfig=etc/config.properties -Dlog.levels-file=etc/log.properties`
+* VM Options: `-ea -XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+UseGCOverheadLimit -XX:+ExplicitGCInvokesConcurrent -Xmx2G -Dconfig=etc/config.properties -Dlog.levels-file=etc/log.properties -Djdk.attach.allowAttachSelf=true`
 * Working directory: `$MODULE_WORKING_DIR$` or `$MODULE_DIR$`(Depends your version of IntelliJ)
 * Use classpath of module: `presto-main`
 
@@ -54,6 +59,32 @@ Additionally, the Hive plugin must be configured with location of your Hive meta
 
     -Dhive.metastore.uri=thrift://localhost:9083
 
+### Additional configuration for Java 17
+
+When running with Java 17, additional `--add-opens` flags are required to allow reflective access used by certain catalogs based on which catalogs are configured.  
+For the default set of catalogs loaded when starting the Presto server in IntelliJ without changes, add the following flags to the **VM Options**:
+
+    --add-opens=java.base/java.io=ALL-UNNAMED
+    --add-opens=java.base/java.lang=ALL-UNNAMED
+    --add-opens=java.base/java.lang.ref=ALL-UNNAMED
+    --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+    --add-opens=java.base/java.net=ALL-UNNAMED
+    --add-opens=java.base/java.nio=ALL-UNNAMED
+    --add-opens=java.base/java.security=ALL-UNNAMED
+    --add-opens=java.base/javax.security.auth=ALL-UNNAMED
+    --add-opens=java.base/javax.security.auth.login=ALL-UNNAMED
+    --add-opens=java.base/java.text=ALL-UNNAMED
+    --add-opens=java.base/java.util=ALL-UNNAMED
+    --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
+    --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+    --add-opens=java.base/java.util.regex=ALL-UNNAMED
+    --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED
+    --add-opens=java.base/sun.security.action=ALL-UNNAMED
+    --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED
+
+These flags ensure that internal JDK modules are accessible at runtime for components used by Presto’s default configuration.
+It is not a comprehensive list. Additional flags may need to be added, depending on the catalogs configured on the server.
+
 ### Using SOCKS for Hive or HDFS
 
 If your Hive metastore or HDFS cluster is not directly accessible to your local machine, you can use SSH port forwarding to access it. Setup a dynamic SOCKS proxy with SSH listening on local port 1080:

pom.xml

@@ -83,7 +83,8 @@
         <dep.commons.compress.version>1.26.2</dep.commons.compress.version>
         <dep.protobuf-java.version>4.29.0</dep.protobuf-java.version>
         <dep.jetty.version>12.0.18</dep.jetty.version>
-        <dep.netty.version>4.1.119.Final</dep.netty.version>
+        <dep.netty.version>4.1.122.Final</dep.netty.version>
+        <dep.reactor-netty.version>1.2.8</dep.reactor-netty.version>
         <dep.snakeyaml.version>2.0</dep.snakeyaml.version>
         <dep.gson.version>2.12.1</dep.gson.version>
         <dep.commons.lang3.version>3.18.0</dep.commons.lang3.version>
@@ -1555,6 +1556,18 @@
                 <version>0.11.5</version>
             </dependency>
 
+            <dependency>
+                <groupId>io.projectreactor.netty</groupId>
+                <artifactId>reactor-netty-core</artifactId>
+                <version>${dep.reactor-netty.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>io.projectreactor.netty</groupId>
+                <artifactId>reactor-netty-http</artifactId>
+                <version>${dep.reactor-netty.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>org.apache.thrift</groupId>
                 <artifactId>libthrift</artifactId>

presto-docs/src/main/sphinx/admin/properties.rst

@@ -150,7 +150,7 @@ To enable the ``OFFSET`` clause in SQL query expressions, set this property to `
 The corresponding session property is :ref:`admin/properties-session:\`\`offset_clause_enabled\`\``. 
 
 ``max-serializable-object-size``
-^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 * **Type:** ``long``
 * **Default value:** ``1000``

presto-docs/src/main/sphinx/connector/hive.rst

@@ -1005,12 +1005,16 @@ Hive catalog is called ``web``::
 
     CALL web.system.example_procedure()
 
-The following procedures are available:
+Create Empty Partition
+^^^^^^^^^^^^^^^^^^^^^^
 
 * ``system.create_empty_partition(schema_name, table_name, partition_columns, partition_values)``
 
   Create an empty partition in the specified table.
 
+Sync Partition Metadata
+^^^^^^^^^^^^^^^^^^^^^^^
+
 * ``system.sync_partition_metadata(schema_name, table_name, mode, case_sensitive)``
 
   Check and update partitions list in metastore. There are three modes available:
@@ -1024,6 +1028,9 @@ The following procedures are available:
   file system paths to use lowercase (e.g. ``col_x=SomeValue``). Partitions on the file system
   not conforming to this convention are ignored, unless the argument is set to ``false``.
 
+Invalidate Directory List Cache
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
 * ``system.invalidate_directory_list_cache()``
 
   Flush full directory list cache.
@@ -1032,6 +1039,9 @@ The following procedures are available:
 
   Invalidate directory list cache for specified directory_path.
 
+Invalidate Metastore Cache
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
 * ``system.invalidate_metastore_cache()``
 
   Invalidate all metastore caches.
@@ -1048,8 +1058,10 @@ The following procedures are available:
 
   Invalidate all metastore cache entries linked to a specific partition.
 
-Note: To enable ``system.invalidate_metastore_cache`` procedure, please refer to the properties that
-apply to Hive Metastore and are listed in the `Metastore Configuration Properties`_ table.
+  .. note::
+
+    To enable ``system.invalidate_metastore_cache`` procedure, ``hive.invalidate-metastore-cache-procedure-enabled`` must be set to ``true``.
+    See the properties in `Metastore Configuration Properties`_ table for more information.
 
 Extra Hidden Columns
 --------------------
@@ -1064,22 +1076,22 @@ columns as a part of the query like any other columns of the table.
 How to invalidate metastore cache?
 ----------------------------------
 
-The Hive connector exposes a procedure over JMX (``com.facebook.presto.hive.metastore.CachingHiveMetastore#flushCache``) to invalidate the metastore cache.
-You can call this procedure to invalidate the metastore cache by connecting via jconsole or jmxterm.
+Invalidating metastore cache is useful when the Hive metastore is updated outside of Presto and you want to make the changes visible to Presto immediately.
+There are a couple of ways for invalidating this cache and are listed below -
 
-This is useful when the Hive metastore is updated outside of Presto and you want to make the changes visible to Presto immediately.
+* The Hive connector exposes a procedure over JMX (``com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore#invalidateAll``) to invalidate the metastore cache. You can call this procedure to invalidate the metastore cache by connecting via jconsole or jmxterm. However, this procedure flushes the cache for all the tables in all the schemas.
 
-Currently, this procedure flushes the cache for all the tables in all the schemas. This is a known limitation and will be enhanced in the future.
+* The Hive connector exposes ``system.invalidate_metastore_cache`` procedure which enables users to invalidate the metastore cache completely or partially as per the requirement and can be invoked with various arguments. See `Invalidate Metastore Cache`_ for more information.
 
 How to invalidate directory list cache?
 ---------------------------------------
 
-The Hive connector exposes a procedure over JMX (``com.facebook.presto.hive.HiveDirectoryLister#flushCache``) to invalidate the directory list cache.
-You can call this procedure to invalidate the directory list cache by connecting via jconsole or jmxterm.
+Invalidating directory list cache is useful when the files are added or deleted in the cache directory path and you want to make the changes visible to Presto immediately.
+There are a couple of ways for invalidating this cache and are listed below -
 
-This is useful when the files are added or deleted in the cache directory path and you want to make the changes visible to Presto immediately.
+* The Hive connector exposes a procedure over JMX (``com.facebook.presto.hive.CachingDirectoryLister#flushCache``) to invalidate the directory list cache. You can call this procedure to invalidate the directory list cache by connecting via jconsole or jmxterm. This procedure flushes all the cache entries.
 
-Currently, this procedure flushes all the cache entries. This is a known limitation and will be enhanced in the future.
+* The Hive connector exposes ``system.invalidate_directory_list_cache`` procedure which gives the flexibility to invalidate the list cache completely or partially as per the requirement and can be invoked in various ways. See `Invalidate Directory List Cache`_ for more information.
 
 Examples
 --------

presto-docs/src/main/sphinx/connector/iceberg.rst

@@ -2208,7 +2208,7 @@ Sorting can be combined with partitioning on the same column. For example::
         sorted_by = ARRAY['join_date']
     )
 
-The Iceberg connector does not support sort order transforms. The following sort order transformations are not supported:
+Sort order does not support transforms. The following transforms are not supported:
 
 .. code-block:: text
 

presto-hive-common/src/main/java/com/facebook/presto/hive/HiveErrorCode.java

@@ -76,6 +76,7 @@ public enum HiveErrorCode
     HIVE_RANGER_SERVER_ERROR(48, EXTERNAL),
     HIVE_FUNCTION_INITIALIZATION_ERROR(49, EXTERNAL),
     HIVE_METASTORE_INITIALIZE_SSL_ERROR(50, EXTERNAL),
+    UNKNOWN_TABLE_TYPE(51, EXTERNAL),
     /**/;
 
     private final ErrorCode errorCode;

presto-iceberg/src/main/java/com/facebook/presto/iceberg/UnknownTableTypeException.java → presto-hive-common/src/main/java/com/facebook/presto/hive/UnknownTableTypeException.java

@@ -11,18 +11,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.facebook.presto.iceberg;
+package com.facebook.presto.hive;
 
 import com.facebook.presto.spi.PrestoException;
-import com.facebook.presto.spi.SchemaTableName;
 
-import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_UNKNOWN_TABLE_TYPE;
+import static com.facebook.presto.hive.HiveErrorCode.UNKNOWN_TABLE_TYPE;
 
 public class UnknownTableTypeException
         extends PrestoException
 {
-    public UnknownTableTypeException(SchemaTableName tableName)
+    public UnknownTableTypeException(String message)
     {
-        super(ICEBERG_UNKNOWN_TABLE_TYPE, "Not an Iceberg table: " + tableName);
+        super(UNKNOWN_TABLE_TYPE, message);
     }
 }

presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java

@@ -15,6 +15,7 @@
 
 import com.facebook.airlift.json.JsonCodec;
 import com.facebook.airlift.json.smile.SmileCodec;
+import com.facebook.airlift.log.Logger;
 import com.facebook.presto.common.CatalogSchemaName;
 import com.facebook.presto.common.Page;
 import com.facebook.presto.common.Subfield;
@@ -389,6 +390,7 @@
 public class HiveMetadata
         implements TransactionalMetadata
 {
+    private static final Logger log = Logger.get(HiveMetadata.class);
     public static final Set<String> RESERVED_ROLES = ImmutableSet.of("all", "default", "none");
     public static final String REFERENCED_MATERIALIZED_VIEWS = "referenced_materialized_views";
 
@@ -674,7 +676,7 @@ private ConnectorTableMetadata getTableMetadata(Optional<Table> table, SchemaTab
         }
 
         if (isIcebergTable(table.get()) || isDeltaLakeTable(table.get())) {
-            throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, format("Not a Hive table '%s'", tableName));
+            throw new UnknownTableTypeException("Not a Hive table: " + tableName);
         }
 
         List<TableConstraint<String>> tableConstraints = metastore.getTableConstraints(metastoreContext, tableName.getSchemaName(), tableName.getTableName());
@@ -862,6 +864,9 @@ public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSess
             catch (TableNotFoundException e) {
                 // table disappeared during listing operation
             }
+            catch (UnknownTableTypeException e) {
+                log.warn(String.format("%s: Unknown table type of table %s", e.getMessage(), tableName));
+            }
         }
         return columns.build();
     }

presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java

@@ -17,6 +17,7 @@
 import com.facebook.airlift.log.Logger;
 import com.facebook.presto.hive.HdfsContext;
 import com.facebook.presto.hive.HdfsEnvironment;
+import com.facebook.presto.hive.UnknownTableTypeException;
 import com.facebook.presto.hive.metastore.ExtendedHiveMetastore;
 import com.facebook.presto.hive.metastore.HivePrivilegeInfo;
 import com.facebook.presto.hive.metastore.MetastoreContext;
@@ -215,7 +216,7 @@ public TableMetadata refresh()
         Table table = getTable();
 
         if (!isIcebergTable(table)) {
-            throw new UnknownTableTypeException(getSchemaTableName());
+            throw new UnknownTableTypeException("Not an Iceberg table: " + getSchemaTableName());
         }
 
         if (isPrestoView(table)) {

presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java

@@ -28,6 +28,7 @@
 import com.facebook.presto.hive.HiveOutputMetadata;
 import com.facebook.presto.hive.HivePartition;
 import com.facebook.presto.hive.NodeVersion;
+import com.facebook.presto.hive.UnknownTableTypeException;
 import com.facebook.presto.iceberg.changelog.ChangelogOperation;
 import com.facebook.presto.iceberg.changelog.ChangelogUtil;
 import com.facebook.presto.iceberg.statistics.StatisticsFileCache;

presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergErrorCode.java

@@ -24,7 +24,6 @@
 public enum IcebergErrorCode
         implements ErrorCodeSupplier
 {
-    ICEBERG_UNKNOWN_TABLE_TYPE(0, EXTERNAL),
     ICEBERG_INVALID_METADATA(1, EXTERNAL),
     ICEBERG_TOO_MANY_OPEN_PARTITIONS(2, USER_ERROR),
     ICEBERG_INVALID_PARTITION_VALUE(3, EXTERNAL),

presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java

@@ -24,6 +24,7 @@
 import com.facebook.presto.hive.HiveTypeTranslator;
 import com.facebook.presto.hive.NodeVersion;
 import com.facebook.presto.hive.TableAlreadyExistsException;
+import com.facebook.presto.hive.UnknownTableTypeException;
 import com.facebook.presto.hive.ViewAlreadyExistsException;
 import com.facebook.presto.hive.metastore.Column;
 import com.facebook.presto.hive.metastore.Database;
@@ -230,7 +231,7 @@ protected boolean tableExists(ConnectorSession session, SchemaTableName schemaTa
             return false;
         }
         if (!isIcebergTable(hiveTable.get())) {
-            throw new UnknownTableTypeException(schemaTableName);
+            throw new UnknownTableTypeException("Not an Iceberg table: " + schemaTableName);
         }
         return true;
     }

presto-jdbc/pom.xml

@@ -123,6 +123,12 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.facebook.presto</groupId>
+            <artifactId>presto-memory</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>com.facebook.presto</groupId>
             <artifactId>presto-main-base</artifactId>
@@ -218,11 +224,13 @@
             <artifactId>jjwt-api</artifactId>
             <scope>test</scope>
         </dependency>
+
         <dependency>
             <groupId>io.jsonwebtoken</groupId>
             <artifactId>jjwt-impl</artifactId>
             <scope>test</scope>
         </dependency>
+
         <dependency>
             <groupId>io.jsonwebtoken</groupId>
             <artifactId>jjwt-jackson</artifactId>

presto-jdbc/src/main/java/com/facebook/presto/jdbc/PrestoDatabaseMetaData.java

@@ -1144,8 +1144,7 @@ public boolean insertsAreDetected(int type)
     public boolean supportsBatchUpdates()
             throws SQLException
     {
-        // TODO: support batch updates
-        return false;
+        return true;
     }
 
     @Override