Merge remote-tracking branch 'apache/main' into regexp-extract-impl

Author: andygrove

.github/actions/setup-iceberg-builder/action.yaml

@@ -25,7 +25,7 @@ runs:
   using: "composite"
   steps:
     - name: Clone Iceberg repo
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
       with:
         repository: apache/iceberg
         path: apache-iceberg

.github/actions/setup-iceberg-rust-builder/action.yaml

@@ -25,7 +25,7 @@ runs:
   using: "composite"
   steps:
     - name: Clone Iceberg repo
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
       with:
         repository: apache/iceberg
         path: apache-iceberg

.github/actions/setup-spark-builder/action.yaml

@@ -28,7 +28,7 @@ runs:
   using: "composite"
   steps:
     - name: Clone Spark repo
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
       with:
         repository: apache/spark
         path: apache-spark

.github/workflows/benchmark-tpcds.yml

@@ -55,7 +55,7 @@ jobs:
           rust-version: ${{env.RUST_VERSION}}
           jdk-version: 11
       - name: Cache Maven dependencies
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.m2/repository
@@ -67,7 +67,7 @@ jobs:
         run: make release
       - name: Cache TPC-DS generated data
         id: cache-tpcds-sf-1
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ./tpcds-sf-1
           key: tpcds-${{ hashFiles('.github/workflows/benchmark.yml') }}
@@ -107,7 +107,7 @@ jobs:
           rust-version: ${{env.RUST_VERSION}}
           jdk-version: 11
       - name: Cache Maven dependencies
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.m2/repository
@@ -117,7 +117,7 @@ jobs:
             ${{ runner.os }}-java-maven-
       - name: Restore TPC-DS generated data
         id: cache-tpcds-sf-1
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@v5
         with:
           path: ./tpcds-sf-1
           key: tpcds-${{ hashFiles('.github/workflows/benchmark.yml') }}

.github/workflows/benchmark-tpch.yml

@@ -55,7 +55,7 @@ jobs:
           rust-version: ${{env.RUST_VERSION}}
           jdk-version: 11
       - name: Cache Maven dependencies
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.m2/repository
@@ -65,7 +65,7 @@ jobs:
             ${{ runner.os }}-java-maven-
       - name: Cache TPC-H generated data
         id: cache-tpch-sf-1
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ./tpch
           key: tpch-${{ hashFiles('.github/workflows/benchmark-tpch.yml') }}
@@ -91,7 +91,7 @@ jobs:
           rust-version: ${{env.RUST_VERSION}}
           jdk-version: 11
       - name: Cache Maven dependencies
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.m2/repository
@@ -101,7 +101,7 @@ jobs:
             ${{ runner.os }}-java-maven-
       - name: Restore TPC-H generated data
         id: cache-tpch-sf-1
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@v5
         with:
           path: ./tpch
           key: tpch-${{ hashFiles('.github/workflows/benchmark-tpch.yml') }}

.github/workflows/pr_build_linux.yml

@@ -132,6 +132,7 @@ jobs:
               org.apache.comet.exec.CometAggregateSuite
               org.apache.comet.exec.CometExec3_4PlusSuite
               org.apache.comet.exec.CometExecSuite
+              org.apache.comet.exec.CometGenerateExecSuite
               org.apache.comet.exec.CometWindowExecSuite
               org.apache.comet.exec.CometJoinSuite
               org.apache.comet.CometNativeSuite
@@ -140,6 +141,8 @@ jobs:
               org.apache.spark.CometPluginsDefaultSuite
               org.apache.spark.CometPluginsNonOverrideSuite
               org.apache.spark.CometPluginsUnifiedModeOverrideSuite
+              org.apache.comet.rules.CometScanRuleSuite
+              org.apache.comet.rules.CometExecRuleSuite
               org.apache.spark.sql.CometTPCDSQuerySuite
               org.apache.spark.sql.CometTPCDSQueryTestSuite
               org.apache.spark.sql.CometTPCHQuerySuite
@@ -158,6 +161,7 @@ jobs:
               org.apache.comet.CometStringExpressionSuite
               org.apache.comet.CometBitwiseExpressionSuite
               org.apache.comet.CometMapExpressionSuite
+              org.apache.comet.CometJsonExpressionSuite
               org.apache.comet.expressions.conditional.CometIfSuite
               org.apache.comet.expressions.conditional.CometCoalesceSuite
               org.apache.comet.expressions.conditional.CometCaseWhenSuite

.github/workflows/pr_build_macos.yml

@@ -57,11 +57,9 @@ jobs:
             java_version: "17"
             maven_opts: "-Pspark-3.5 -Pscala-2.13"
 
-            # TODO fails with OOM
-            # https://github.com/apache/datafusion-comet/issues/1949
-#          - name: "Spark 4.0, JDK 17, Scala 2.13"
-#            java_version: "17"
-#            maven_opts: "-Pspark-4.0 -Pscala-2.13"
+          - name: "Spark 4.0, JDK 17, Scala 2.13"
+            java_version: "17"
+            maven_opts: "-Pspark-4.0 -Pscala-2.13"
 
         suite:
           - name: "fuzz"
@@ -97,6 +95,7 @@ jobs:
               org.apache.comet.exec.CometAggregateSuite
               org.apache.comet.exec.CometExec3_4PlusSuite
               org.apache.comet.exec.CometExecSuite
+              org.apache.comet.exec.CometGenerateExecSuite
               org.apache.comet.exec.CometWindowExecSuite
               org.apache.comet.exec.CometJoinSuite
               org.apache.comet.CometNativeSuite
@@ -105,6 +104,8 @@ jobs:
               org.apache.spark.CometPluginsDefaultSuite
               org.apache.spark.CometPluginsNonOverrideSuite
               org.apache.spark.CometPluginsUnifiedModeOverrideSuite
+              org.apache.comet.rules.CometScanRuleSuite
+              org.apache.comet.rules.CometExecRuleSuite
               org.apache.spark.sql.CometTPCDSQuerySuite
               org.apache.spark.sql.CometTPCDSQueryTestSuite
               org.apache.spark.sql.CometTPCHQuerySuite
@@ -123,6 +124,7 @@ jobs:
               org.apache.comet.CometStringExpressionSuite
               org.apache.comet.CometBitwiseExpressionSuite
               org.apache.comet.CometMapExpressionSuite
+              org.apache.comet.CometJsonExpressionSuite
               org.apache.comet.expressions.conditional.CometIfSuite
               org.apache.comet.expressions.conditional.CometCoalesceSuite
               org.apache.comet.expressions.conditional.CometCaseWhenSuite

.github/workflows/spark_sql_test.yml

@@ -59,6 +59,10 @@ jobs:
           - {name: "sql_hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"}
           - {name: "sql_hive-2", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.ExtendedHiveTest"}
           - {name: "sql_hive-3", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.SlowHiveTest"}
+        # Skip sql_hive-1 for Spark 4.0 due to https://github.com/apache/datafusion-comet/issues/2946
+        exclude:
+          - spark-version: {short: '4.0', full: '4.0.1', java: 17}
+            module: {name: "sql_hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"}
       fail-fast: false
     name: spark-sql-${{ matrix.module.name }}/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.spark-version.java }}
     runs-on: ${{ matrix.os }}
@@ -89,7 +93,7 @@ jobs:
           LC_ALL: "C.UTF-8"
       - name: Upload fallback log
         if: ${{ github.event.inputs.collect-fallback-logs == 'true' }}
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: fallback-log-spark-sql-${{ matrix.module.name }}-${{ matrix.os }}-spark-${{ matrix.spark-version.full }}-java-${{ matrix.spark-version.java }}
           path: "**/fallback.log"
@@ -138,7 +142,7 @@ jobs:
           LC_ALL: "C.UTF-8"
       - name: Upload fallback log
         if: ${{ github.event.inputs.collect-fallback-logs == 'true' }}
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: fallback-log-spark-sql-native-comet-${{ matrix.module.name }}-${{ matrix.os }}-spark-${{ matrix.spark-version.full }}-java-${{ matrix.java-version }}
           path: "**/fallback.log"
@@ -187,7 +191,7 @@ jobs:
           LC_ALL: "C.UTF-8"
       - name: Upload fallback log
         if: ${{ github.event.inputs.collect-fallback-logs == 'true' }}
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: fallback-log-spark-sql-iceberg-compat-${{ matrix.module.name }}-${{ matrix.os }}-spark-${{ matrix.spark-version.full }}-java-${{ matrix.java-version }}
           path: "**/fallback.log"
@@ -199,14 +203,14 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Download fallback log artifacts
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v7
         with:
           path: fallback-logs/
       - name: Merge fallback logs
         run: |
           find ./fallback-logs/ -type f -name "fallback.log" -print0 | xargs -0 cat | sort -u > all_fallback.log
       - name: Upload merged fallback log
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: all-fallback-log
           path: all_fallback.log

README.md

@@ -102,7 +102,7 @@ To get started with Apache DataFusion Comet, follow the
 [DataFusion Slack and Discord channels](https://datafusion.apache.org/contributor-guide/communication.html) to connect
 with other users, ask questions, and share your experiences with Comet.
 
-Follow [Apache DataFusion Comet Overview](https://datafusion.apache.org/comet/user-guide/overview.html) to get more detailed information
+Follow [Apache DataFusion Comet Overview](https://datafusion.apache.org/comet/about/index.html#comet-overview) to get more detailed information
 
 ## Contributing
 

common/src/main/java/org/apache/comet/parquet/CometFileKeyUnwrapper.java

@@ -101,13 +101,35 @@ public class CometFileKeyUnwrapper {
   // Cache the hadoopConf just to assert the assumption above.
   private Configuration conf = null;
 
+  /**
+   * Normalizes S3 URI schemes to a canonical form. S3 can be accessed via multiple schemes (s3://,
+   * s3a://, s3n://) that refer to the same logical filesystem. This method ensures consistent cache
+   * lookups regardless of which scheme is used.
+   *
+   * @param filePath The file path that may contain an S3 URI
+   * @return The file path with normalized S3 scheme (s3a://)
+   */
+  private String normalizeS3Scheme(final String filePath) {
+    // Normalize s3:// and s3n:// to s3a:// for consistent cache lookups
+    // This handles the case where ObjectStoreUrl uses s3:// but Spark uses s3a://
+    String s3Prefix = "s3://";
+    String s3nPrefix = "s3n://";
+    if (filePath.startsWith(s3Prefix)) {
+      return "s3a://" + filePath.substring(s3Prefix.length());
+    } else if (filePath.startsWith(s3nPrefix)) {
+      return "s3a://" + filePath.substring(s3nPrefix.length());
+    }
+    return filePath;
+  }
+
   /**
    * Creates and stores a DecryptionKeyRetriever instance for the given file path.
    *
    * @param filePath The path to the Parquet file
    * @param hadoopConf The Hadoop Configuration to use for this file path
    */
   public void storeDecryptionKeyRetriever(final String filePath, final Configuration hadoopConf) {
+    final String normalizedPath = normalizeS3Scheme(filePath);
     // Use DecryptionPropertiesFactory.loadFactory to get the factory and then call
     // getFileDecryptionProperties
     if (factory == null) {
@@ -122,7 +144,7 @@ public void storeDecryptionKeyRetriever(final String filePath, final Configurati
         factory.getFileDecryptionProperties(hadoopConf, path);
 
     DecryptionKeyRetriever keyRetriever = decryptionProperties.getKeyRetriever();
-    retrieverCache.put(filePath, keyRetriever);
+    retrieverCache.put(normalizedPath, keyRetriever);
   }
 
   /**
@@ -136,7 +158,8 @@ public void storeDecryptionKeyRetriever(final String filePath, final Configurati
    */
   public byte[] getKey(final String filePath, final byte[] keyMetadata)
       throws ParquetCryptoRuntimeException {
-    DecryptionKeyRetriever keyRetriever = retrieverCache.get(filePath);
+    final String normalizedPath = normalizeS3Scheme(filePath);
+    DecryptionKeyRetriever keyRetriever = retrieverCache.get(normalizedPath);
     if (keyRetriever == null) {
       throw new ParquetCryptoRuntimeException(
           "Failed to find DecryptionKeyRetriever for path: " + filePath);