Add native_iceberg_compat tests to CI (apache#1487)

Author: andygrove

.github/workflows/pr_build.yml

@@ -77,20 +77,21 @@ jobs:
           upload-test-reports: ${{ matrix.java_version == '17' }}
 
   linux-test-native-datafusion-scan:
-    env:
-      COMET_PARQUET_SCAN_IMPL: "native_datafusion"
     strategy:
       matrix:
         os: [ubuntu-latest]
+        scan_impl: ['native_datafusion', 'native_iceberg_compat']
         java_version: [17]
         test-target: [rust, java]
         spark-version: ['3.5']
         scala-version: ['2.12']
         is_push_event:
           - ${{ github.event_name == 'push' }}
       fail-fast: false
-    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}-native-datafusion
+    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}-${{ matrix.scan_impl }}
     runs-on: ${{ matrix.os }}
+    env:
+      COMET_PARQUET_SCAN_IMPL: ${{ matrix.scan_impl }}
     container:
       image: amd64/rust
     steps:

spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala

@@ -525,6 +525,9 @@ class CometExecSuite extends CometTestBase {
   }
 
   test("Comet native metrics: scan") {
+    // https://github.com/apache/datafusion-comet/issues/1441
+    assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() != CometConf.SCAN_NATIVE_ICEBERG_COMPAT)
+
     withSQLConf(CometConf.COMET_EXEC_ENABLED.key -> "true") {
       withTempDir { dir =>
         val path = new Path(dir.toURI.toString, "native-scan.parquet")

spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala

@@ -1028,6 +1028,9 @@ abstract class ParquetReadSuite extends CometTestBase {
   }
 
   test("scan metrics") {
+    // https://github.com/apache/datafusion-comet/issues/1441
+    assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() != CometConf.SCAN_NATIVE_ICEBERG_COMPAT)
+
     val cometScanMetricNames = Seq(
       "ParquetRowGroups",
       "ParquetNativeDecodeTime",

spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala

@@ -49,6 +49,9 @@ class ParquetEncryptionITCase extends QueryTest with SQLTestUtils {
   private val key2 = encoder.encodeToString("1234567890123451".getBytes(StandardCharsets.UTF_8))
 
   test("SPARK-34990: Write and read an encrypted parquet") {
+    // https://github.com/apache/datafusion-comet/issues/1488
+    assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() != CometConf.SCAN_NATIVE_ICEBERG_COMPAT)
+
     import testImplicits._
 
     Seq("org.apache.parquet.crypto.keytools.PropertiesDrivenCryptoFactory").foreach {
@@ -85,6 +88,9 @@ class ParquetEncryptionITCase extends QueryTest with SQLTestUtils {
   }
 
   test("SPARK-37117: Can't read files in Parquet encryption external key material mode") {
+    // https://github.com/apache/datafusion-comet/issues/1488
+    assume(CometConf.COMET_NATIVE_SCAN_IMPL.get() != CometConf.SCAN_NATIVE_ICEBERG_COMPAT)
+
     import testImplicits._
 
     Seq("org.apache.parquet.crypto.keytools.PropertiesDrivenCryptoFactory").foreach {