chore: Bump Spark from 3.5.6 to 3.5.7 (#2574)

Author: cfmcgrady

.github/actions/setup-spark-builder/action.yaml

@@ -22,7 +22,7 @@ inputs:
     description: 'The Apache Spark short version (e.g., 3.5) to build'
     required: true
   spark-version:
-    description: 'The Apache Spark version (e.g., 3.5.6) to build'
+    description: 'The Apache Spark version (e.g., 3.5.7) to build'
     required: true
 runs:
   using: "composite"

.github/workflows/iceberg_spark_test.yml

@@ -47,7 +47,7 @@ jobs:
         os: [ubuntu-24.04]
         java-version: [11, 17]
         iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}]
-        spark-version: [{short: '3.5', full: '3.5.6'}]
+        spark-version: [{short: '3.5', full: '3.5.7'}]
         scala-version: ['2.13']
       fail-fast: false
     name: iceberg-spark/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -86,7 +86,7 @@ jobs:
         os: [ubuntu-24.04]
         java-version: [11, 17]
         iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}]
-        spark-version: [{short: '3.5', full: '3.5.6'}]
+        spark-version: [{short: '3.5', full: '3.5.7'}]
         scala-version: ['2.13']
       fail-fast: false
     name: iceberg-spark-extensions/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -125,7 +125,7 @@ jobs:
         os: [ubuntu-24.04]
         java-version: [11, 17]
         iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}]
-        spark-version: [{short: '3.5', full: '3.5.6'}]
+        spark-version: [{short: '3.5', full: '3.5.7'}]
         scala-version: ['2.13']
       fail-fast: false
     name: iceberg-spark-runtime/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -155,4 +155,4 @@ jobs:
           rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
           ENABLE_COMET=true ENABLE_COMET_ONHEAP=true ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \
             :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:integrationTest \
-            -Pquick=true -x javadoc
+            -Pquick=true -x javadoc

.github/workflows/pr_build_linux.yml

@@ -74,16 +74,16 @@ jobs:
             maven_opts: "-Pspark-3.4 -Pscala-2.12"
             scan_impl: "native_comet"
 
-          - name: "Spark 3.5.4, JDK 11, Scala 2.12"
-            java_version: "11"
-            maven_opts: "-Pspark-3.5 -Dspark.version=3.5.4 -Pscala-2.12"
-            scan_impl: "native_comet"
-
           - name: "Spark 3.5.5, JDK 17, Scala 2.13"
             java_version: "17"
             maven_opts: "-Pspark-3.5 -Dspark.version=3.5.5 -Pscala-2.13"
             scan_impl: "native_comet"
 
+          - name: "Spark 3.5.6, JDK 17, Scala 2.13"
+            java_version: "17"
+            maven_opts: "-Pspark-3.5 -Dspark.version=3.5.6 -Pscala-2.13"
+            scan_impl: "native_comet"
+
           - name: "Spark 3.5, JDK 17, Scala 2.12 native_datafusion"
             java_version: "17"
             maven_opts: "-Pspark-3.5 -Pscala-2.12"
@@ -175,4 +175,4 @@ jobs:
           suites: ${{ matrix.suite.name == 'sql' && matrix.profile.name == 'Spark 3.4, JDK 11, Scala 2.12' && '' || matrix.suite.value }}
           maven_opts: ${{ matrix.profile.maven_opts }}
           scan_impl: ${{ matrix.profile.scan_impl }}
-          upload-test-reports: true
+          upload-test-reports: true

.github/workflows/spark_sql_test.yml

@@ -51,7 +51,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
         java-version: [11]
-        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.6'}]
+        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.7'}]
         module:
           - {name: "catalyst", args1: "catalyst/test", args2: ""}
           - {name: "sql_core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}
@@ -100,7 +100,7 @@ jobs:
       matrix:
         os: [ ubuntu-24.04 ]
         java-version: [ 11 ]
-        spark-version: [ { short: '3.4', full: '3.4.3' }, { short: '3.5', full: '3.5.6' } ]
+        spark-version: [ { short: '3.4', full: '3.4.3' }, { short: '3.5', full: '3.5.7' } ]
         module:
           - { name: "catalyst", args1: "catalyst/test", args2: "" }
           - { name: "sql_core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest }
@@ -149,7 +149,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
         java-version: [11]
-        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.6'}]
+        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.7'}]
         module:
           - {name: "catalyst", args1: "catalyst/test", args2: ""}
           - {name: "sql_core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}

.github/workflows/spark_sql_test_native_datafusion.yml

@@ -35,7 +35,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
         java-version: [11]
-        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.6'}]
+        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.7'}]
         module:
           - {name: "catalyst", args1: "catalyst/test", args2: ""}
           - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}

.github/workflows/spark_sql_test_native_iceberg_compat.yml

@@ -35,7 +35,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
         java-version: [11]
-        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.6'}]
+        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.7'}]
         module:
           - {name: "catalyst", args1: "catalyst/test", args2: ""}
           - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}

dev/diffs/3.5.6.diff dev/diffs/3.5.7.diffdev/diffs/3.5.6.diff renamed to dev/diffs/3.5.7.diff

@@ -38,9 +38,9 @@ Other versions may work well enough for development and evaluation purposes.
 | Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI |
 |---------------| ------------ | ------------- |-------------------|-----------------------|
 | 3.4.3         | 11/17        | 2.12/2.13     | Yes               | Yes                   |
-| 3.5.4         | 11/17        | 2.12/2.13     | Yes               | No                    |
 | 3.5.5         | 11/17        | 2.12/2.13     | Yes               | No                    |
-| 3.5.6         | 11/17        | 2.12/2.13     | Yes               | Yes                   |
+| 3.5.6         | 11/17        | 2.12/2.13     | Yes               | No                    |
+| 3.5.7         | 11/17        | 2.12/2.13     | Yes               | Yes                   |
 
 Note that we do not test the full matrix of supported Java and Scala versions in CI for every Spark version.
 

docs/source/user-guide/latest/installation.md

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-FROM apache/spark:3.5.6 AS builder
+FROM apache/spark:3.5.7 AS builder
 
 USER root
 
@@ -70,7 +70,7 @@ RUN mkdir -p /root/.m2 && \
 RUN cd /comet \
     && JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION"
 
-FROM apache/spark:3.5.6
+FROM apache/spark:3.5.7
 ENV SPARK_VERSION=3.5
 ENV SCALA_VERSION=2.12
 USER root

kube/Dockerfile

@@ -70,7 +70,7 @@ under the License.
     <scala.plugin.version>4.9.6</scala.plugin.version>
     <scalatest.version>3.2.16</scalatest.version>
     <scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
-    <spark.version>3.5.6</spark.version>
+    <spark.version>3.5.7</spark.version>
     <spark.version.short>3.5</spark.version.short>
     <spark.maven.scope>provided</spark.maven.scope>
     <protobuf.version>3.25.5</protobuf.version>
@@ -609,7 +609,7 @@ under the License.
       <id>spark-3.5</id>
       <properties>
         <scala.version>2.12.18</scala.version>
-        <spark.version>3.5.6</spark.version>
+        <spark.version>3.5.7</spark.version>
         <spark.version.short>3.5</spark.version.short>
         <parquet.version>1.13.1</parquet.version>
         <slf4j.version>2.0.7</slf4j.version>