upgraded spark 3.5.5 to 3.5.6 (#1861)

Author: Yaniv Kunda

.github/actions/setup-spark-builder/action.yaml

@@ -22,7 +22,7 @@ inputs:
     description: 'The Apache Spark short version (e.g., 3.5) to build'
     required: true
   spark-version:
-    description: 'The Apache Spark version (e.g., 3.5.5) to build'
+    description: 'The Apache Spark version (e.g., 3.5.6) to build'
     required: true
 runs:
   using: "composite"

.github/workflows/spark_sql_test.yml

@@ -45,7 +45,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
         java-version: [11]
-        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.4'}, {short: '3.5', full: '3.5.5'}]
+        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.4'}, {short: '3.5', full: '3.5.5'}, {short: '3.5', full: '3.5.6'}]
         module:
           - {name: "catalyst", args1: "catalyst/test", args2: ""}
           - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}
@@ -84,7 +84,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
         java-version: [11]
-        spark-version: [{short: '3.5', full: '3.5.5'}]
+        spark-version: [{short: '3.5', full: '3.5.6'}]
         module:
           - {name: "catalyst", args1: "catalyst/test", args2: ""}
 #          - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}

.github/workflows/spark_sql_test_native_datafusion.yml

@@ -35,7 +35,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
         java-version: [11]
-        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.5'}]
+        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.6'}]
         module:
           - {name: "catalyst", args1: "catalyst/test", args2: ""}
           - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}

.github/workflows/spark_sql_test_native_iceberg_compat.yml

@@ -35,7 +35,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
         java-version: [11]
-        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.5'}]
+        spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.6'}]
         module:
           - {name: "catalyst", args1: "catalyst/test", args2: ""}
           - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}

dev/diffs/3.5.6.diff

@@ -90,12 +90,12 @@ of Apache Spark to enable Comet when running tests. This is a highly manual proc
 vary depending on the changes in the new version of Spark, but here is a general guide to the process.
 
 We typically start by applying a patch from a previous version of Spark. For example, when enabling the tests
-for Spark version 3.5.5 we may start by applying the existing diff for 3.4.3 first.
+for Spark version 3.5.6 we may start by applying the existing diff for 3.5.5 first.
 
 ```shell
 cd git/apache/spark
-git checkout v3.5.5
-git apply --reject --whitespace=fix ../datafusion-comet/dev/diffs/3.4.3.diff
+git checkout v3.5.6
+git apply --reject --whitespace=fix ../datafusion-comet/dev/diffs/3.5.5.diff
 ```
 
 Any changes that cannot be cleanly applied will instead be written out to reject files. For example, the above
@@ -140,7 +140,7 @@ configuration setting to use 11 digits hashes for consistency with existing diff
 
 ```shell
 git config core.abbrev 11;
-git diff v3.5.5 > ../datafusion-comet/dev/diffs/3.5.5.diff
+git diff v3.5.6 > ../datafusion-comet/dev/diffs/3.5.6.diff
 ```
 
 ## Running Tests in CI

docs/source/contributor-guide/spark-sql-tests.md

@@ -38,12 +38,13 @@ We recommend only using Comet with Spark versions where we currently have both C
 Other versions may work well enough for development and evaluation purposes.
 
 | Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark Tests in CI |
-| ------------- | ------------ | ------------- | ----------------- | ----------------- |
+|---------------| ------------ | ------------- | ----------------- | ----------------- |
 | 3.4.3         | 11/17        | 2.12/2.13     | Yes               | Yes               |
 | 3.5.2         | 11/17        | 2.12/2.13     | Partial\*         | No                |
 | 3.5.3         | 11/17        | 2.12/2.13     | Partial\*         | No                |
 | 3.5.4         | 11/17        | 2.12/2.13     | Partial\*         | Yes               |
-| 3.5.5         | 11/17        | 2.12/2.13     | Yes               | Yes               |
+| 3.5.5         | 11/17        | 2.12/2.13     | Partial\*         | Yes               |
+| 3.5.6         | 11/17        | 2.12/2.13     | Yes               | Yes               |
 
 \* For older Spark 3.5.x releases, we do not test the full matrix of supported Java and Scala versions in CI.
 

docs/source/user-guide/installation.md

@@ -80,17 +80,17 @@ spec:
     "spark.comet.exec.shuffle.enabled": "true"
     "spark.comet.exec.shuffle.mode": "auto"
     "spark.shuffle.manager": "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager"
-  sparkVersion: 3.5.5
+  sparkVersion: 3.5.6
   driver:
     labels:
-      version: 3.5.5
+      version: 3.5.6
     cores: 1
     coreLimit: 1200m
     memory: 512m
     serviceAccount: spark-operator-spark
   executor:
     labels:
-      version: 3.5.5
+      version: 3.5.6
     instances: 1
     cores: 1
     coreLimit: 1200m

docs/source/user-guide/kubernetes.md

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-FROM apache/spark:3.5.5 AS builder
+FROM apache/spark:3.5.6 AS builder
 
 USER root
 
@@ -61,7 +61,7 @@ RUN mkdir -p /root/.m2 && \
 RUN cd /comet \
     && JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION"
 
-FROM apache/spark:3.5.5
+FROM apache/spark:3.5.6
 ENV SPARK_VERSION=3.5
 ENV SCALA_VERSION=2.12
 USER root

kube/Dockerfile

@@ -70,7 +70,7 @@ under the License.
     <scala.plugin.version>4.8.0</scala.plugin.version>
     <scalatest.version>3.2.16</scalatest.version>
     <scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
-    <spark.version>3.5.5</spark.version>
+    <spark.version>3.5.6</spark.version>
     <spark.version.short>3.5</spark.version.short>
     <spark.maven.scope>provided</spark.maven.scope>
     <protobuf.version>3.25.5</protobuf.version>
@@ -601,7 +601,7 @@ under the License.
       <id>spark-3.5</id>
       <properties>
         <scala.version>2.12.18</scala.version>
-        <spark.version>3.5.5</spark.version>
+        <spark.version>3.5.6</spark.version>
         <spark.version.short>3.5</spark.version.short>
         <parquet.version>1.13.1</parquet.version>
         <slf4j.version>2.0.7</slf4j.version>