Upgrade Flink and Spark to latest patched version

.github/workflows/master.yml

@@ -82,7 +82,7 @@ jobs:
           - java: 8
             python: '3.9'
             spark: '3.5'
-            spark-archive: '-Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-3.4.3 -Dspark.archive.name=spark-3.4.3-bin-hadoop3.tgz -Pzookeeper-3.6'
+            spark-archive: '-Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-3.4.4 -Dspark.archive.name=spark-3.4.4-bin-hadoop3.tgz -Pzookeeper-3.6'
             exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
             comment: 'verify-on-spark-3.4-binary'
           - java: 17
@@ -277,7 +277,7 @@ jobs:
             comment: 'verify-on-flink-1.18-binary'
           - java: 8
             flink: '1.20'
-            flink-archive: '-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.19.1 -Dflink.archive.name=flink-1.19.1-bin-scala_2.12.tgz'
+            flink-archive: '-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.19.3 -Dflink.archive.name=flink-1.19.3-bin-scala_2.12.tgz'
             comment: 'verify-on-flink-1.19-binary'
     steps:
       - uses: actions/checkout@v4
@@ -439,14 +439,14 @@ jobs:
           cache-binary: false
       - name: Pull Spark image
         run: |
-          docker pull apache/spark:3.5.5
+          docker pull apache/spark:3.5.7
       - name: Build Kyuubi Docker Image
         uses: docker/build-push-action@v6
         with:
           # passthrough CI into build container
           build-args: |
             CI=${CI}
-            BASE_IMAGE=apache/spark:3.5.5
+            BASE_IMAGE=apache/spark:3.5.7
             MVN_ARG=--spark-provided --flink-provided --hive-provided
           context: .
           file: build/Dockerfile.CI
@@ -463,7 +463,7 @@ jobs:
           # https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime
           minikube image load apache/kyuubi:ci
           # pre-install spark into minikube
-          minikube image load apache/spark:3.5.5
+          minikube image load apache/spark:3.5.7
       - name: kubectl pre-check
         run: |
           kubectl get nodes

bin/docker-image-tool.sh

@@ -181,8 +181,8 @@ Examples:
     $0 -r docker.io/myrepo -t v1.8.1 build
     $0 -r docker.io/myrepo -t v1.8.1 push
 
-  - Build and push with tag "v1.8.1" and Spark-3.5.5 as base image to docker.io/myrepo
-    $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.5 build
+  - Build and push with tag "v1.8.1" and Spark-3.5.7 as base image to docker.io/myrepo
+    $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.7 build
     $0 -r docker.io/myrepo -t v1.8.1 push
 
   - Build and push for multiple archs to docker.io/myrepo

docker/playground/.env

@@ -24,7 +24,7 @@ KYUUBI_HADOOP_VERSION=3.3.6
 POSTGRES_VERSION=12
 POSTGRES_JDBC_VERSION=42.3.4
 SCALA_BINARY_VERSION=2.12
-SPARK_VERSION=3.4.3
+SPARK_VERSION=3.4.4
 SPARK_BINARY_VERSION=3.4
 SPARK_HADOOP_VERSION=3.3.4
 ZOOKEEPER_VERSION=3.6.3

docs/deployment/kyuubi_on_kubernetes.md

@@ -42,8 +42,8 @@ Examples:
     $0 -r docker.io/myrepo -t v1.8.1 build
     $0 -r docker.io/myrepo -t v1.8.1 push
 
-  - Build and push with tag "v1.8.1" and Spark-3.5.5 as base image to docker.io/myrepo
-    $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.5 build
+  - Build and push with tag "v1.8.1" and Spark-3.5.7 as base image to docker.io/myrepo
+    $0 -r docker.io/myrepo -t v1.8.1 -b BASE_IMAGE=repo/spark:3.5.7 build
     $0 -r docker.io/myrepo -t v1.8.1 push
 
   - Build and push for multiple archs to docker.io/myrepo

docs/extensions/engines/spark/lineage.md

@@ -117,7 +117,7 @@ Sometimes, it may be incompatible with other Spark distributions, then you may n
 For example,
 
 ```shell
-build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -am -DskipTests -Dspark.version=3.5.5
+build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -am -DskipTests -Dspark.version=3.5.7
 ```
 
 The available `spark.version`s are shown in the following table.

extensions/spark/kyuubi-spark-connector-hive/src/main/scala/org/apache/kyuubi/spark/connector/hive/HiveConnectorUtils.scala

@@ -122,7 +122,7 @@ object HiveConnectorUtils extends Logging {
           isSplitable,
           maxSplitBytes,
           partitionValues)
-    }.recover { case _: Exception => // SPARK-51185: Spark 3.5.5
+    }.recover { case _: Exception => // SPARK-51185: Spark 3.5.7
       val fileStatusWithMetadataClz = DynClasses.builder()
         .impl("org.apache.spark.sql.execution.datasources.FileStatusWithMetadata")
         .buildChecked()

integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala

@@ -56,7 +56,7 @@ class KyuubiOnKubernetesWithSparkTestsBase extends WithKyuubiServerOnKubernetes
       Map(
         "spark.master" -> s"k8s://$miniKubeApiMaster",
         // We should update spark docker image in ./github/workflows/master.yml at the same time
-        "spark.kubernetes.container.image" -> "apache/spark:3.5.5",
+        "spark.kubernetes.container.image" -> "apache/spark:3.5.7",
         "spark.kubernetes.container.image.pullPolicy" -> "IfNotPresent",
         "spark.executor.memory" -> "512M",
         "spark.driver.memory" -> "1024M",

integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala

@@ -51,7 +51,7 @@ abstract class SparkOnKubernetesSuiteBase
     // TODO Support more Spark version
     // Spark official docker image: https://hub.docker.com/r/apache/spark/tags
     KyuubiConf().set("spark.master", s"k8s://$apiServerAddress")
-      .set("spark.kubernetes.container.image", "apache/spark:3.5.5")
+      .set("spark.kubernetes.container.image", "apache/spark:3.5.7")
       .set("spark.kubernetes.container.image.pullPolicy", "IfNotPresent")
       .set("spark.executor.instances", "1")
       .set("spark.executor.memory", "512M")

pom.xml

@@ -141,7 +141,7 @@
         <failsafe.verion>3.3.2</failsafe.verion>
         <fb303.version>0.9.3</fb303.version>
         <flexmark.version>0.62.2</flexmark.version>
-        <flink.version>1.20.0</flink.version>
+        <flink.version>1.20.3</flink.version>
         <flink.archive.name>flink-${flink.version}-bin-scala_2.12.tgz</flink.archive.name>
         <flink.archive.mirror>${apache.archive.dist}/flink/flink-${flink.version}</flink.archive.mirror>
         <flink.archive.query>?action=download</flink.archive.query>
@@ -207,7 +207,7 @@
           DO NOT forget to change the following properties when change the minor version of Spark:
           `delta.version`, `delta.artifact`, `maven.plugin.scalatest.exclude.tags`
           -->
-        <spark.version>3.5.5</spark.version>
+        <spark.version>3.5.7</spark.version>
         <spark.binary.version>3.5</spark.binary.version>
         <spark.archive.scala.suffix></spark.archive.scala.suffix>
         <spark.archive.name>spark-${spark.version}-bin-hadoop3${spark.archive.scala.suffix}.tgz</spark.archive.name>
@@ -2012,7 +2012,7 @@
                 <module>extensions/spark/kyuubi-spark-connector-hive</module>
             </modules>
             <properties>
-                <spark.version>3.4.3</spark.version>
+                <spark.version>3.4.4</spark.version>
                 <spark.binary.version>3.4</spark.binary.version>
                 <delta.version>2.4.0</delta.version>
                 <delta.artifact>delta-core_${scala.binary.version}</delta.artifact>
@@ -2027,7 +2027,7 @@
                 <module>extensions/spark/kyuubi-spark-connector-hive</module>
             </modules>
             <properties>
-                <spark.version>3.5.5</spark.version>
+                <spark.version>3.5.7</spark.version>
                 <spark.binary.version>3.5</spark.binary.version>
                 <delta.version>3.3.1</delta.version>
                 <delta.artifact>delta-spark_${scala.binary.version}</delta.artifact>
@@ -2125,14 +2125,14 @@
         <profile>
             <id>flink-1.19</id>
             <properties>
-                <flink.version>1.19.1</flink.version>
+                <flink.version>1.19.3</flink.version>
             </properties>
         </profile>
 
         <profile>
             <id>flink-1.20</id>
             <properties>
-                <flink.version>1.20.0</flink.version>
+                <flink.version>1.20.3</flink.version>
             </properties>
         </profile>