chore(spark): update images for 25.11.0 (#1280)

* spark: remove 3.5.5

* spark: add 3.5.7

* spark-connect-client: add 3.5.7 and remove unused example

* update changelog

* update changelog

Author: razvan

CHANGELOG.md

@@ -17,6 +17,8 @@ All notable changes to this project will be documented in this file.
 - kafka: Add version `4.1.0` ([#1277]).
 - druid: Add `34.0.0` ([#1263]).
 - opa: Add `1.8.0` ([#1279]).
+- spark: Add `3.5.7` ([#1280]).
+- spark-connect-client: Add `3.5.7` ([#1280]).
 
 ### Changed
 
@@ -36,6 +38,8 @@ All notable changes to this project will be documented in this file.
 - kafka: Remove version `4.0.0` ([#1277]).
 - druid: Remove `31.0.1` ([#1263]).
 - opa: Remove `1.0.1` ([#1279]).
+- spark: Remove `3.5.5` ([#1280]).
+- spark-connect-client: Remove unused example script from the image ([#1280]).
 
 [#1207]: https://github.com/stackabletech/docker-images/pull/1207
 [#1215]: https://github.com/stackabletech/docker-images/pull/1215

spark-connect-client/Dockerfile

@@ -8,18 +8,17 @@ ARG PYTHON_VERSION
 ARG RELEASE_VERSION
 ARG STACKABLE_USER_UID
 
-LABEL name="Stackable Spark Connect Examples" \
+LABEL name="Stackable Spark Connect Client" \
     maintainer="[email protected]" \
     vendor="Stackable GmbH" \
     version="${PRODUCT_VERSION}" \
     release="${RELEASE_VERSION}" \
     summary="Spark Connect Examples" \
-    description="Spark Connect client libraries for Python and the JVM, including some examples."
+    description="Spark Connect client libraries for Python and the JVM."
 
 # Need root to install setuptools
 USER root
 
-COPY --chown=${STACKABLE_USER_UID}:0 spark-connect-client/stackable/spark-connect-examples /stackable/spark-connect-examples
 COPY --chown=${STACKABLE_USER_UID}:0 spark-connect-client/stackable/.jupyter /stackable/.jupyter
 
 RUN <<EOF
@@ -48,4 +47,4 @@ RUN pip install --no-cache-dir --user \
   "scikit-learn==1.3.1" \
   "matplotlib==3.10.1"
 
-WORKDIR /stackable/spark-connect-examples/python
+WORKDIR /stackable

spark-connect-client/boil-config.toml

@@ -5,6 +5,13 @@ java-base = "17"
 [versions."3.5.6".build-arguments]
 python-version = "3.11"
 
+[versions."3.5.7".local-images]
+spark-k8s = "3.5.7"
+java-base = "17"
+
+[versions."3.5.7".build-arguments]
+python-version = "3.11"
+
 [versions."4.0.0".local-images]
 spark-k8s = "4.0.0"
 java-base = "17"

spark-connect-client/stackable/spark-connect-examples/python/simple-connect-app.py

@@ -197,10 +197,8 @@ RUN <<EOF
     sed -i "s/${NEW_VERSION}/${ORIGINAL_VERSION}/g" assembly/target/bom.json
 EOF
 
-
-# As of version 3.5.5, spark-connect jars are not included in the dist folder.
 # To avoid classpath conflicts with existing spark applications,
-# we create a new dist/connect folder, and copy them here.
+# we create a new dist/connect folder, and copy spark connect jars here.
 RUN <<EOF
 
     SCALA_BINARY_VERSION=$(grep "scala.binary.version" pom.xml | head -n1 | awk -F '[<>]' '{print $3}')

spark-k8s/Dockerfile

@@ -1,11 +1,11 @@
-[versions."3.5.5".local-images]
+[versions."3.5.6".local-images]
 "hadoop/hadoop" = "3.4.1" # Current Stackable LTS version. Source of the AWS and Azure artifacts to Spark's classpath.
 java-base = "17"
 java-devel = "17"
 hbase = "2.6.2" # Current Stackable LTS version. Used to build the HBase connector.
 vector = "0.49.0"
 
-[versions."3.5.5".build-arguments]
+[versions."3.5.6".build-arguments]
 python-version = "3.11"
 aws-java-sdk-bundle-version = "2.24.6" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.1
 azure-storage-version = "7.0.1" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.1
@@ -17,14 +17,14 @@ jmx-exporter-version = "1.3.0"
 tini-version = "0.19.0"
 hbase-connector-version = "1.0.1"
 
-[versions."3.5.6".local-images]
+[versions."3.5.7".local-images]
 "hadoop/hadoop" = "3.4.1" # Current Stackable LTS version. Source of the AWS and Azure artifacts to Spark's classpath.
 java-base = "17"
 java-devel = "17"
 hbase = "2.6.2" # Current Stackable LTS version. Used to build the HBase connector.
 vector = "0.49.0"
 
-[versions."3.5.6".build-arguments]
+[versions."3.5.7".build-arguments]
 python-version = "3.11"
 aws-java-sdk-bundle-version = "2.24.6" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.1
 azure-storage-version = "7.0.1" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.1

spark-k8s/boil-config.toml

@@ -1,4 +1,4 @@
-From 37ad7dcbb2a77267bbddafc4f3b3d99fddda0b28 Mon Sep 17 00:00:00 2001
+From 5e3c5568d4bf0fdd4feb19e559d606a55a0d2569 Mon Sep 17 00:00:00 2001
 From: Razvan-Daniel Mihai <[email protected]>
 Date: Tue, 11 Mar 2025 17:29:39 +0200
 Subject: Update CycloneDX plugin
@@ -21,7 +21,7 @@ index ef7c010e93..0f4c1c74e4 100755
 
  # Actually build the jar
 diff --git a/pom.xml b/pom.xml
-index 9b009c3a42..483dceb8ea 100644
+index a0e25ce4d8..b3755a04e7 100644
 --- a/pom.xml
 +++ b/pom.xml
 @@ -3534,7 +3534,12 @@

spark-k8s/stackable/patches/3.5.5/0001-Update-CycloneDX-plugin.patch renamed to spark-k8s/stackable/patches/3.5.7/0001-Update-CycloneDX-plugin.patch

@@ -1,6 +1,6 @@
-From c4a23f0060f34a2e1c3b826b9698ad56a5ce7176 Mon Sep 17 00:00:00 2001
+From 0cfb1d6b733579ea91df842e685df9db5512ece8 Mon Sep 17 00:00:00 2001
 From: Razvan-Daniel Mihai <[email protected]>
-Date: Tue, 22 Jul 2025 18:51:28 +0200
+Date: Tue, 22 Jul 2025 17:34:03 +0200
 Subject: Backport [SPARK-51311][BUILD] Promote bcprov-jdk18on to compile scope
 
 ---
@@ -24,7 +24,7 @@ index 05645977a0..9834cf333f 100644
  org.slf4j:jul-to-slf4j
  org.slf4j:slf4j-api
 diff --git a/assembly/pom.xml b/assembly/pom.xml
-index 2066bbeb7e..05edd80958 100644
+index ffa6a0e2b0..19032d6054 100644
 --- a/assembly/pom.xml
 +++ b/assembly/pom.xml
 @@ -85,8 +85,22 @@
@@ -51,7 +51,7 @@ index 2066bbeb7e..05edd80958 100644
      <plugins>
        <plugin>
 diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
-index 4feea62dfe..df85dcb6f5 100644
+index f110a1988f..88324d3e27 100644
 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3
 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
 @@ -28,6 +28,7 @@ aws-java-sdk-bundle/1.12.262//aws-java-sdk-bundle-1.12.262.jar

spark-k8s/stackable/patches/3.5.5/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch renamed to spark-k8s/stackable/patches/3.5.7/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch

@@ -1,2 +1,2 @@
-base = "7c29c664cdc9321205a98a14858aaf8daaa19db2"
+base = "ed00d046951a7ecda6429accd3b9c5b2dc792b65"
 mirror = "https://github.com/stackabletech/spark.git"