diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ea48652d..8ff1385dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ All notable changes to this project will be documented in this file. - kafka: Add version `4.1.0` ([#1277]). - druid: Add `34.0.0` ([#1263]). - opa: Add `1.8.0` ([#1279]). +- spark: Add `3.5.7` ([#1280]). +- spark-connect-client: Add `3.5.7` ([#1280]). ### Changed @@ -36,6 +38,8 @@ All notable changes to this project will be documented in this file. - kafka: Remove version `4.0.0` ([#1277]). - druid: Remove `31.0.1` ([#1263]). - opa: Remove `1.0.1` ([#1279]). +- spark: Remove `3.5.5` ([#1280]). +- spark-connect-client: Remove unused example script from the image ([#1280]). [#1207]: https://github.com/stackabletech/docker-images/pull/1207 [#1215]: https://github.com/stackabletech/docker-images/pull/1215 diff --git a/spark-connect-client/Dockerfile b/spark-connect-client/Dockerfile index fd2c9e950..b7a017eda 100644 --- a/spark-connect-client/Dockerfile +++ b/spark-connect-client/Dockerfile @@ -8,18 +8,17 @@ ARG PYTHON_VERSION ARG RELEASE_VERSION ARG STACKABLE_USER_UID -LABEL name="Stackable Spark Connect Examples" \ +LABEL name="Stackable Spark Connect Client" \ maintainer="info@stackable.tech" \ vendor="Stackable GmbH" \ version="${PRODUCT_VERSION}" \ release="${RELEASE_VERSION}" \ summary="Spark Connect Examples" \ - description="Spark Connect client libraries for Python and the JVM, including some examples." + description="Spark Connect client libraries for Python and the JVM." # Need root to install setuptools USER root -COPY --chown=${STACKABLE_USER_UID}:0 spark-connect-client/stackable/spark-connect-examples /stackable/spark-connect-examples COPY --chown=${STACKABLE_USER_UID}:0 spark-connect-client/stackable/.jupyter /stackable/.jupyter RUN <]' '{print $3}') diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index c7de0ae7f..8b2cb468e 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -1,11 +1,11 @@ -[versions."3.5.5".local-images] +[versions."3.5.6".local-images] "hadoop/hadoop" = "3.4.1" # Current Stackable LTS version. Source of the AWS and Azure artifacts to Spark's classpath. java-base = "17" java-devel = "17" hbase = "2.6.2" # Current Stackable LTS version. Used to build the HBase connector. vector = "0.49.0" -[versions."3.5.5".build-arguments] +[versions."3.5.6".build-arguments] python-version = "3.11" aws-java-sdk-bundle-version = "2.24.6" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.1 azure-storage-version = "7.0.1" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.1 @@ -17,14 +17,14 @@ jmx-exporter-version = "1.3.0" tini-version = "0.19.0" hbase-connector-version = "1.0.1" -[versions."3.5.6".local-images] +[versions."3.5.7".local-images] "hadoop/hadoop" = "3.4.1" # Current Stackable LTS version. Source of the AWS and Azure artifacts to Spark's classpath. java-base = "17" java-devel = "17" hbase = "2.6.2" # Current Stackable LTS version. Used to build the HBase connector. vector = "0.49.0" -[versions."3.5.6".build-arguments] +[versions."3.5.7".build-arguments] python-version = "3.11" aws-java-sdk-bundle-version = "2.24.6" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.1 azure-storage-version = "7.0.1" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.1 diff --git a/spark-k8s/stackable/patches/3.5.5/0001-Update-CycloneDX-plugin.patch b/spark-k8s/stackable/patches/3.5.7/0001-Update-CycloneDX-plugin.patch similarity index 92% rename from spark-k8s/stackable/patches/3.5.5/0001-Update-CycloneDX-plugin.patch rename to spark-k8s/stackable/patches/3.5.7/0001-Update-CycloneDX-plugin.patch index f1cf7df81..b3c2d89f7 100644 --- a/spark-k8s/stackable/patches/3.5.5/0001-Update-CycloneDX-plugin.patch +++ b/spark-k8s/stackable/patches/3.5.7/0001-Update-CycloneDX-plugin.patch @@ -1,4 +1,4 @@ -From 37ad7dcbb2a77267bbddafc4f3b3d99fddda0b28 Mon Sep 17 00:00:00 2001 +From 5e3c5568d4bf0fdd4feb19e559d606a55a0d2569 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 11 Mar 2025 17:29:39 +0200 Subject: Update CycloneDX plugin @@ -21,7 +21,7 @@ index ef7c010e93..0f4c1c74e4 100755 # Actually build the jar diff --git a/pom.xml b/pom.xml -index 9b009c3a42..483dceb8ea 100644 +index a0e25ce4d8..b3755a04e7 100644 --- a/pom.xml +++ b/pom.xml @@ -3534,7 +3534,12 @@ diff --git a/spark-k8s/stackable/patches/3.5.5/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch b/spark-k8s/stackable/patches/3.5.7/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch similarity index 95% rename from spark-k8s/stackable/patches/3.5.5/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch rename to spark-k8s/stackable/patches/3.5.7/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch index 08e9415b6..4f5dbc208 100644 --- a/spark-k8s/stackable/patches/3.5.5/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch +++ b/spark-k8s/stackable/patches/3.5.7/0002-Backport-SPARK-51311-BUILD-Promote-bcprov-jdk18on-to.patch @@ -1,6 +1,6 @@ -From c4a23f0060f34a2e1c3b826b9698ad56a5ce7176 Mon Sep 17 00:00:00 2001 +From 0cfb1d6b733579ea91df842e685df9db5512ece8 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> -Date: Tue, 22 Jul 2025 18:51:28 +0200 +Date: Tue, 22 Jul 2025 17:34:03 +0200 Subject: Backport [SPARK-51311][BUILD] Promote bcprov-jdk18on to compile scope --- @@ -24,7 +24,7 @@ index 05645977a0..9834cf333f 100644 org.slf4j:jul-to-slf4j org.slf4j:slf4j-api diff --git a/assembly/pom.xml b/assembly/pom.xml -index 2066bbeb7e..05edd80958 100644 +index ffa6a0e2b0..19032d6054 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -85,8 +85,22 @@ @@ -51,7 +51,7 @@ index 2066bbeb7e..05edd80958 100644 diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 -index 4feea62dfe..df85dcb6f5 100644 +index f110a1988f..88324d3e27 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -28,6 +28,7 @@ aws-java-sdk-bundle/1.12.262//aws-java-sdk-bundle-1.12.262.jar diff --git a/spark-k8s/stackable/patches/3.5.5/patchable.toml b/spark-k8s/stackable/patches/3.5.7/patchable.toml similarity index 51% rename from spark-k8s/stackable/patches/3.5.5/patchable.toml rename to spark-k8s/stackable/patches/3.5.7/patchable.toml index fbb635ebc..49202d309 100644 --- a/spark-k8s/stackable/patches/3.5.5/patchable.toml +++ b/spark-k8s/stackable/patches/3.5.7/patchable.toml @@ -1,2 +1,2 @@ -base = "7c29c664cdc9321205a98a14858aaf8daaa19db2" +base = "ed00d046951a7ecda6429accd3b9c5b2dc792b65" mirror = "https://github.com/stackabletech/spark.git"