Adapt spark k8s versions for release 24.7 (#771)

* bump spark versions

* fix comments

* adapted changelog

* bump 3.5.1 java version

* remove 4.0.0-preview1

* remove 4.0.0 commented section

---------

Co-authored-by: Razvan-Daniel Mihai <[email protected]>

Author: maltesander

CHANGELOG.md

@@ -32,8 +32,9 @@ All notable changes to this project will be documented in this file.
 - trino & trino-cli: Add version 451 ([#758]).
 - airflow: Add version `2.8.4` and `2.9.2` ([#762]).
 - superset: Add version `3.1.3` and `4.0.2` ([#768]).
-- druid: Support for 30.0.0 using Java 17 ([#731])
+- druid: Support for 30.0.0 using Java 17 ([#731]).
 - hbase: Support for HBase 2.4.18 ([#740]).
+- spark-k8s: Support for `3.5.1` using Java 17 ([#771]).
 - airflow & superset: Build `statsd_exporter` from source ([#777]).
 
 ### Changed
@@ -82,6 +83,7 @@ All notable changes to this project will be documented in this file.
 - airflow: Remove unsupprted version `2.7.2`, `2.7.3` and `2.8.3` ([#762]).
 - superset: Remove version `2.1.1`, `3.0.1` and `3.0.3` ([#768]).
 - druid: Remove support for 27.0.0 ([#731])
+- spark-k8s: Remove support for `3.4.1` and `3.5.0` ([#771]).
 
 [#583]: https://github.com/stackabletech/docker-images/pull/583
 [#611]: https://github.com/stackabletech/docker-images/pull/611
@@ -133,6 +135,7 @@ All notable changes to this project will be documented in this file.
 [#762]: https://github.com/stackabletech/docker-images/pull/762
 [#767]: https://github.com/stackabletech/docker-images/pull/767
 [#768]: https://github.com/stackabletech/docker-images/pull/768
+[#771]: https://github.com/stackabletech/docker-images/pull/771
 [#777]: https://github.com/stackabletech/docker-images/pull/777
 
 ## [24.3.0] - 2024-03-20

spark-k8s/Dockerfile

@@ -52,7 +52,6 @@ RUN curl -L --fail -O https://repo.stackable.tech/repository/packages/aws/hadoop
     && curl -L --fail -O https://repo.stackable.tech/repository/packages/aws/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE}.jar \
     && curl -L --fail -O https://repo.stackable.tech/repository/packages/azure/hadoop-azure-${HADOOP_LONG_VERSION}.jar \
     && curl -L --fail -O https://repo.stackable.tech/repository/packages/azure/azure-storage-${AZURE_STORAGE}.jar \
-    && curl -L --fail -O https://repo.stackable.tech/repository/packages/azure/azure-storage-${AZURE_STORAGE}.jar \
     && curl -L --fail -O https://repo.stackable.tech/repository/packages/azure/azure-keyvault-core-${AZURE_KEYVAULT_CORE}.jar
 
 WORKDIR /stackable/spark-${PRODUCT}/dist/extra-jars

spark-k8s/versions.py

@@ -1,57 +1,40 @@
 versions = [
     {
-        "product": "3.4.1",
+        "product": "3.4.2",
         "java-base": "11",
         "java-devel": "11",
         "python": "3.11",
         "hadoop_long_version": "3.3.4",  # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122
         "aws_java_sdk_bundle": "1.12.262",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4
         "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4
         "azure_keyvault_core": "1.0.0",  # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
-        "jackson_dataformat_xml": "2.14.2",  # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.1
+        "jackson_dataformat_xml": "2.14.2",  # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.2
         "stax2_api": "4.2.1",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2
         "woodstox_core": "6.5.0",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2
         "vector": "0.39.0",
         "jmx_exporter": "1.0.1",
         "tini": "0.19.0",
     },
     {
-        "product": "3.4.2",
+        "product": "3.4.3",
         "java-base": "11",
         "java-devel": "11",
         "python": "3.11",
         "hadoop_long_version": "3.3.4",  # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122
         "aws_java_sdk_bundle": "1.12.262",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4
         "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4
         "azure_keyvault_core": "1.0.0",  # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
-        "jackson_dataformat_xml": "2.14.2",  # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.2
+        "jackson_dataformat_xml": "2.14.2",  # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.3
         "stax2_api": "4.2.1",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2
         "woodstox_core": "6.5.0",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2
         "vector": "0.39.0",
         "jmx_exporter": "1.0.1",
         "tini": "0.19.0",
-    },
-    {
-        "product": "3.5.0",
-        "java-base": "11",
-        "java-devel": "11",
-        "python": "3.11",
-        "hadoop_long_version": "3.3.4",  # https://github.com/apache/spark/blob/6a5747d66e53ed0d934cdd9ca5c9bd9fde6868e6/pom.xml#L125
-        "aws_java_sdk_bundle": "1.12.262",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4
-        "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4
-        "azure_keyvault_core": "1.0.0",  # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
-        "jackson_dataformat_xml": "2.15.2",  # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.0
-        "stax2_api": "4.2.1",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
-        "woodstox_core": "6.5.1",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
-        "vector": "0.39.0",
-        "jmx_exporter": "1.0.1",
-        "tini": "0.19.0",
-    },
+    },    
     {
         "product": "3.5.1",
-        "spark": "3.5.1",
-        "java-base": "11",
-        "java-devel": "11",
+        "java-base": "17",
+        "java-devel": "17",
         "python": "3.11",
         "hadoop_long_version": "3.3.4",  # https://github.com/apache/spark/blob/6a5747d66e53ed0d934cdd9ca5c9bd9fde6868e6/pom.xml#L125
         "aws_java_sdk_bundle": "1.12.262",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4