diff --git a/CHANGELOG.md b/CHANGELOG.md index e28ed6dab..90795d3fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ All notable changes to this project will be documented in this file. - hive: Add version `4.0.0` ([#818]). - trino: Add version `455` ([#822]). - trino-cli: Add version `455` ([#822]). +- spark: Add version `3.5.2` ([#848]). ### Changed @@ -30,6 +31,7 @@ All notable changes to this project will be documented in this file. - trino-cli: Remove version `451` ([#822]). - hbase: Remove 2.4.17 ([#846]). - omid: Remove 1.1.0 and 1.1.1 ([#846]). +- spark: Remove 3.4.2 and 3.4.3 ([#848]). ### Fixed @@ -45,6 +47,7 @@ All notable changes to this project will be documented in this file. [#819]: https://github.com/stackabletech/docker-images/pull/819 [#822]: https://github.com/stackabletech/docker-images/pull/822 [#846]: https://github.com/stackabletech/docker-images/pull/846 +[#848]: https://github.com/stackabletech/docker-images/pull/848 ## [24.7.0] - 2024-07-24 diff --git a/spark-k8s/stackable/patches/3.4.2/001-cyclonedx-plugin.patch b/spark-k8s/stackable/patches/3.4.2/001-cyclonedx-plugin.patch deleted file mode 100644 index 8407d2ce2..000000000 --- a/spark-k8s/stackable/patches/3.4.2/001-cyclonedx-plugin.patch +++ /dev/null @@ -1,29 +0,0 @@ -diff --git a/pom.xml b/pom.xml -index fab9834..be3cb03 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -3451,6 +3451,24 @@ - - - -+ -+ org.cyclonedx -+ cyclonedx-maven-plugin -+ 2.8.0 -+ -+ application -+ 1.5 -+ false -+ -+ -+ -+ package -+ -+ makeBom -+ -+ -+ -+ - - - diff --git a/spark-k8s/stackable/patches/3.4.3/001-cyclonedx-plugin.patch b/spark-k8s/stackable/patches/3.4.3/001-cyclonedx-plugin.patch deleted file mode 100644 index 6bc1b8786..000000000 --- a/spark-k8s/stackable/patches/3.4.3/001-cyclonedx-plugin.patch +++ /dev/null @@ -1,29 +0,0 @@ -diff --git a/pom.xml b/pom.xml -index d354488..d5a58b0 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -3469,6 +3469,24 @@ - - - -+ -+ org.cyclonedx -+ cyclonedx-maven-plugin -+ 2.8.0 -+ -+ application -+ 1.5 -+ false -+ -+ -+ -+ package -+ -+ makeBom -+ -+ -+ -+ - - - diff --git a/spark-k8s/stackable/patches/3.5.2/001-cyclonedx-plugin.patch b/spark-k8s/stackable/patches/3.5.2/001-cyclonedx-plugin.patch new file mode 100644 index 000000000..06989b142 --- /dev/null +++ b/spark-k8s/stackable/patches/3.5.2/001-cyclonedx-plugin.patch @@ -0,0 +1,30 @@ +diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh +index ef7c010..0f4c1c7 100755 +--- a/dev/make-distribution.sh ++++ b/dev/make-distribution.sh +@@ -171,7 +171,6 @@ BUILD_COMMAND=("$MVN" clean package \ + -Dmaven.javadoc.skip=true \ + -Dmaven.scaladoc.skip=true \ + -Dmaven.source.skip \ +- -Dcyclonedx.skip=true \ + $@) + + # Actually build the jar +diff --git a/pom.xml b/pom.xml +index 0f504db..6cca7db 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -3482,7 +3482,12 @@ + + org.cyclonedx + cyclonedx-maven-plugin +- 2.7.9 ++ 2.8.0 ++ ++ application ++ 1.5 ++ false ++ + + + package diff --git a/spark-k8s/versions.py b/spark-k8s/versions.py index 6e3d9472a..850860b24 100644 --- a/spark-k8s/versions.py +++ b/spark-k8s/versions.py @@ -1,38 +1,22 @@ versions = [ { - "product": "3.4.2", - "java-base": "11", - "java-devel": "11", - "python": "3.11", - "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122 - "aws_java_sdk_bundle": "1.12.262", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4 - "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4 - "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 - "jackson_dataformat_xml": "2.14.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.2 - "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 - "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 - "vector": "0.40.1", - "jmx_exporter": "1.0.1", - "tini": "0.19.0", - }, - { - "product": "3.4.3", - "java-base": "11", - "java-devel": "11", + "product": "3.5.1", + "java-base": "17", + "java-devel": "17", "python": "3.11", - "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122 + "hadoop_long_version": "3.3.4", # https://github.com/apache/spark/blob/6a5747d66e53ed0d934cdd9ca5c9bd9fde6868e6/pom.xml#L125 "aws_java_sdk_bundle": "1.12.262", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4 "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4 "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 - "jackson_dataformat_xml": "2.14.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.4.3 - "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 - "woodstox_core": "6.5.0", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2 + "jackson_dataformat_xml": "2.15.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.1 + "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 + "woodstox_core": "6.5.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 "vector": "0.40.1", "jmx_exporter": "1.0.1", "tini": "0.19.0", }, { - "product": "3.5.1", + "product": "3.5.2", "java-base": "17", "java-devel": "17", "python": "3.11",