From 0449c9d16c7be3b42e82a211e174728df60a1c5a Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 4 Jun 2025 16:03:05 +0200 Subject: [PATCH 1/6] chore: bump spark version to 3.5.6 --- .../3.5.6/0001-Update-CycloneDX-plugin.patch | 40 +++++++++++++++++++ .../stackable/patches/3.5.6/patchable.toml | 1 + spark-k8s/versions.py | 18 +++++++++ 3 files changed, 59 insertions(+) create mode 100644 spark-k8s/stackable/patches/3.5.6/0001-Update-CycloneDX-plugin.patch create mode 100644 spark-k8s/stackable/patches/3.5.6/patchable.toml diff --git a/spark-k8s/stackable/patches/3.5.6/0001-Update-CycloneDX-plugin.patch b/spark-k8s/stackable/patches/3.5.6/0001-Update-CycloneDX-plugin.patch new file mode 100644 index 000000000..9ad9eea3c --- /dev/null +++ b/spark-k8s/stackable/patches/3.5.6/0001-Update-CycloneDX-plugin.patch @@ -0,0 +1,40 @@ +From 94ccf32b4d0eb7c3191b4e5a646605e7386c39ff Mon Sep 17 00:00:00 2001 +From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> +Date: Tue, 11 Mar 2025 17:29:39 +0200 +Subject: Update CycloneDX plugin + +--- + dev/make-distribution.sh | 1 - + pom.xml | 7 ++++++- + 2 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh +index ef7c010e93..0f4c1c74e4 100755 +--- a/dev/make-distribution.sh ++++ b/dev/make-distribution.sh +@@ -171,7 +171,6 @@ BUILD_COMMAND=("$MVN" clean package \ + -Dmaven.javadoc.skip=true \ + -Dmaven.scaladoc.skip=true \ + -Dmaven.source.skip \ +- -Dcyclonedx.skip=true \ + $@) + + # Actually build the jar +diff --git a/pom.xml b/pom.xml +index 68e2c422a2..6216ebb08e 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -3534,7 +3534,12 @@ + + org.cyclonedx + cyclonedx-maven-plugin +- 2.7.9 ++ 2.8.0 ++ ++ application ++ 1.5 ++ false ++ + + + package diff --git a/spark-k8s/stackable/patches/3.5.6/patchable.toml b/spark-k8s/stackable/patches/3.5.6/patchable.toml new file mode 100644 index 000000000..633d26c88 --- /dev/null +++ b/spark-k8s/stackable/patches/3.5.6/patchable.toml @@ -0,0 +1 @@ +base = "303c18c74664f161b9b969ac343784c088b47593" diff --git a/spark-k8s/versions.py b/spark-k8s/versions.py index 851587858..52beccc23 100644 --- a/spark-k8s/versions.py +++ b/spark-k8s/versions.py @@ -17,4 +17,22 @@ "tini": "0.19.0", "hbase_connector": "1.0.1", }, + { + "product": "3.5.6", + "java-base": "17", + "java-devel": "17", + "python": "3.11", + "hadoop": "3.4.1", # Current Stackable LTS version. Source of the AWS and Azure artifacts to Spark's classpath. + "hbase": "2.6.2", # Current Stackable LTS version. Used to build the HBase connector. + "aws_java_sdk_bundle": "2.24.6", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.1 + "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.1 + "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 + "jackson_dataformat_xml": "2.15.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.2 + "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 + "woodstox_core": "6.5.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 + "vector": "0.46.1", + "jmx_exporter": "1.2.0", + "tini": "0.19.0", + "hbase_connector": "1.0.1", + }, ] From f760876c773d9140820e54b498f812ed7b190441 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 4 Jun 2025 16:13:57 +0200 Subject: [PATCH 2/6] chore: update spark-connect-client to 3.5.6 --- spark-connect-client/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark-connect-client/versions.py b/spark-connect-client/versions.py index c0d2a9351..2311806bc 100644 --- a/spark-connect-client/versions.py +++ b/spark-connect-client/versions.py @@ -1,6 +1,6 @@ versions = [ { - "product": "3.5.5", + "product": "3.5.6", "spark-k8s": "3.5.5", "java-base": "17", "python": "3.11", From 0f30f94d4e6450d91ca0ed6a61cfc08b4ba755dc Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 4 Jun 2025 16:16:47 +0200 Subject: [PATCH 3/6] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8661b02e3..1c8e55d14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,8 @@ All notable changes to this project will be documented in this file. - kafka: Add `4.0.0` ([#1117]) - Include `.tar.gz` snapshots of the product source code in container images ([#1126]) - airflow: OPA authorizer for Airflow 3.x ([#1127]) +- spark-k8s: Add `3.5.6` ([#1141]) +- spark-connect-client: Add `3.5.6` ([#1141]) ### Changed @@ -107,6 +109,7 @@ All notable changes to this project will be documented in this file. - druid: Remove `30.0.0` ([#1110]). - nifi: Remove `2.2.0` ([#1114]). - kafka: Remove `3.7.1` and `3.8.0` ([#1117]) +- spark-connect-client: Add `3.5.5` ([#1141]) [nifi-iceberg-bundle]: https://github.com/stackabletech/nifi-iceberg-bundle [#1025]: https://github.com/stackabletech/docker-images/pull/1025 @@ -160,6 +163,7 @@ All notable changes to this project will be documented in this file. [#1133]: https://github.com/stackabletech/docker-images/pull/1133 [#1137]: https://github.com/stackabletech/docker-images/pull/1137 [#1138]: https://github.com/stackabletech/docker-images/pull/1138 +[#1141]: https://github.com/stackabletech/docker-images/pull/1141 ## [25.3.0] - 2025-03-21 From d780c3eb1098ecb087503b401aa5c358c25fb175 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 4 Jun 2025 16:19:26 +0200 Subject: [PATCH 4/6] update pr no --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c8e55d14..0ddc9cbe1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,8 +48,8 @@ All notable changes to this project will be documented in this file. - kafka: Add `4.0.0` ([#1117]) - Include `.tar.gz` snapshots of the product source code in container images ([#1126]) - airflow: OPA authorizer for Airflow 3.x ([#1127]) -- spark-k8s: Add `3.5.6` ([#1141]) -- spark-connect-client: Add `3.5.6` ([#1141]) +- spark-k8s: Add `3.5.6` ([#1142]) +- spark-connect-client: Add `3.5.6` ([#1142]) ### Changed @@ -109,7 +109,7 @@ All notable changes to this project will be documented in this file. - druid: Remove `30.0.0` ([#1110]). - nifi: Remove `2.2.0` ([#1114]). - kafka: Remove `3.7.1` and `3.8.0` ([#1117]) -- spark-connect-client: Add `3.5.5` ([#1141]) +- spark-connect-client: Add `3.5.5` ([#1142]) [nifi-iceberg-bundle]: https://github.com/stackabletech/nifi-iceberg-bundle [#1025]: https://github.com/stackabletech/docker-images/pull/1025 @@ -163,7 +163,7 @@ All notable changes to this project will be documented in this file. [#1133]: https://github.com/stackabletech/docker-images/pull/1133 [#1137]: https://github.com/stackabletech/docker-images/pull/1137 [#1138]: https://github.com/stackabletech/docker-images/pull/1138 -[#1141]: https://github.com/stackabletech/docker-images/pull/1141 +[#1142]: https://github.com/stackabletech/docker-images/pull/1142 ## [25.3.0] - 2025-03-21 From e47ac6a1ee7b7b96e13b87b364330276851bb1df Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 4 Jun 2025 16:21:04 +0200 Subject: [PATCH 5/6] fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ddc9cbe1..43dd13e99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,7 +109,7 @@ All notable changes to this project will be documented in this file. - druid: Remove `30.0.0` ([#1110]). - nifi: Remove `2.2.0` ([#1114]). - kafka: Remove `3.7.1` and `3.8.0` ([#1117]) -- spark-connect-client: Add `3.5.5` ([#1142]) +- spark-connect-client: Remove `3.5.5` ([#1142]) [nifi-iceberg-bundle]: https://github.com/stackabletech/nifi-iceberg-bundle [#1025]: https://github.com/stackabletech/docker-images/pull/1025 From e8d1c3d15a9ebefae595c42b21613987d1cdaaf2 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 6 Jun 2025 11:42:50 +0200 Subject: [PATCH 6/6] fix md lint --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7db46dc9..6b71e8f23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,7 +52,6 @@ All notable changes to this project will be documented in this file. - spark-connect-client: Add `3.5.6` ([#1142]) - git-sync: Bump version to 4.4.1 ([#1151]). - ### Changed - airflow,superset: Use `uv` to build the product ([#1116]). @@ -171,7 +170,6 @@ All notable changes to this project will be documented in this file. [#1142]: https://github.com/stackabletech/docker-images/pull/1142 [#1151]: https://github.com/stackabletech/docker-images/pull/1151 - ## [25.3.0] - 2025-03-21 ### Added