feat(spark): Add support for 3.5.1 (#588)

* feat(spark): Add support for 3.5.1

* changelog

* document commend to load Apache PGP keys

* fix links

* fix links

* changelog

Author: sbernauer

CHANGELOG.md

@@ -35,6 +35,7 @@ All notable changes to this project will be documented in this file.
 - hadoop: Add hdfs-utils ([#566]).
 - testing-tools: add pytest `8.0.1` ([#575]).
 - trino-cli: Command line for Trino version 440 ([#587])
+- spark: Add version `3.5.1` ([#588]).
 
 ### Changed
 
@@ -107,6 +108,7 @@ All notable changes to this project will be documented in this file.
 [#579]: https://github.com/stackabletech/docker-images/pull/579
 [#585]: https://github.com/stackabletech/docker-images/pull/585
 [#587]: https://github.com/stackabletech/docker-images/pull/587
+[#588]: https://github.com/stackabletech/docker-images/pull/588
 
 ## [23.11.0] - 2023-11-30
 

conf.py

@@ -298,6 +298,23 @@
                 "jmx_exporter": "0.20.0",
                 "tini": "0.19.0",
             },
+            {
+                "product": "3.5.1",
+                "spark": "3.5.1",
+                "java-base": "11",
+                "python": "3.11",
+                "hadoop_short_version": "3",
+                "hadoop_long_version": "3.3.4",  # https://github.com/apache/spark/blob/6a5747d66e53ed0d934cdd9ca5c9bd9fde6868e6/pom.xml#L125
+                "aws_java_sdk_bundle": "1.12.262",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4
+                "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4
+                "azure_keyvault_core": "1.0.0",  # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1
+                "jackson_dataformat_xml": "2.15.2",  # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.1
+                "stax2_api": "4.2.1",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
+                "woodstox_core": "6.5.1",  # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2
+                "vector": "0.35.0",
+                "jmx_exporter": "0.20.0",
+                "tini": "0.19.0",
+            },
         ],
     },
     {

druid/upload_new_druid_version.sh

@@ -44,7 +44,7 @@ if ! (sha512sum "${bin_file}" | cut -d " " -f 1 | diff -Z - "${bin_file}.sha512"
 fi
 
 echo "Validating signature"
-echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://www.apache.org/dist/druid/KEYS) to GPG: https://www.apache.org/info/verification.html'
+echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://www.apache.org/dist/druid/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/spark/KEYS | gpg --import")'
 
 if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null); then
   echo "ERROR: The signature could not be verified"

hadoop/upload_new_hadoop_version.sh

@@ -51,7 +51,7 @@ if ! (sha512sum --tag "$bin_file" | diff - "$bin_file.sha512" && sha512sum --tag
 fi
 
 echo "Validating signatures"
-echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/hadoop/common/KEYS) to GPG: https://www.apache.org/info/verification.html'
+echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/hadoop/common/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/hadoop/common/KEYS | gpg --import")'
 
 if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null && gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then
   echo "ERROR: One of the signatures could not be verified"

hbase/upload_new_hbase-operator-tools_version.sh

@@ -47,7 +47,7 @@ if ! (gpg --print-md SHA512 "$src_file" | diff - "$src_file.sha512"); then
 fi
 
 echo "Validating signatures"
-echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/hbase/KEYS) to GPG: https://www.apache.org/info/verification.html'
+echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/hbase/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://downloads.apache.org/hbase/KEYS | gpg --import")'
 if ! (gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then
   echo "ERROR: One of the signatures could not be verified"
   exit 1

hbase/upload_new_hbase_version.sh

@@ -53,7 +53,7 @@ if ! (gpg --print-md SHA512 "$bin_file" | diff - "$bin_file.sha512" && gpg --pri
 fi
 
 echo "Validating signatures"
-echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/hbase/KEYS) to GPG: https://www.apache.org/info/verification.html'
+echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/hbase/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://downloads.apache.org/hbase/KEYS | gpg --import")'
 if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null && gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then
   echo "ERROR: One of the signatures could not be verified"
   exit 1

hive/upload_new_hive_version.sh

@@ -44,7 +44,7 @@ if ! (sha256sum "${bin_file}" | diff - "${bin_file}.sha256"); then
 fi
 
 echo "Validating signatures"
-echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://dlcdn.apache.org/hive/KEYS) to GPG: https://www.apache.org/info/verification.html'
+echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://dlcdn.apache.org/hive/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://dlcdn.apache.org/hive/KEYS | gpg --import")'
 
 if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null); then
   echo "ERROR: Signature could not be verified"

hive/upload_new_postgres_driver_version.sh

@@ -55,7 +55,7 @@ if ! (sha1sum -c ${bin_file}.sha1 && sha1sum -c ${src_file}.sha1); then
 fi
 
 echo "Validating signatures"
-echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://raw.githubusercontent.com/pgjdbc/pgjdbc/master/KEYS) to GPG: https://www.apache.org/info/verification.html'
+echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://raw.githubusercontent.com/pgjdbc/pgjdbc/master/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://raw.githubusercontent.com/pgjdbc/pgjdbc/master/KEYS | gpg --import")'
 
  if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null && gpg --verify "${src_file}.asc" "${src_file}" 2> /dev/null); then
   echo "ERROR: One of the signatures could not be verified"

kafka/upload_new_kafka_version.sh

@@ -44,7 +44,7 @@ if ! (gpg --print-md SHA512 "${bin_file}" | diff - "${bin_file}.sha512" ); then
 fi
 
 echo "Validating signature"
-echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/kafka/KEYS) to GPG: https://www.apache.org/info/verification.html'
+echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://downloads.apache.org/kafka/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://downloads.apache.org/kafka/KEYS | gpg --import")'
 
 if ! (gpg --verify "${bin_file}.asc" "${bin_file}" 2> /dev/null); then
   echo "ERROR: The signature could not be verified"

nifi/upload_new_nifi_version.sh

@@ -45,7 +45,7 @@ if ! (sha512sum "$bin_file" | cut -d ' ' -f 1 | diff - "$bin_file.sha512"); then
 fi
 
 echo "Validating signatures"
-echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/nifi/common/KEYS) to GPG: https://www.apache.org/info/verification.html'
+echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/nifi/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/nifi/KEYS | gpg --import")'
 
 if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null); then
   echo "ERROR: One of the signatures could not be verified"