chore(hadoop): Make the base url switchable

Author: NickLarsenNZ

hadoop/upload_new_hadoop_version.sh

@@ -5,6 +5,11 @@ set -euo pipefail
 VERSION=${1:?"Missing version number argument (arg 1)"}
 NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"}
 
+# We prefer fast downloads...
+BASE_DOWNLOAD_URL="https://dlcdn.apache.org/hadoop/common"
+# However, if the version is not available, use the slow archive instead:
+# BASE_DOWNLOAD_URL="https://archive.apache.org/dist/hadoop/common"
+
 read -r -s -p "Nexus Password: " NEXUS_PASSWORD
 echo ""
 
@@ -34,14 +39,14 @@ cd "$WORK_DIR" || exit
 bin_file=hadoop-$VERSION.tar.gz
 src_file=hadoop-$VERSION-src.tar.gz
 
-echo "Downloading Hadoop"
-curl --fail -LO --progress-bar "https://dlcdn.apache.org/hadoop/common/hadoop-$VERSION/$bin_file"
-curl --fail -LO --progress-bar "https://dlcdn.apache.org/hadoop/common/hadoop-$VERSION/$bin_file.asc"
-curl --fail -LO --progress-bar "https://dlcdn.apache.org/hadoop/common/hadoop-$VERSION/$bin_file.sha512"
+echo "Downloading Hadoop (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)"
+curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$bin_file"
+curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$bin_file.asc"
+curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$bin_file.sha512"
 
-curl --fail -LO --progress-bar "https://dlcdn.apache.org/hadoop/common/hadoop-$VERSION/$src_file"
-curl --fail -LO --progress-bar "https://dlcdn.apache.org/hadoop/common/hadoop-$VERSION/$src_file.asc"
-curl --fail -LO --progress-bar "https://dlcdn.apache.org/hadoop/common/hadoop-$VERSION/$src_file.sha512"
+curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$src_file"
+curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$src_file.asc"
+curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$src_file.sha512"
 
 
 # It is probably redundant to check both the checksum and the signature but it's cheap and why not
@@ -53,7 +58,7 @@ if ! (sha512sum --tag "$bin_file" | diff - "$bin_file.sha512" && sha512sum --tag
 fi
 
 echo "Validating signatures"
-echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://dlcdn.apache.org/hadoop/common/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://dlcdn.apache.org/hadoop/common/KEYS | gpg --import")'
+echo "--> NOTE: Make sure you have downloaded and added the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\")"
 
 if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null && gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then
   echo "ERROR: One of the signatures could not be verified"