Update Spark to 3.0.1 version (Java 11, Python 3.9)

Author: luisbelloch

.envrc

@@ -1,5 +1,5 @@
-layout python3
-# use sdk java 8.0.232.hs-adpt
+layout pyenv 3.9.0
+use sdk java 11.0.9.hs-adpt
 export VBOXMANAGE=/Applications/VirtualBox.app/Contents/MacOS
 export SPARK_HOME=$(pwd)/.spark
 export PATH=$SPARK_HOME/bin:$VBOXMANAGE:$PATH

infra/docker/.envrc

@@ -0,0 +1,2 @@
+export DOCKER_BUILDKIT=1
+export COMPOSE_DOCKER_CLI_BUILD=1

infra/docker/Dockerfile

@@ -1,14 +1,14 @@
-FROM openjdk:8-jdk-slim
+FROM openjdk:11-jdk-slim
 LABEL maintainer="Luis Belloch <[email protected]>"
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
     apt-get install -y --no-install-recommends python3-software-properties python3-numpy curl && \
     rm -rf /var/lib/apt/lists/*
 
-ARG SPARK_VERSION=2.4.5
+ARG SPARK_VERSION=3.0.1
 ENV SPARK_HOME=/opt/spark
-RUN mkdir -p /opt/spark && curl -s http://apache.rediris.es/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz | tar -xz -C "${SPARK_HOME}" --strip-components=1
+RUN mkdir -p /opt/spark && curl -s https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz | tar -xz -C "${SPARK_HOME}" --strip-components=1
 ENV PATH=$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH
 
 RUN cp "${SPARK_HOME}/conf/log4j.properties.template" "${SPARK_HOME}/conf/log4j.properties" && \

infra/docker/Makefile

@@ -1,5 +1,5 @@
-SPARK_VERSION:=2.4.5
-COURSE_VERSION:=2020.1
+SPARK_VERSION:=3.0.1
+COURSE_VERSION:=2020.12
 IMAGE_NAME:=luisbelloch/spark
 
 .PHONY: help

local_setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -euo pipefail
-SPARK_URL=${SPARK_URL:-http://apache.rediris.es/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz}
+SPARK_URL=${SPARK_URL:-https://downloads.apache.org/spark/spark-3.0.1/spark-3.0.1-bin-hadoop2.7.tgz}
 SPARK_PKG=${SPARK_URL##*/}
 SPARK_HOME=${SPARK_HOME:-$(pwd)/.spark}
 

playbook.yml

@@ -2,8 +2,8 @@
 - hosts: all
   vars:
     spark_home: /opt/spark
-    spark_pkg_name: spark-2.4.5-bin-hadoop2.7
-    spark_pkg_url: http://apache.rediris.es/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz
+    spark_pkg_name: spark-3.0.1-bin-hadoop2.7
+    spark_pkg_url: https://downloads.apache.org/spark/spark-3.0.1/spark-3.0.1-bin-hadoop2.7.tgz
 
   tasks:
     - name: Update all packages to the latest version
@@ -20,7 +20,7 @@
         update_cache: yes
         force_apt_get: true
 
-    - name: Install AdoptOpenJDK 8
+    - name: Install AdoptOpenJDK 11
       become: true
       block:
         - name: Import keys
@@ -33,7 +33,7 @@
             state: present
         - name: Install package
           apt:
-            name: ['adoptopenjdk-8-hotspot', 'ca-certificates']
+            name: ['adoptopenjdk-11-hotspot', 'ca-certificates']
             state: latest
             update_cache: yes
             force_apt_get: true