update spark version and polish dockerfile

Signed-off-by: Phil Varner <[email protected]>

Author: Phil Varner

build/circleci/Dockerfile

@@ -6,47 +6,40 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
 
 # most of these libraries required for
 # python-pip pandoc && pip install setuptools => required for pyrasterframes testing
-RUN sudo apt-get update && \
+RUN \
+    sudo apt-get update && \
     sudo apt remove \
       python python-minimal python2.7 python2.7-minimal \
       libpython-stdlib libpython2.7 libpython2.7-minimal libpython2.7-stdlib \
-    && sudo apt-get install -y \
-      pandoc \
-      wget \
-      gcc g++ build-essential \
-      libreadline-gplv2-dev libncursesw5-dev \
-      libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
-      liblzma-dev \
-      libcurl4-gnutls-dev \
-      libproj-dev \
-      libgeos-dev \
-      libhdf4-alt-dev \
-      bash-completion \
-      cmake \
-      imagemagick \
-      libpng-dev \
-      libffi-dev \
-    && sudo apt autoremove \
-    && sudo apt-get clean all
-#    && sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-# todo s
+      && \
+    sudo apt-get install -y \
+      pandoc wget \
+      gcc g++ build-essential bash-completion cmake imagemagick \
+      libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
+      liblzma-dev libcurl4-gnutls-dev libproj-dev libgeos-dev libhdf4-alt-dev libpng-dev libffi-dev \
+      && \
+    sudo apt autoremove && \
+    sudo apt-get clean all
 
-RUN cd /tmp && \
-  wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz && \
-  tar xzf Python-3.7.4.tgz && \
-  cd Python-3.7.4 && \
-  ./configure  --with-ensurepip=install --prefix=/usr/local --enable-optimization && \
-  make && \
-  sudo make altinstall && \
-  rm -rf Python-3.7.4*
+RUN \
+    cd /tmp && \
+    wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz && \
+    tar xzf Python-3.7.4.tgz && \
+    cd Python-3.7.4 && \
+    ./configure  --with-ensurepip=install --prefix=/usr/local --enable-optimization && \
+    make && \
+    sudo make altinstall && \
+    rm -rf Python-3.7.4*
 
-RUN sudo ln -s /usr/local/bin/python3.7 /usr/local/bin/python && \
-  sudo curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
-  sudo python get-pip.py && \
-  sudo pip3 install setuptools ipython==6.2.1
+RUN \
+    sudo ln -s /usr/local/bin/python3.7 /usr/local/bin/python && \
+    sudo curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    sudo python get-pip.py && \
+    sudo pip3 install setuptools ipython==6.2.1
 
 # install OpenJPEG
-RUN cd /tmp && \
+RUN \
+    cd /tmp && \
     wget https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz && \
     tar -xf v${OPENJPEG_VERSION}.tar.gz && \
     cd openjpeg-${OPENJPEG_VERSION}/ && \
@@ -58,7 +51,8 @@ RUN cd /tmp && \
     cd /tmp && rm -Rf v${OPENJPEG_VERSION}.tar.gz openjpeg*
 
 # Compile and install GDAL with Java bindings
-RUN cd /tmp && \
+RUN \
+    cd /tmp && \
     wget http://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz && \
     tar -xf gdal-${GDAL_VERSION}.tar.gz && \
     cd gdal-${GDAL_VERSION} && \

rf-notebook/src/main/docker/Dockerfile

@@ -1,49 +1,52 @@
-FROM jupyter/scipy-notebook:latest
+# jupyter/scipy-notebook isn't semantically versioned.
+# We pick this arbitrary one from Sept 2019 because it's what latest was on Oct 17 2019.
+FROM jupyter/scipy-notebook:1386e2046833
 
-MAINTAINER Astraea, Inc.
+LABEL maintainer="Astraea, Inc. <[email protected]>"
 
 EXPOSE 4040 4041 4042 4043 4044
 
-ENV RF_LIB_LOC=/usr/local/rasterframes \
-    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/conda/lib"
-
 USER root
 
-RUN mkdir $RF_LIB_LOC
-
-RUN apt-get -y update && \
+RUN \
+    apt-get -y update && \
     apt-get install --no-install-recommends -y openjdk-8-jre-headless ca-certificates-java && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
 # Spark dependencies
-ENV APACHE_SPARK_VERSION 2.3.4
+ENV APACHE_SPARK_VERSION 2.4.4
 ENV HADOOP_VERSION 2.7
+ENV APACHE_SPARK_CHECKSUM 2E3A5C853B9F28C7D4525C0ADCB0D971B73AD47D5CCE138C85335B9F53A6519540D3923CB0B5CEE41E386E49AE8A409A51AB7194BA11A254E037A848D0C4A9E5
+ENV APACHE_SPARK_FILENAME spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
+ENV APACHE_SPARK_REMOTE_PATH spark-${APACHE_SPARK_VERSION}/${APACHE_SPARK_FILENAME}
+
+RUN \
+    cd /tmp && \
+    wget --quiet http://apache.mirrors.pair.com/spark/${APACHE_SPARK_REMOTE_PATH} && \
+    echo "${APACHE_SPARK_CHECKSUM} *${APACHE_SPARK_FILENAME}" | sha512sum -c - && \
+    tar xzf ${APACHE_SPARK_FILENAME} -C /usr/local --owner root --group root --no-same-owner && \
+    rm ${APACHE_SPARK_FILENAME}
 
-RUN cd /tmp && \
-        wget -q http://apache.mirrors.pair.com/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz && \
-        echo "9FBEFCE2739990FFEDE6968A9C2F3FE399430556163BFDABDF5737A8F9E52CD535489F5CA7D641039A87700F50BFD91A706CA47979EE51A3A18787A92E2D6D53 *spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" | sha512sum -c - && \
-        tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz -C /usr/local --owner root --group root --no-same-owner && \
-        rm spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
 RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark
 
 # Spark config
 ENV SPARK_HOME /usr/local/spark
 ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip
 ENV SPARK_OPTS --driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info
 
+COPY conda_cleanup.sh .
+RUN chmod u+x conda_cleanup.sh
+
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/conda/lib"
 # Sphinx (for Notebook->html) and pyarrow (from pyspark build)
-RUN conda install --quiet --yes pyarrow \
-    anaconda sphinx nbsphinx shapely numpy folium geopandas geojsonio rasterio descartes \
-    && conda clean --all \
-    && rm -rf /home/$NB_USER/.local \
-    && find /opt/conda/ -type f,l -name '*.a' -delete \
-    && find /opt/conda/ -type f,l -name '*.pyc' -delete \
-    && find /opt/conda/ -type f,l -name '*.js.map' -delete \
-    && find /opt/conda/lib/python*/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete \
-    && rm -rf /opt/conda/pkgs \
-    && fix-permissions $CONDA_DIR \
-    && fix-permissions /home/$NB_USER
+RUN \
+    conda install --quiet --yes pyarrow \
+    anaconda sphinx nbsphinx shapely numpy folium geopandas geojsonio rasterio descartes && \
+    ./conda_cleanup.sh $NB_USER $CONDA_DIR
+
+ENV RF_LIB_LOC=/usr/local/rasterframes
+RUN mkdir $RF_LIB_LOC
 
 COPY *.whl $RF_LIB_LOC
 COPY jupyter_notebook_config.py $HOME/.jupyter

rf-notebook/src/main/docker/conda_cleanup.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+NB_USER=$1
+CONDA_DIR=$2
+conda clean --all --force-pkgs-dirs --yes && \
+    rm -rf /home/$NB_USER/.local && \
+    find /opt/conda/ -type f,l -name '*.a' -delete && \
+    find /opt/conda/ -type f,l -name '*.pyc' -delete && \
+    find /opt/conda/ -type f,l -name '*.js.map' -delete && \
+    find /opt/conda/lib/python*/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete && \
+    rm -rf /opt/conda/pkgs && \
+    fix-permissions $CONDA_DIR && \
+    fix-permissions /home/$NB_USER