Skip to content

Commit 31abe23

Browse files
metasimmetasim
authored
Merge pull request #392 from s22s/update-spark-and-polish-dockerfile
update spark version and polish dockerfile
2 parents 932d157 + d366bfe commit 31abe23

File tree

3 files changed

+70
-60
lines changed

3 files changed

+70
-60
lines changed

‎build/circleci/Dockerfile‎

Lines changed: 29 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -6,47 +6,40 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
66

77
# most of these libraries required for
88
# python-pip pandoc && pip install setuptools => required for pyrasterframes testing
9-
RUN sudo apt-get update && \
9+
RUN \
10+
sudo apt-get update && \
1011
sudo apt remove \
1112
python python-minimal python2.7 python2.7-minimal \
1213
libpython-stdlib libpython2.7 libpython2.7-minimal libpython2.7-stdlib \
13-
&& sudo apt-get install -y \
14-
pandoc \
15-
wget \
16-
gcc g++ build-essential \
17-
libreadline-gplv2-dev libncursesw5-dev \
18-
libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
19-
liblzma-dev \
20-
libcurl4-gnutls-dev \
21-
libproj-dev \
22-
libgeos-dev \
23-
libhdf4-alt-dev \
24-
bash-completion \
25-
cmake \
26-
imagemagick \
27-
libpng-dev \
28-
libffi-dev \
29-
&& sudo apt autoremove \
30-
&& sudo apt-get clean all
31-
# && sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1
32-
# todo s
14+
&& \
15+
sudo apt-get install -y \
16+
pandoc wget \
17+
gcc g++ build-essential bash-completion cmake imagemagick \
18+
libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
19+
liblzma-dev libcurl4-gnutls-dev libproj-dev libgeos-dev libhdf4-alt-dev libpng-dev libffi-dev \
20+
&& \
21+
sudo apt autoremove && \
22+
sudo apt-get clean all
3323

34-
RUN cd /tmp && \
35-
wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz && \
36-
tar xzf Python-3.7.4.tgz && \
37-
cd Python-3.7.4 && \
38-
./configure --with-ensurepip=install --prefix=/usr/local --enable-optimization && \
39-
make && \
40-
sudo make altinstall && \
41-
rm -rf Python-3.7.4*
24+
RUN \
25+
cd /tmp && \
26+
wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz && \
27+
tar xzf Python-3.7.4.tgz && \
28+
cd Python-3.7.4 && \
29+
./configure --with-ensurepip=install --prefix=/usr/local --enable-optimization && \
30+
make && \
31+
sudo make altinstall && \
32+
rm -rf Python-3.7.4*
4233

43-
RUN sudo ln -s /usr/local/bin/python3.7 /usr/local/bin/python && \
44-
sudo curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
45-
sudo python get-pip.py && \
46-
sudo pip3 install setuptools ipython==6.2.1
34+
RUN \
35+
sudo ln -s /usr/local/bin/python3.7 /usr/local/bin/python && \
36+
sudo curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
37+
sudo python get-pip.py && \
38+
sudo pip3 install setuptools ipython==6.2.1
4739

4840
# install OpenJPEG
49-
RUN cd /tmp && \
41+
RUN \
42+
cd /tmp && \
5043
wget https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz && \
5144
tar -xf v${OPENJPEG_VERSION}.tar.gz && \
5245
cd openjpeg-${OPENJPEG_VERSION}/ && \
@@ -58,7 +51,8 @@ RUN cd /tmp && \
5851
cd /tmp && rm -Rf v${OPENJPEG_VERSION}.tar.gz openjpeg*
5952

6053
# Compile and install GDAL with Java bindings
61-
RUN cd /tmp && \
54+
RUN \
55+
cd /tmp && \
6256
wget http://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz && \
6357
tar -xf gdal-${GDAL_VERSION}.tar.gz && \
6458
cd gdal-${GDAL_VERSION} && \

‎rf-notebook/src/main/docker/Dockerfile‎

Lines changed: 28 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,49 +1,52 @@
1-
FROM jupyter/scipy-notebook:latest
1+
# jupyter/scipy-notebook isn't semantically versioned.
2+
# We pick this arbitrary one from Sept 2019 because it's what latest was on Oct 17 2019.
3+
FROM jupyter/scipy-notebook:1386e2046833
24

3-
MAINTAINER Astraea, Inc.
5+
LABEL maintainer="Astraea, Inc. <[email protected]>"
46

57
EXPOSE 4040 4041 4042 4043 4044
68

7-
ENV RF_LIB_LOC=/usr/local/rasterframes \
8-
LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/conda/lib"
9-
109
USER root
1110

12-
RUN mkdir $RF_LIB_LOC
13-
14-
RUN apt-get -y update && \
11+
RUN \
12+
apt-get -y update && \
1513
apt-get install --no-install-recommends -y openjdk-8-jre-headless ca-certificates-java && \
1614
apt-get clean && \
1715
rm -rf /var/lib/apt/lists/*
1816

1917
# Spark dependencies
20-
ENV APACHE_SPARK_VERSION 2.3.4
18+
ENV APACHE_SPARK_VERSION 2.4.4
2119
ENV HADOOP_VERSION 2.7
20+
ENV APACHE_SPARK_CHECKSUM 2E3A5C853B9F28C7D4525C0ADCB0D971B73AD47D5CCE138C85335B9F53A6519540D3923CB0B5CEE41E386E49AE8A409A51AB7194BA11A254E037A848D0C4A9E5
21+
ENV APACHE_SPARK_FILENAME spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
22+
ENV APACHE_SPARK_REMOTE_PATH spark-${APACHE_SPARK_VERSION}/${APACHE_SPARK_FILENAME}
23+
24+
RUN \
25+
cd /tmp && \
26+
wget --quiet http://apache.mirrors.pair.com/spark/${APACHE_SPARK_REMOTE_PATH} && \
27+
echo "${APACHE_SPARK_CHECKSUM} *${APACHE_SPARK_FILENAME}" | sha512sum -c - && \
28+
tar xzf ${APACHE_SPARK_FILENAME} -C /usr/local --owner root --group root --no-same-owner && \
29+
rm ${APACHE_SPARK_FILENAME}
2230

23-
RUN cd /tmp && \
24-
wget -q http://apache.mirrors.pair.com/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz && \
25-
echo "9FBEFCE2739990FFEDE6968A9C2F3FE399430556163BFDABDF5737A8F9E52CD535489F5CA7D641039A87700F50BFD91A706CA47979EE51A3A18787A92E2D6D53 *spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" | sha512sum -c - && \
26-
tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz -C /usr/local --owner root --group root --no-same-owner && \
27-
rm spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
2831
RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark
2932

3033
# Spark config
3134
ENV SPARK_HOME /usr/local/spark
3235
ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip
3336
ENV SPARK_OPTS --driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info
3437

38+
COPY conda_cleanup.sh .
39+
RUN chmod u+x conda_cleanup.sh
40+
41+
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/conda/lib"
3542
# Sphinx (for Notebook->html) and pyarrow (from pyspark build)
36-
RUN conda install --quiet --yes pyarrow \
37-
anaconda sphinx nbsphinx shapely numpy folium geopandas geojsonio rasterio descartes \
38-
&& conda clean --all \
39-
&& rm -rf /home/$NB_USER/.local \
40-
&& find /opt/conda/ -type f,l -name '*.a' -delete \
41-
&& find /opt/conda/ -type f,l -name '*.pyc' -delete \
42-
&& find /opt/conda/ -type f,l -name '*.js.map' -delete \
43-
&& find /opt/conda/lib/python*/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete \
44-
&& rm -rf /opt/conda/pkgs \
45-
&& fix-permissions $CONDA_DIR \
46-
&& fix-permissions /home/$NB_USER
43+
RUN \
44+
conda install --quiet --yes pyarrow \
45+
anaconda sphinx nbsphinx shapely numpy folium geopandas geojsonio rasterio descartes && \
46+
./conda_cleanup.sh $NB_USER $CONDA_DIR
47+
48+
ENV RF_LIB_LOC=/usr/local/rasterframes
49+
RUN mkdir $RF_LIB_LOC
4750

4851
COPY *.whl $RF_LIB_LOC
4952
COPY jupyter_notebook_config.py $HOME/.jupyter

‎rf-notebook/src/main/docker/conda_cleanup.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
NB_USER=$1
4+
CONDA_DIR=$2
5+
conda clean --all --force-pkgs-dirs --yes && \
6+
rm -rf /home/$NB_USER/.local && \
7+
find /opt/conda/ -type f,l -name '*.a' -delete && \
8+
find /opt/conda/ -type f,l -name '*.pyc' -delete && \
9+
find /opt/conda/ -type f,l -name '*.js.map' -delete && \
10+
find /opt/conda/lib/python*/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete && \
11+
rm -rf /opt/conda/pkgs && \
12+
fix-permissions $CONDA_DIR && \
13+
fix-permissions /home/$NB_USER

0 commit comments

Comments
 (0)