Added Spark Distribution in Docker Image. Optimized debian base image.

Author: cgiraldo

Dockerfile

@@ -1,24 +1,61 @@
-FROM centos:7.6.1810
+FROM python:3.7-buster as dependencies
+COPY api/src/main/resources/requirements.txt /
+RUN apt-get update && \
+    apt-get install -y libssl-dev libsasl2-dev libffi-dev cython3
+RUN pip3 install -r requirements.txt
 
-RUN yum install -y epel-release
-RUN yum install -y python36-devel python36-pip cyrus-sasl-devel cyrus-sasl-gssapi cyrus-sasl-plain libffi-devel openssl-devel gcc gcc-c++
 
-COPY api/src/main/resources /deployment-manager/
+FROM python:3.7-slim-buster
 
-WORKDIR /deployment-manager
+LABEL maintainer="[email protected]" \
+      organization="gradiant.org"
 
-RUN pip3 install -r requirements.txt
+ENV VERSION=2.1.2
+# Create app directory
+WORKDIR /deployment-manager
+# Adding python runtime dependencies
+COPY --from=dependencies /requirements.txt ./
+COPY --from=dependencies /root/.cache /root/.cache
+## Adding OS runtime dependencies
+RUN apt-get update && \
+    apt-get install -y libssl1.1 libsasl2-2 libsasl2-modules-gssapi-mit  libffi6 && \
+    rm -rf /var/lib/apt/lists/* && \
+    pip3 install -r requirements.txt
 
 # GIT http server to serve notebooks to jupyterhub
-RUN yum install -y git nginx fcgiwrap && \
+RUN apt-get update && \ 
+    apt-get install -y git nginx fcgiwrap && \
+    rm -rf /var/lib/apt/lists/* && \
     git config --global user.email "[email protected]" && \
     git config --global user.name "pnda" && \
     mkdir -p /data/git-repos/ && \
-    mkdir -p /data/stage/ 
+    mkdir -p /data/stage/
+
+# Spark distribution to submit spark jobs.
+# Installed openjdk-8 since spark 2.4.4 does not yet support Java 11
+# need unstable repo from debian buster
+ENV JAVA_HOME=/usr/lib/jvm/default-jvm/ \
+    SPARK_VERSION=2.4.4 \
+    SPARK_HOME=/opt/spark
+ENV PATH="$PATH:$SPARK_HOME/sbin:$SPARK_HOME/bin" \
+    SPARK_URL="local[*]" \
+    PYTHONPATH="${SPARK_HOME}/python/lib/pyspark.zip:${SPARK_HOME}/python/lib/py4j-src.zip:$PYTHONPATH" \
+    SPARK_OPTS="" \
+    PYSPARK_PYTHON=/usr/bin/python3 
+RUN mkdir -p /usr/share/man/man1 && \
+    echo "deb http://deb.debian.org/debian unstable main" > /etc/apt/sources.list.d/91-unstable.list && \
+    apt-get update && apt-get install -y openjdk-8-jre-headless wget && rm /etc/apt/sources.list.d/91-unstable.list && rm -rf /var/lib/apt/lists/* && \
+    cd /usr/lib/jvm && ln -s java-8-openjdk-amd64 default-jvm && \
+    wget -qO- https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop2.7.tgz | tar xvz -C /opt && \
+    ln -s /opt/spark-$SPARK_VERSION-bin-hadoop2.7 /opt/spark && \
+    cd /opt/spark/python/lib && ln -s py4j-*-src.zip py4j-src.zip
+
+# Bundle app source
+COPY api/src/main/resources ./
 COPY docker/nginx.conf /etc/nginx/nginx.conf
-COPY docker/entrypoint.sh /entrypoint.sh
+COPY docker/entrypoint.sh .
+
 # PNDA platform users must transition from Linux SO users to cloud-native. For now we add a pnda user to container images.
 RUN useradd pnda
 
-ENTRYPOINT "/entrypoint.sh"
-
+ENTRYPOINT ["./entrypoint.sh"]

api/src/main/resources/application_summary_registrar.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import happybase
-from thriftpy.transport import TTransportException
+from thriftpy2.transport import TTransportException
 from Hbase_thrift import AlreadyExists
 
 #pylint: disable=E0602

api/src/main/resources/requirements.txt

@@ -1,9 +1,9 @@
 argparse==1.2.1
 certifi==2017.4.17
 cffi==1.9.1
-cryptography==1.7.1
+cryptography==2.8
 enum34==1.1.6
-happybase==1.0.0
+happybase==1.2.0
 idna==2.1
 ipaddress==1.0.17
 ordereddict==1.1
@@ -14,13 +14,13 @@ pycparser==2.17
 pyhs2==0.6.0
 pywebhdfs==0.4.1
 pyyaml==4.2b1
-requests==2.20.0
+requests==2.22.0
 sasl==0.2.1
 singledispatch==3.4.0.3
 six==1.10.0
 spur==0.3.20
 starbase==0.3.3
 thrift==0.9.3
-thriftpy==0.3.9
+thriftpy2==0.4.8
 tornado==4.4.2
 tornado-cors==0.6.0

docker/nginx.conf

@@ -43,7 +43,7 @@ server {
     location ~ (/.*) {
         fastcgi_pass  unix:/var/run/fcgiwrap.socket;
         include       fastcgi_params;
-        fastcgi_param SCRIPT_FILENAME /usr/libexec/git-core/git-http-backend;
+        fastcgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend;
         fastcgi_param GIT_HTTP_EXPORT_ALL "";
         fastcgi_param GIT_PROJECT_ROOT    /data/git-repos;
         fastcgi_param REMOTE_USER         $remote_user;