diff --git a/Dockerfile b/Dockerfile index e6d0f19..5a2be59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,24 +30,31 @@ RUN mkdir /var/apache-spark-binaries/ # binaries # apache livy -RUN wget http://mirror.23media.de/apache/incubator/livy/0.5.0-incubating/livy-0.5.0-incubating-bin.zip -O /tmp/livy.zip -RUN unzip /tmp/livy.zip -d /opt/ +RUN wget https://apache.mirrors.tworzy.net/incubator/livy/0.7.1-incubating/apache-livy-0.7.1-incubating-bin.zip -O /tmp/livy.zip +RUN unzip /tmp/livy.zip -d /opt/ && mv /opt/apache-livy-0.7.1-incubating-bin /opt/livy + # Logging dir -RUN mkdir /opt/livy-0.5.0-incubating-bin/logs +RUN mkdir /opt/livy/logs # apache spark -RUN wget https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz -O /tmp/spark-2.3.1-bin-hadoop2.7.tgz -RUN tar -xvzf /tmp/spark-2.3.1-bin-hadoop2.7.tgz -C /opt/ +#RUN wget https://archive.apache.org/dist/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz -O /tmp/spark-3.1.1-bin-hadoop3.2.tgz +RUN wget https://archive.apache.org/dist/spark/spark-2.4.8/spark-2.4.8-bin-hadoop2.7.tgz -O /tmp/spark-2.4.8-bin-hadoop2.7.tgz +#RUN tar -xvzf /tmp/spark-3.1.1-bin-hadoop3.2.tgz -C /opt/ && mv /opt/spark-3.1.1-bin-hadoop3.2 /opt/spark +RUN tar -xvzf /tmp/spark-2.4.8-bin-hadoop2.7.tgz -C /opt/ && mv /opt/spark-2.4.8-bin-hadoop2.7 /opt/spark # set Python3 as default RUN rm /usr/bin/python RUN ln -s /usr/bin/python3 /usr/bin/python +# install R +RUN DEBIAN_FRONTEND="noninteractive" apt-get install r-base-dev -y +# make jars directory (from livy.conf) +RUN mkdir /jars # expose ports EXPOSE 8998 # start from init folder WORKDIR /opt/docker-init -ENTRYPOINT ["./entrypoint"] \ No newline at end of file +ENTRYPOINT ["./entrypoint"] diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..55b5b3f --- /dev/null +++ b/build.sh @@ -0,0 +1,4 @@ +#/bin/sh + +docker build . -t stellars/livy-server-docker:spark2.4.8 +#docker build . -t stellars/livy-server-docker:spark3.1.1 diff --git a/conf/livy.conf b/conf/livy.conf index 99d3029..75756f5 100644 --- a/conf/livy.conf +++ b/conf/livy.conf @@ -77,7 +77,7 @@ livy.server.session.state-retain.sec = 1800s # List of local directories from where files are allowed to be added to user sessions. By # default it's empty, meaning users can only reference remote URIs when starting their # sessions. -livy.file.local-dir-whitelist = /tmp/ +livy.file.local-dir-whitelist = /jars/ # Whether to enable csrf protection, by default it is false. If it is enabled, client should add # http-header "X-Requested-By" in request if the http method is POST/DELETE/PUT/PATCH. diff --git a/init/entrypoint b/init/entrypoint index 53252eb..3285a44 100755 --- a/init/entrypoint +++ b/init/entrypoint @@ -35,7 +35,7 @@ function writeConfigOptions(){ export SPARK_MASTER_PORT=$SPARK_MASTER_PORT export DEPLOY_MODE=$DEPLOY_MODE - cat /opt/docker-conf/livy.conf | envsubst > /opt/livy-0.5.0-incubating-bin/conf/livy.conf + cat /opt/docker-conf/livy.conf | envsubst > /opt/livy/conf/livy.conf } function init(){ @@ -58,9 +58,9 @@ function init(){ function livy_server_service(){ - export SPARK_HOME=/opt/spark-2.3.1-bin-hadoop2.7/ + export SPARK_HOME=/opt/spark/ echo "starting Livy Server!" - /opt/livy-0.5.0-incubating-bin/bin/livy-server start + /opt/livy/bin/livy-server start # whatever blocking call tail -f /dev/null diff --git a/push.sh b/push.sh new file mode 100644 index 0000000..365ea50 --- /dev/null +++ b/push.sh @@ -0,0 +1,3 @@ +#/bin/sh +#docker push stellars/livy-server-docker:spark3.1.1 +docker push stellars/livy-server-docker:spark2.4.8