Merge pull request #9 from riptano/DSP-11503

DSP-11503 Upgrade Spark Jobserver for Spark 2.0.2

Author: jtgrabowski

.dockerignore

@@ -0,0 +1,6 @@
+/target
+*/target
+**/target
+.git
+.ensime
+.ensime_cache

.gitignore

@@ -7,12 +7,15 @@ test-reports/
 .ensime*
 .idea*
 .vagrant
+*.pyc
 
 # ignore deployment configs
 config/*.conf
 config/*.sh
+config/*.ini
 job-server/config/*.conf
 job-server/config/*.sh
+job-server/config/*.ini
 metastore_db/
 
 #ignore generated config
@@ -22,4 +25,4 @@ bin/ec2_example.sh
 ec2Cluster/
 
 # don't ignore the ec2 config and sh files
-!job-server/config/ec2.sh
+!job-server/config/ec2.sh

.travis.yml

@@ -1,10 +1,35 @@
-language: scala
+sudo: required
+
 env:
   global:
-    _JAVA_OPTIONS="-Xmx1500m -XX:MaxPermSize=512m -Dakka.test.timefactor=3"
-scala:
-   - 2.10.6
-   - 2.11.8
-jdk:
-   - oraclejdk8
-   - oraclejdk7
+    - DOCKER_IMAGE="spark-jobserver:ci"
+
+language: go
+go:
+  - 1.7
+
+services:
+  - docker
+
+cache:
+  directories:
+    - docker
+
+before_install:
+  - go get github.com/tonistiigi/buildcache/cmd/buildcache
+
+install:
+  - if [ -e docker/latest_image.tar.gz ]; then
+      docker load -i docker/latest_image.tar.gz;
+    fi
+  - if [ -e docker/latest_cache.tar.gz ]; then
+      docker load -i docker/latest_cache.tar.gz;
+    fi
+  - echo "docker/" >> .dockerignore
+  - docker build -f Dockerfile.test -t $DOCKER_IMAGE .
+  - docker save $DOCKER_IMAGE | gzip > docker/latest_image.tar.gz
+  - sudo `which buildcache` save -o docker/latest_cache.tar.gz $DOCKER_IMAGE
+  - sudo chown $USER docker/latest_cache.tar.gz
+
+script:
+  - docker run --rm -t -i $DOCKER_IMAGE

Dockerfile.test

@@ -0,0 +1,30 @@
+FROM instructure/java:8
+
+USER root
+# install and cache sbt, python
+
+RUN echo 'deb http://dl.bintray.com/sbt/debian /' > /etc/apt/sources.list.d/sbt.list && \
+    apt-get -qq update && \
+    apt-get install -y --force-yes python3 python3-pip python-pip sbt=0.13.8 && \
+    sbt
+# running sbt downloads some of its internals, speed up sebsequent sbt runs
+
+WORKDIR /usr/src/app/
+
+# install other ci deps
+COPY ci ci
+RUN ci/install-python-dependencies.sh && \
+    ci/install-spark.sh
+
+# add sbt and cache deps
+COPY project project
+COPY build.sbt .
+RUN sbt update
+
+# add the rest of the code
+COPY . .
+
+ENV SPARK_HOME /tmp/spark-2.0.2-bin-hadoop2.7
+ENV JAVA_OPTIONS "-Xmx1500m -XX:MaxPermSize=512m -Dakka.test.timefactor=3"
+
+CMD ["/usr/src/app/run_tests.sh"]

README.md

@@ -1,8 +1,7 @@
-package ooyala.common.akka
+package spark.jobserver.common.akka
 
 import java.util.concurrent.TimeUnit
 
-import akka.actor.Actor
 import com.yammer.metrics.Metrics
 
 /**

akka-app/src/ooyala.common.akka/ActorMetrics.scala renamed to akka-app/src/main/scala/spark/jobserver/common/akka/ActorMetrics.scala

@@ -1,4 +1,4 @@
-package ooyala.common.akka
+package spark.jobserver.common.akka
 
 import akka.actor.Actor
 

akka-app/src/ooyala.common.akka/ActorStack.scala renamed to akka-app/src/main/scala/spark/jobserver/common/akka/ActorStack.scala

@@ -1,4 +1,4 @@
-package ooyala.common.akka
+package spark.jobserver.common.akka
 
 import akka.actor.{ActorSystem, ActorRef}
 import akka.pattern.gracefulStop

akka-app/src/ooyala.common.akka/AkkaTestUtils.scala renamed to akka-app/src/main/scala/spark/jobserver/common/akka/AkkaTestUtils.scala

@@ -1,4 +1,4 @@
-package ooyala.common.akka
+package spark.jobserver.common.akka
 
 import akka.actor.Actor
 

akka-app/src/ooyala.common.akka/InstrumentedActor.scala renamed to akka-app/src/main/scala/spark/jobserver/common/akka/InstrumentedActor.scala

@@ -1,6 +1,5 @@
-package ooyala.common.akka
+package spark.jobserver.common.akka
 
-import akka.actor.Actor
 import org.slf4j.LoggerFactory
 
 /**