Merge pull request spark-jobserver#838 from noorul/noorul_fix_spark_version_used_by_ci

fix(ci): Use spark version 2.1.0 in ci tests

Author: noorul

Dockerfile.test

@@ -24,7 +24,7 @@ RUN sbt update
 # add the rest of the code
 COPY . .
 
-ENV SPARK_HOME /tmp/spark-2.0.1-bin-hadoop2.7
+ENV SPARK_HOME /tmp/spark-2.1.0-bin-hadoop2.7
 ENV JAVA_OPTIONS "-Xmx1500m -XX:MaxPermSize=512m -Dakka.test.timefactor=3"
 
 CMD ["/usr/src/app/run_tests.sh"]

ci/install-spark.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 set -e
-curl -L -o /tmp/spark.tgz http://d3kbcqa49mib13.cloudfront.net/spark-2.0.1-bin-hadoop2.7.tgz
+curl -L -o /tmp/spark.tgz http://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz
 tar -xvzf /tmp/spark.tgz -C /tmp

job-server-extras/src/test/scala/spark/jobserver/python/PythonSparkContextFactorySpec.scala

@@ -74,7 +74,7 @@ object PythonSparkContextFactorySpec {
   lazy val jobServerAPIExamplePath = jobServerPaths.find(_.getAbsolutePath.contains("examples"))
 
   lazy val pysparkPath = sys.env.get("SPARK_HOME").map(d => s"$d/python/lib/pyspark.zip")
-  lazy val py4jPath  = sys.env.get("SPARK_HOME").map(d => s"$d/python/lib/py4j-0.10.3-src.zip")
+  lazy val py4jPath  = sys.env.get("SPARK_HOME").map(d => s"$d/python/lib/py4j-0.10.4-src.zip")
   lazy val originalPythonPath  = sys.env.get("PYTHONPATH")
 
   case object DummyJobCache extends JobCache {

job-server-python/src/python/run-tests.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-PYTHONPATH=.:$SPARK_HOME/python/lib/pyspark.zip:$SPARK_HOME/python/lib/py4j-0.10.3-src.zip:$PYTHONPATH python test/apitests.py
+PYTHONPATH=.:$SPARK_HOME/python/lib/pyspark.zip:$SPARK_HOME/python/lib/py4j-0.10.4-src.zip:$PYTHONPATH python test/apitests.py
 exitCode=$?
 #This sleep is here so that all of Spark's shutdown stdout if written before we exit,
 #so that we return cleanly to the command prompt.

job-server-python/src/test/scala/spark/jobserver/python/SubprocessSpec.scala

@@ -34,10 +34,10 @@ object SubprocessSpec {
   lazy val jobServerPath = getPythonDir("src/python")
 
   lazy val pysparkPath = sys.env.get("SPARK_HOME").map(d => s"$d/python/lib/pyspark.zip")
-  lazy val py4jPath  = sys.env.get("SPARK_HOME").map(d => s"$d/python/lib/py4j-0.10.3-src.zip")
+  lazy val py4jPath  = sys.env.get("SPARK_HOME").map(d => s"$d/python/lib/py4j-0.10.4-src.zip")
   lazy val sparkPaths = sys.env.get("SPARK_HOME").map{sh =>
     val pysparkPath = s"$sh/python/lib/pyspark.zip"
-    val py4jPath  = s"$sh/python/lib/py4j-0.10.3-src.zip"
+    val py4jPath  = s"$sh/python/lib/py4j-0.10.4-src.zip"
     Seq(pysparkPath, py4jPath)
   }.getOrElse(Seq())
   lazy val originalPythonPath  = sys.env.get("PYTHONPATH")

project/Versions.scala

@@ -17,7 +17,7 @@ object Versions {
   lazy val metrics = "2.2.0"
   lazy val netty =  "4.0.42.Final"
   lazy val postgres = "9.4.1209"
-  lazy val py4j = "0.10.3"
+  lazy val py4j = "0.10.4"
   lazy val scalaTest = "2.2.6"
   lazy val scalatic = "2.2.6"
   lazy val shiro = "1.2.4"