fix(core): Update spark version to 2.2.0 (spark-jobserver#845)

* Explicitly add hadoop-client dependency
* Update netty version
* Update C* driver version
* Remove metastore_db lock across tests

Author: noorul

Dockerfile.test

@@ -24,7 +24,7 @@ RUN sbt update
 # add the rest of the code
 COPY . .
 
-ENV SPARK_HOME /tmp/spark-2.1.0-bin-hadoop2.7
+ENV SPARK_HOME /tmp/spark-2.2.0-bin-hadoop2.7
 ENV JAVA_OPTIONS "-Xmx1500m -XX:MaxPermSize=512m -Dakka.test.timefactor=3"
 
 CMD ["/usr/src/app/run_tests.sh"]

README.md

@@ -119,7 +119,7 @@ Spark Job Server is now included in Datastax Enterprise 4.8!
 | 0.6.1       | 1.5.2         |
 | 0.6.2       | 1.6.1         |
 | 0.7.0       | 1.6.2         |
-| 0.8.0-SNAPSHOT | 2.1.0    |
+| 0.8.0-SNAPSHOT | 2.2.0    |
 
 For release notes, look in the `notes/` directory.
 

ci/install-spark.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 set -e
-curl -L -o /tmp/spark.tgz http://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz
+curl -L -o /tmp/spark.tgz http://d3kbcqa49mib13.cloudfront.net/spark-2.2.0-bin-hadoop2.7.tgz
 tar -xvzf /tmp/spark.tgz -C /tmp

job-server-python/src/python/test/apitests.py

@@ -1,3 +1,5 @@
+import errno
+import os
 import unittest
 from pyhocon import ConfigFactory
 from pyspark import SparkConf, SparkContext
@@ -6,6 +8,14 @@
 from py4j.java_gateway import java_import
 
 
+def silentremove(filename):
+    try:
+        os.remove(filename)
+    except OSError as exc:
+        if exc.errno != errno.ENOENT:
+            raise
+
+
 class WordCountSparkJob(SparkJob):
     """
     Simple example of a SparkContext job for use in tests
@@ -56,6 +66,11 @@ class TestSJSApi(unittest.TestCase):
 
     def setUp(self):
         conf = SparkConf().setAppName('test').setMaster('local[*]')
+        pwd = os.path.dirname(os.path.realpath(__file__))
+        metastore_dir = os.path.abspath(os.path.join(pwd, '..',
+                                                     'metastore_db'))
+        silentremove(os.path.join(metastore_dir, "dbex.lck"))
+        silentremove(os.path.join(metastore_dir, "db.lck"))
         self.sc = SparkContext(conf=conf)
         self.jvm = self.sc._gateway.jvm
         java_import(self.jvm, "org.apache.spark.sql.*")

project/Dependencies.scala

@@ -34,6 +34,8 @@ object Dependencies {
   )
 
   lazy val sparkExtraDeps = Seq(
+    "org.apache.derby" % "derby" % derby % Provided excludeAll(excludeNettyIo, excludeQQ),
+    "org.apache.hadoop" % "hadoop-client" % hadoop % Provided excludeAll(excludeNettyIo, excludeQQ),
     "org.apache.spark" %% "spark-mllib" % spark % Provided excludeAll(excludeNettyIo, excludeQQ),
     "org.apache.spark" %% "spark-sql" % spark % Provided excludeAll(excludeNettyIo, excludeQQ),
     "org.apache.spark" %% "spark-streaming" % spark % Provided excludeAll(excludeNettyIo, excludeQQ),

project/Versions.scala

@@ -1,21 +1,23 @@
 import scala.util.Properties.isJavaAtLeast
 
 object Versions {
-  lazy val spark = sys.env.getOrElse("SPARK_VERSION", "2.1.0")
+  lazy val spark = sys.env.getOrElse("SPARK_VERSION", "2.2.0")
 
   lazy val akka = "2.4.9"
-  lazy val cassandra = "3.0.3"
+  lazy val cassandra = "3.3.0"
   lazy val cassandraUnit = "2.2.2.1"
   lazy val commons = "1.4"
+  lazy val derby = "10.12.1.1"
   lazy val flyway = "3.2.1"
+  lazy val hadoop = "2.7.3"
   lazy val h2 = "1.3.176"
   lazy val java = sys.env.getOrElse("JAVA_VERSION", "8-jdk")
   lazy val jodaConvert = "1.8.1"
   lazy val jodaTime = "2.9.3"
   lazy val logback = "1.0.7"
   lazy val mesos = sys.env.getOrElse("MESOS_VERSION", "1.0.0-2.0.89.ubuntu1404")
   lazy val metrics = "2.2.0"
-  lazy val netty = "4.0.42.Final"
+  lazy val netty = "4.0.44.Final"
   lazy val postgres = "9.4.1209"
   lazy val mysql = "5.1.42"
   lazy val py4j = "0.10.4"