Merge pull request #10 from riptano/DSP-13836

DSP-13836 bump versions of cassandra driver and netty

Author: jtgrabowski

bin/server_package.sh

@@ -1,6 +1,8 @@
-#!/bin/bash -ue
+#!/bin/bash -e
+
+set -x
 # Script for packaging all the job server files to .tar.gz for Mesos or other single-image deploys
-WORK_DIR=/tmp/job-server
+WORK_DIR="$TMPDIR"/tmp/job-server
 
 if [ "$#" -ne 1 ]; then
   echo "Syntax: ${0} <Environment>"
@@ -50,8 +52,7 @@ pushd "${bin}/.." > /dev/null
          bin/manager_start.sh
          bin/setenv.sh
          ${CONFIG_DIR}/${ENV}.conf
-         config/shiro.ini
-         config/log4j-server.properties"
+         config/logback-server.xml"
 
   rm -rf $WORK_DIR
   mkdir -p $WORK_DIR
@@ -62,7 +63,7 @@ popd > /dev/null
 pushd "${WORK_DIR}" > /dev/null
   TAR_FILE="${WORK_DIR}/job-server.tar.gz"
   rm -f "${TAR_FILE}"
-  tar zcvf "${TAR_FILE}" ./*
+  tar zcvf "${TAR_FILE}" *
 popd > /dev/null
 
 echo "Created distribution at ${TAR_FILE}"

build.sbt

@@ -89,7 +89,7 @@ lazy val root = Project(id = "root", base = file("."))
   .disablePlugins(SbtScalariform).enablePlugins(DockerPlugin)
 
 lazy val jobServerExtrasSettings = revolverSettings ++ Assembly.settings ++ publishSettings ++ Seq(
-  libraryDependencies ++= sparkExtraDeps,
+  libraryDependencies ++= sparkExtraDeps ++ sparkExtraDepsTest,
   // Extras packages up its own jar for testing itself
   test in Test <<= (test in Test).dependsOn(packageBin in Compile)
     .dependsOn(clean in Compile),

job-server/src/main/scala/spark/jobserver/WebApi.scala

@@ -134,6 +134,7 @@ class WebApi(system: ActorSystem,
       .fold(100 * 1024)(config.getBytes(_).toInt)
 
   val contextTimeout = SparkJobUtils.getContextCreationTimeout(config)
+  val contextDeletionTimeout = SparkJobUtils.getContextDeletionTimeout(config)
   val bindAddress = config.getString("spark.jobserver.bind-address")
 
   val logger = LoggerFactory.getLogger(getClass)
@@ -390,7 +391,7 @@ class WebApi(system: ActorSystem,
           //  and currently running jobs will be lost.  Use with care!
           path(Segment) { (contextName) =>
             val (cName, _) = determineProxyUser(config, authInfo, contextName)
-            val future = supervisor ? StopContext(cName)
+            val future = (supervisor ? StopContext(cName))(contextDeletionTimeout.seconds)
             respondWithMediaType(MediaTypes.`application/json`) { ctx =>
               future.map {
                 case ContextStopped => ctx.complete(StatusCodes.OK, successMap("Context stopped"))

project/Dependencies.scala

@@ -42,6 +42,10 @@ object Dependencies {
       )
   )
 
+  lazy val sparkExtraDepsTest = Seq(
+    "com.google.guava" % "guava" % "16.0.1" % Test force()
+  )
+
   lazy val sparkPythonDeps = Seq(
     "net.sf.py4j" % "py4j" % py4j,
     "io.spray" %% "spray-json" % sprayJson % Test

project/Versions.scala

@@ -1,10 +1,10 @@
 import scala.util.Properties.isJavaAtLeast
 
 object Versions {
-  lazy val spark = sys.env.getOrElse("SPARK_VERSION", "2.1.0")
+  lazy val spark = sys.env.getOrElse("SPARK_VERSION", "2.2.0")
 
   lazy val akka = "2.4.9"
-  lazy val cassandra = "3.0.3"
+  lazy val cassandra = "3.2.0"
   lazy val cassandraUnit = "2.2.2.1"
   lazy val commons = "1.4"
   lazy val flyway = "3.2.1"
@@ -15,12 +15,12 @@ object Versions {
   lazy val logback = "1.0.7"
   lazy val mesos = sys.env.getOrElse("MESOS_VERSION", "1.0.0-2.0.89.ubuntu1404")
   lazy val metrics = "2.2.0"
-  lazy val netty = "4.0.42.Final"
+  lazy val netty = "4.0.44.Final"
   lazy val postgres = "9.4.1209"
   lazy val py4j = "0.10.4"
   lazy val scalaTest = "2.2.6"
   lazy val scalatic = "2.2.6"
-  lazy val shiro = "1.2.4"
+  lazy val shiro = "1.3.2"
   lazy val slick = "3.1.1"
   lazy val spray = "1.3.3"
   lazy val sprayJson = "1.3.2"

project/plugins.sbt

@@ -6,7 +6,7 @@ addSbtPlugin("io.spray" % "sbt-revolver" % "0.8.0")       // For quick restarts
 
 addSbtPlugin("org.scalariform" % "sbt-scalariform" % "1.6.0")
 
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5")
 
 addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0")