Merge remote-tracking branch 'oss/master' into dse-2.2

job-server-extras/src/test/scala/spark/jobserver/python/PythonHiveContextFactorySpec.scala
job-server-extras/src/test/scala/spark/jobserver/python/PythonSQLContextFactorySpec.scala
job-server-extras/src/test/scala/spark/jobserver/python/PythonSparkContextFactorySpec.scala

Author: jtgrabowski

.github/PULL_REQUEST_TEMPLATE

@@ -0,0 +1,21 @@
+**Pull Request checklist**
+
+- [ ] The commit(s) message(s) follows the contribution [guidelines](doc/contribution-guidelines.md#commit-message-format) ?
+- [ ] Tests for the changes have been added (for bug fixes / features) ?
+- [ ] Docs have been added / updated (for bug fixes / features) ?
+
+**Current behavior :** (link exiting issues here : https://help.github.com/articles/basic-writing-and-formatting-syntax/#referencing-issues-and-pull-requests)
+
+
+
+**New behavior :**
+
+
+
+**BREAKING CHANGES**
+
+If this PR contains a breaking change, please describe the impact and migration
+path for existing applications.
+If not please remove this section.
+
+**Other information**:

.gitignore

@@ -4,6 +4,7 @@ src_managed/
 project/boot/
 test-reports/
 *.log*
+*.patch
 .ensime*
 .idea*
 .vagrant
@@ -14,8 +15,8 @@ config/*.conf
 config/*.sh
 config/*.ini
 job-server/config/*.conf
-job-server/config/*.sh
 job-server/config/*.ini
+job-server-extras/spark-warehouse/
 metastore_db/
 
 #ignore generated config

.travis.yml

@@ -3,6 +3,7 @@ sudo: required
 env:
   global:
     - DOCKER_IMAGE="spark-jobserver:ci"
+    - DOCKER_API_VERSION="1.24"
 
 language: go
 go:
@@ -12,8 +13,12 @@ services:
   - docker
 
 cache:
+  apt: true
   directories:
     - docker
+    - $HOME/.m2
+    - $HOME/.ivy2/cache
+    - $HOME/.sbt/boot/scala-$TRAVIS_SCALA_VERSION
 
 before_install:
   - go get github.com/tonistiigi/buildcache/cmd/buildcache
@@ -32,4 +37,4 @@ install:
   - sudo chown $USER docker/latest_cache.tar.gz
 
 script:
-  - docker run --rm -t -i $DOCKER_IMAGE
+  - docker run `/bin/bash <(curl -s https://codecov.io/env)` --rm -t -i $DOCKER_IMAGE

Dockerfile.test

@@ -24,7 +24,7 @@ RUN sbt update
 # add the rest of the code
 COPY . .
 
-ENV SPARK_HOME /tmp/spark-2.0.2-bin-hadoop2.7
+ENV SPARK_HOME /tmp/spark-2.1.0-bin-hadoop2.7
 ENV JAVA_OPTIONS "-Xmx1500m -XX:MaxPermSize=512m -Dakka.test.timefactor=3"
 
 CMD ["/usr/src/app/run_tests.sh"]

README.md

@@ -81,8 +81,8 @@ object MetricsSerializer {
 
   private def process(metric: Metric): Map[String, Any] = {
     metric match {
-      case c: Counter =>   Map("type" -> "counter", "count" -> c.count())
-      case m: Meter =>     Map("type" -> "meter") ++ meterToMap(m)
+      case c: Counter => Map("type" -> "counter", "count" -> c.count())
+      case m: Meter => Map("type" -> "meter") ++ meterToMap(m)
       case g: Gauge[_] => Map("type" -> "gauge", "value" -> g.value())
       // For Timers, ignore the min/max/mean values, as they are for all time.  We're just interested
       // in the recent (biased) histogram values.
@@ -97,17 +97,17 @@ object MetricsSerializer {
   private def meterToMap(m: Metered) =
     Map("units" -> m.rateUnit.toString.toLowerCase,
         "count" -> m.count,
-        "mean"  -> m.meanRate,
-        "m1"    -> m.oneMinuteRate,
-        "m5"    -> m.fiveMinuteRate,
-        "m15"   -> m.fifteenMinuteRate)
+        "mean" -> m.meanRate,
+        "m1" -> m.oneMinuteRate,
+        "m5" -> m.fiveMinuteRate,
+        "m15" -> m.fifteenMinuteRate)
 
   /** Extracts the histogram (Median, 75%, 95%, 98%, 99% 99.9%) values to a map */
   private def histogramToMap(h: Sampling) =
     Map("median" -> h.getSnapshot().getMedian(),
-        "p75"    -> h.getSnapshot().get75thPercentile(),
-        "p95"    -> h.getSnapshot().get95thPercentile(),
-        "p98"    -> h.getSnapshot().get98thPercentile(),
-        "p99"    -> h.getSnapshot().get99thPercentile(),
-        "p999"   -> h.getSnapshot().get999thPercentile())
+        "p75" -> h.getSnapshot().get75thPercentile(),
+        "p95" -> h.getSnapshot().get95thPercentile(),
+        "p98" -> h.getSnapshot().get98thPercentile(),
+        "p99" -> h.getSnapshot().get99thPercentile(),
+        "p999" -> h.getSnapshot().get999thPercentile())
 }

akka-app/src/main/scala/spark/jobserver/common/akka/web/CommonRoutes.scala

@@ -35,13 +35,13 @@ object JsonUtils {
           JsObject(pairs)
       }
       case a: Array[_] => seqFormat[Any].write(a.toSeq)
-      case true        => JsTrue
-      case false       => JsFalse
-      case p: Product  => seqFormat[Any].write(p.productIterator.toSeq)
-      case null        => JsNull
+      case true => JsTrue
+      case false => JsFalse
+      case p: Product => seqFormat[Any].write(p.productIterator.toSeq)
+      case null => JsNull
       case m: java.util.Map[_, _] => AnyJsonFormat.write(m.asScala.toMap)
       case l: java.util.List[_] => seqFormat[Any].write(l.asScala)
-      case x           => JsString(x.toString)
+      case x => JsString(x.toString)
     }
     def read(value: JsValue): Any = value match {
       case JsNumber(n) => n.intValue()

akka-app/src/main/scala/spark/jobserver/common/akka/web/JsonUtils.scala

@@ -1,15 +1,18 @@
 package spark.jobserver.common.akka
 
-import org.scalatest.matchers.ShouldMatchers
-import org.scalatest.{Matchers, FunSpec}
-import akka.testkit.TestActorRef
+import org.scalatest.{BeforeAndAfterAll, Matchers, FunSpec}
+import akka.testkit.{TestKit, TestActorRef}
 
 import akka.actor.{Actor, ActorSystem}
 
 
-class ActorMetricsSpec extends FunSpec with Matchers {
+class ActorMetricsSpec extends FunSpec with Matchers with BeforeAndAfterAll {
   implicit val system = ActorSystem("test")
 
+  override def afterAll(): Unit = {
+    TestKit.shutdownActorSystem(system)
+  }
+
   describe("actor metrics") {
     it("should increment receive count metric when a message is received") {
       val actorRef = TestActorRef(new DummyActor with ActorMetrics)
@@ -19,4 +22,4 @@ class ActorMetricsSpec extends FunSpec with Matchers {
       actor.metricReceiveTimer.count should equal (1)
     }
   }
-}
+}

akka-app/src/test/scala/spark/jobserver/common/akka/ActorMetricsSpec.scala

@@ -1,8 +1,7 @@
 package spark.jobserver.common.akka
 
-import org.scalatest.matchers.ShouldMatchers
-import org.scalatest.{Matchers, FunSpec}
-import akka.testkit.TestActorRef
+import org.scalatest.{BeforeAndAfterAll, Matchers, FunSpec}
+import akka.testkit.{TestKit, TestActorRef}
 
 import akka.actor.{Actor, ActorSystem}
 
@@ -23,9 +22,13 @@ trait AddPrefix extends ActorStack {
   }
 }
 
-class ActorStackSpec extends FunSpec with Matchers {
+class ActorStackSpec extends FunSpec with Matchers with BeforeAndAfterAll {
   implicit val system = ActorSystem("test")
 
+  override def afterAll(): Unit = {
+    TestKit.shutdownActorSystem(system)
+  }
+
   describe("stacking traits") {
     it("should be able to stack traits and receive messages") {
       val actorRef = TestActorRef(new DummyActor with AddPrefix)

akka-app/src/test/scala/spark/jobserver/common/akka/ActorStackSpec.scala

@@ -3,6 +3,7 @@ package spark.jobserver.common.akka.actor
 import akka.actor.{ActorSystem, Props, ActorRef}
 import akka.testkit.{TestKit, ImplicitSender, TestProbe}
 import org.scalatest.{MustMatchers, FunSpecLike, BeforeAndAfterAll}
+import spark.jobserver.common.akka.AkkaTestUtils
 
 // Our test reaper.  Sends the snooper a message when all
 // the souls have been reaped
@@ -19,7 +20,7 @@ class ReaperSpec extends TestKit(ActorSystem("ReaperSpec")) with ImplicitSender
   import scala.concurrent.duration._
 
   override def afterAll() {
-    system.shutdown()
+    AkkaTestUtils.shutdownAndWait(system)
   }
 
   describe("Reaper") {