Revised based on feedback from Velvia. Added note to make a clarification on yarn page.

Author: carbribi

doc/yarn.md

@@ -4,6 +4,8 @@
 
 (I would like to thank Jon Buffington for sharing the config tips below.... @velvia)
 
+Note:  This is for yarn with docker.  If you are looking to deploy on a yarn cluster via EMR, then this link would be more useful [EMR](https://github.com/spark-jobserver/spark-jobserver/blob/master/doc/EMR.md)
+
 ### Configuring the Spark-Jobserver Docker package to run in Yarn-Client Mode
 
 To run the Spark-Jobserver in yarn-client mode you have to do a little bit extra of configuration.

job-server-extras/src/spark.jobserver/NamedBroadcast.scala

@@ -12,9 +12,6 @@ case class NamedBroadcast[T](broadcast: Broadcast[T]) extends NamedObject
   */
 class BroadcastPersister[T] extends NamedObjectPersister[NamedBroadcast[T]] {
   override def persist(namedObj: NamedBroadcast[T], name: String) {
-    namedObj match {
-      case NamedBroadcast(broadcast) =>{}
-    }
   }
   override def unpersist(namedObj: NamedBroadcast[T]) {
     namedObj match {

job-server-extras/test/spark.jobserver/NamedObjectsJobSpec.scala

@@ -28,8 +28,8 @@ class NamedObjectsJobSpec extends JobSpecBase(JobManagerSpec.getNewSystem) {
 
   val jobName = "spark.jobserver.NamedObjectsTestJob"
 
-  private def getCreateConfig(createDF: Boolean, createRDD: Boolean, createBroadcast: Boolean) : Config = {
-    ConfigFactory.parseString("spark.jobserver.named-object-creation-timeout = 60 s, " + 
+  private def getCreateConfig(createDF: Boolean, createRDD: Boolean, createBroadcast: Boolean = false) : Config = {
+    ConfigFactory.parseString("spark.jobserver.named-object-creation-timeout = 60 s, " +
         NamedObjectsTestJobConfig.CREATE_DF + " = " + createDF + ", " +
         NamedObjectsTestJobConfig.CREATE_RDD + " = " + createRDD + ", " +
         NamedObjectsTestJobConfig.CREATE_BROADCAST + " = " + createBroadcast)
@@ -43,11 +43,11 @@ class NamedObjectsJobSpec extends JobSpecBase(JobManagerSpec.getNewSystem) {
   describe("NamedObjects (RDD)") {
     it("should survive from one job to another one") {
 
-      manager ! JobManagerActor.StartJob("demo", jobName, getCreateConfig(false, true, false), errorEvents ++ syncEvents)
+      manager ! JobManagerActor.StartJob("demo", jobName, getCreateConfig(false, true), errorEvents ++ syncEvents)
       val JobResult(_, names: Array[String]) = expectMsgClass(classOf[JobResult])
       names should contain("rdd1")
 
-      manager ! JobManagerActor.StartJob("demo", jobName, getCreateConfig(false, false, false), errorEvents ++ syncEvents)
+      manager ! JobManagerActor.StartJob("demo", jobName, getCreateConfig(false, false), errorEvents ++ syncEvents)
       val JobResult(_, names2: Array[String]) = expectMsgClass(classOf[JobResult])
 
       names2 should contain("rdd1")
@@ -65,12 +65,12 @@ class NamedObjectsJobSpec extends JobSpecBase(JobManagerSpec.getNewSystem) {
   describe("NamedObjects (DataFrame)") {
     it("should survive from one job to another one") {
 
-      manager ! JobManagerActor.StartJob("demo", jobName, getCreateConfig(true, false, false), errorEvents ++ syncEvents)
+      manager ! JobManagerActor.StartJob("demo", jobName, getCreateConfig(true, false), errorEvents ++ syncEvents)
       val JobResult(_, names: Array[String]) = expectMsgClass(classOf[JobResult])
 
       names should contain("df1")
 
-      manager ! JobManagerActor.StartJob("demo", jobName, getCreateConfig(false, false, false), errorEvents ++ syncEvents)
+      manager ! JobManagerActor.StartJob("demo", jobName, getCreateConfig(false, false), errorEvents ++ syncEvents)
       val JobResult(_, names2: Array[String]) = expectMsgClass(classOf[JobResult])
 
       names2 should equal(names)
@@ -84,13 +84,13 @@ class NamedObjectsJobSpec extends JobSpecBase(JobManagerSpec.getNewSystem) {
   describe("NamedObjects (DataFrame + RDD)") {
     it("should survive from one job to another one") {
 
-      manager ! JobManagerActor.StartJob("demo", jobName, getCreateConfig(true, true, false), errorEvents ++ syncEvents)
+      manager ! JobManagerActor.StartJob("demo", jobName, getCreateConfig(true, true), errorEvents ++ syncEvents)
       val JobResult(_, names: Array[String]) = expectMsgClass(classOf[JobResult])
 
       names should contain("rdd1")
       names should contain("df1")
 
-      manager ! JobManagerActor.StartJob("demo", jobName, getCreateConfig(false, false, false), errorEvents ++ syncEvents)
+      manager ! JobManagerActor.StartJob("demo", jobName, getCreateConfig(false, false), errorEvents ++ syncEvents)
       val JobResult(_, names2: Array[String]) = expectMsgClass(classOf[JobResult])
 
       names2 should equal(names)