Style changes, renamed ContextConfig to GetContextConfig, added readme and application examples

Author: Stephen Carman

README.md

@@ -600,6 +600,14 @@ To pass settings directly to the sparkConf that do not use the "spark." prefix "
         }
     }
 
+To add to the underlying Hadoop configuration in a Spark context, add the hadoop section to the context settings
+
+    spark.context-settings {
+        hadoop {
+            mapreduce.framework.name = "Foo"
+        }
+    }
+
 For the exact context configuration parameters, see JobManagerActor docs as well as application.conf.
 
 Also see the [yarn doc](doc/yarn.md) for more tips.

job-server/src/main/resources/application.conf

@@ -127,6 +127,11 @@ spark {
     passthrough {
       spark.driver.allowMultipleContexts = true  # Ignore the Multiple context exception related with SPARK-2243
     }
+
+    #This adds configuration to the underlying Hadoop configuration in the Spark Context
+    #hadoop {
+    #  mapreduce.framework.name = "FooFramework"
+    #}
   }
 }
 

job-server/src/spark.jobserver/JobManagerActor.scala

@@ -24,7 +24,7 @@ object JobManagerActor {
   case class StartJob(appName: String, classPath: String, config: Config,
                       subscribedEvents: Set[Class[_]])
   case class KillJob(jobId: String)
-  case object ContextConfig
+  case object GetContextConfig
   case object SparkContextStatus
 
   // Results/Data
@@ -164,10 +164,10 @@ class JobManagerActor(contextConfig: Config) extends InstrumentedActor {
         }
       }
     }
-    case ContextConfig => {
-      if (jobContext.sparkContext == null){
+    case GetContextConfig => {
+      if (jobContext.sparkContext == null) {
         sender ! SparkContextDead
-      }else{
+      } else {
         try {
           val conf: SparkConf = jobContext.sparkContext.getConf
           val hadoopConf: Configuration = jobContext.sparkContext.hadoopConfiguration

job-server/test/spark.jobserver/LocalContextSupervisorSpec.scala

@@ -111,7 +111,7 @@ class LocalContextSupervisorSpec extends TestKit(LocalContextSupervisorSpec.syst
       supervisor ! GetContext("c1")
       expectMsgPF(5 seconds, "I can't find that context :'-(") {
         case (contextActor: ActorRef, resultActor: ActorRef) => {
-          contextActor ! ContextConfig
+          contextActor ! GetContextConfig
           val cc = expectMsgClass(classOf[ContextConfig])
           cc.contextName shouldBe "c1"
           cc.contextConfig.get("spark.ui.enabled") shouldBe "false"