Resolve environment vars in Typesafe configs everywhere

Author: velvia

job-server/src/spark.jobserver/JobServer.scala

@@ -36,7 +36,7 @@ object JobServer {
         println("Could not find configuration file " + configFile)
         sys.exit(1)
       }
-      ConfigFactory.parseFile(configFile).withFallback(defaultConfig)
+      ConfigFactory.parseFile(configFile).withFallback(defaultConfig).resolve()
     } else {
       defaultConfig
     }

job-server/src/spark.jobserver/WebApi.scala

@@ -117,7 +117,7 @@ class WebApi(system: ActorSystem,
             complete(StatusCodes.BadRequest, errMap("context name must start with letters"))
           } else {
             parameterMap { (params) =>
-              val config = ConfigFactory.parseMap(params.asJava)
+              val config = ConfigFactory.parseMap(params.asJava).resolve()
               val future = (supervisor ? AddContext(contextName, config))(contextTimeout.seconds)
               respondWithMediaType(MediaTypes.`application/json`) { ctx =>
                 future.map {
@@ -290,7 +290,7 @@ class WebApi(system: ActorSystem,
               try {
                 val async = !syncOpt.getOrElse(false)
                 val postedJobConfig = ConfigFactory.parseString(configString)
-                val jobConfig = postedJobConfig.withFallback(config)
+                val jobConfig = postedJobConfig.withFallback(config).resolve()
                 val contextConfig = Try(jobConfig.getConfig("spark.context-settings")).
                                       getOrElse(ConfigFactory.empty)
                 val jobManager = getJobManagerForContext(contextOpt, contextConfig, classPath)

notes/0.5.2.markdown

@@ -1,5 +1,6 @@
 #Scala #akka @ApacheSpark
 
 * Spark streaming context support !!  (@zeitos)
-* Change `server_start.sh` to use `spark-submit`
+* Change `server_start.sh` to use `spark-submit`.  This should fix some edge case bugs.
 * Configurable driver memory (@acidghost)
+* Be able to accept environment vars in job server config files, eg `master = ${?MY_SPARK_HOST}`