feat(job-server): Client authentication (spark-jobserver#920)

* Support for certificate based client authentication
* Updated docs

Author: marcoboi

README.md

@@ -28,12 +28,15 @@ Also see [Chinese docs / 中文](doc/chinese/job-server.md).
   - [Creating a project from scratch using giter8 template](#creating-a-project-from-scratch-using-giter8-template)
   - [Creating a project manually assuming that you already have sbt project structure](#creating-a-project-manually-assuming-that-you-already-have-sbt-project-structure)
   - [NEW SparkJob API](#new-sparkjob-api)
+  - [NEW SparkJob API with Spark v2.1](#new-sparkjob-api-with-spark-v21)
   - [Dependency jars](#dependency-jars)
   - [Named Objects](#named-objects)
     - [Using Named RDDs](#using-named-rdds)
     - [Using Named Objects](#using-named-objects)
   - [HTTPS / SSL Configuration](#https--ssl-configuration)
-  - [Authentication](#authentication)
+    - [Server authentication](#server-authentication)
+    - [Client authentication](#client-authentication)
+  - [Basic authentication](#basic-authentication)
 - [Deployment](#deployment)
   - [Manual steps](#manual-steps)
   - [Context per JVM](#context-per-jvm)
@@ -497,7 +500,8 @@ def validate(sc:SparkContext, config: Config): SparkJobValidation = {
 ```
 
 ### HTTPS / SSL Configuration
-To activate ssl communication, set these flags in your application.conf file (Section 'spray.can.server'):
+#### Server authentication
+To activate server authentication and ssl communication, set these flags in your application.conf file (Section 'spray.can.server'):
 ```
   ssl-encryption = on
   # absolute path to keystore file
@@ -516,9 +520,19 @@ curl -k https://localhost:8090/contexts
 ```
 The ```-k``` flag tells curl to "Allow connections to SSL sites without certs". Export your server certificate and import it into the client's truststore to fully utilize ssl security.
 
-### Authentication
+#### Client authentication
+Client authentication can be enabled by simply pointing Job Server to a valid Trust Store. 
+As for server authentication, this is done by setting appropriate values in the application.conf.
+The minimum set of parameters to enable client authentication consists of:
+```
+  # truststore = "/some/path/server-truststore.jks"
+  # truststorePW = "changeit"
+```
+Note, client authentication implies server authentication, therefore client authentication will only be enabled once server authentication is activated.
 
-Authentication uses the [Apache Shiro](http://shiro.apache.org/index.html) framework. Authentication is activated by setting this flag (Section 'shiro'):
+### Basic authentication
+Basic authentication (username and password) in Job Server relies on the [Apache Shiro](http://shiro.apache.org/index.html) framework. 
+Basic authentication is activated by setting this flag (Section 'shiro'):
 ```
 authentication = on
 # absolute path to shiro config file, including file name

job-server/src/main/resources/application.conf

@@ -202,18 +202,23 @@ akka {
 
 # check the reference.conf in spray-can/src/main/resources for all defined settings
 spray.can.server {
-  # uncomment the next lines for making this an HTTPS example
+  ## uncomment the next lines for making this an HTTPS example
   # ssl-encryption = on
-  # path to keystore
-  #keystore = "/some/path/sjs.jks"
-  #keystorePW = "changeit"
-
-  # see http://docs.oracle.com/javase/7/docs/technotes/guides/security/StandardNames.html#SSLContext for more examples
-  # typical are either SSL or TLS
+  # keystore = "/some/path/server-keystore.jks"
+  # keystorePW = "changeit"
   encryptionType = "SSL"
   keystoreType = "JKS"
-  # key manager factory provider
   provider = "SunX509"
+  #
+  ## Client Authentication (optional): activated upon providing a truststore file
+  # truststore = "/some/path/server-truststore.jks"
+  # truststorePW = "changeit"
+  truststore-type = "JKS"
+  truststore-provider = "SunX509"
+  #
+  # see http://docs.oracle.com/javase/7/docs/technotes/guides/security/StandardNames.html#SSLContext for more examples
+  # typical are either SSL or TLS
+  # key manager factory provider
   # ssl engine provider protocols
   enabledProtocols = ["SSLv3", "TLSv1"]
   idle-timeout = 60 s

job-server/src/main/scala/spark/jobserver/WebApi.scala

@@ -182,6 +182,11 @@ class WebApi(system: ActorSystem,
       ServerSSLEngineProvider { engine =>
         val protocols = config.getStringList("spray.can.server.enabledProtocols")
         engine.setEnabledProtocols(protocols.toArray(Array[String]()))
+        val sprayConfig = config.getConfig("spray.can.server")
+        if(sprayConfig.hasPath("truststore")) {
+          engine.setNeedClientAuth(true)
+          logger.info("Client authentication activated.")
+        }
         engine
       }
     }

job-server/src/main/scala/spark/jobserver/util/SSLContextFactory.scala

@@ -1,46 +1,55 @@
 package spark.jobserver.util
 
-import javax.net.ssl.SSLContext
-import javax.net.ssl.KeyManagerFactory
-import java.security.KeyStore
 import java.io.FileInputStream
+import java.security.KeyStore
+import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManager, TrustManagerFactory}
+
 import com.typesafe.config.Config
 import org.slf4j.LoggerFactory
 
 /**
- * creates SSLContext based on configuration, the SSLContext is used for communication between
- * the SJS web server and its clients
- *
- * if encryption is not activated in the configuration, then the default ssl context is used which
- * falls back to un-encrypted communication
- */
+  * Creates SSLContext based on configuration, the SSLContext is used for communication between
+  * the SJS web server and its clients.
+  *
+  * If encryption is not activated in the configuration, then the default ssl context is used which
+  * falls back to un-encrypted communication.
+  */
 object SSLContextFactory {
+
   val logger = LoggerFactory.getLogger(getClass)
 
-  /**
-   * based on
-   * https://github.com/spray/spray/blob/v1.2-M8/examples/
-   * spray-can/simple-http-server/src/main/scala/spray/examples/MySslConfiguration.scala
-   */
   def createContext(config: Config): SSLContext = {
-    if (config.hasPath("ssl-encryption") &&
-      config.getBoolean("ssl-encryption")) {
 
-      checkRequiredParamsSet(config)
-      val sslContext = SSLContext.getInstance(config.getString("encryptionType"))
+    if (config.hasPath("ssl-encryption") && config.getBoolean("ssl-encryption")) {
 
-      val ksName = config.getString("keystore")
-      val ksPassphrase = config.getString("keystorePW").toCharArray()
-      val keystoreType = config.getString("keystoreType")
+      checkRequiredParamsSet(config)
       val encryptionType = config.getString("encryptionType")
+      val sslContext = SSLContext.getInstance(encryptionType)
       logger.info(encryptionType + " encryption activated.")
-      val ks = KeyStore.getInstance(keystoreType)
-      //throws exception if keystore cannot be found or accessed
-      // and prevents start-up
-      ks.load(new FileInputStream(ksName), ksPassphrase)
-      val kmf = KeyManagerFactory.getInstance(config.getString("provider"))
-      kmf.init(ks, ksPassphrase)
-      sslContext.init(kmf.getKeyManagers(), null, null)
+
+      val keyStoreFileName = config.getString("keystore")
+      val keyStorePassword = config.getString("keystorePW").toCharArray()
+      val keyStoreType = config.getString("keystoreType")
+      val keyStore = KeyStore.getInstance(keyStoreType)
+      keyStore.load(new FileInputStream(keyStoreFileName), keyStorePassword)
+      val keyManagerFactory = KeyManagerFactory.getInstance(config.getString("provider"))
+      keyManagerFactory.init(keyStore, keyStorePassword)
+      logger.info("Using KeyStore: " + keyStoreFileName)
+
+      var trustManagers: Array[TrustManager] = null
+      if (config.hasPath("truststore")) {
+        val trustStoreFileName = config.getString("truststore")
+        val trustStorePassword = config.getString("truststorePW").toCharArray()
+        val trustStoreType = config.getString("truststore-type")
+        val trustStore = KeyStore.getInstance(trustStoreType)
+        trustStore.load(new FileInputStream(trustStoreFileName), trustStorePassword)
+        val trustManagerFactory = TrustManagerFactory.getInstance(config.getString("truststore-provider"))
+        trustManagerFactory.init(trustStore)
+        trustManagers = trustManagerFactory.getTrustManagers
+        logger.info("Using TrustStore: " + trustStoreFileName)
+      }
+
+      sslContext.init(keyManagerFactory.getKeyManagers(), trustManagers, null)
 
       sslContext
     } else {
@@ -54,7 +63,6 @@ object SSLContextFactory {
     "activated, but keystore password is not configured."
 
   def checkRequiredParamsSet(config: Config) {
-    //all other parameters have default values in application.conf
     if (!config.hasPath("keystore")) {
       throw new RuntimeException(MISSING_KEYSTORE_MSG)
     }