diff --git a/README.md b/README.md
index a5109f4e12897..9905e6c9f5751 100644
--- a/README.md
+++ b/README.md
@@ -10,6 +10,7 @@ This is a collaboratively maintained project working on [SPARK-18278](https://is
 
 - [Usage guide](https://apache-spark-on-k8s.github.io/userdocs/) shows how to run the code
 - [Development docs](resource-managers/kubernetes/README.md) shows how to get set up for development
+- [Architecture docs](resource-managers/kubernetes/architecture-docs/) shows the high level architecture of Spark on Kubernetes
 - Code is primarily located in the [resource-managers/kubernetes](resource-managers/kubernetes) folder
 
 ## Why does this fork exist?
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 13930630f2338..29ab454fee253 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 2d29bfc8ea89a..f437084ec6dd5 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index b837c8a2be8a4..b80c1df889731 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 45e128b6e1cfd..ae9ec59180486 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 121b2489fbb72..60549568be0ef 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 31d0d2efc654a..764be312470cf 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index 78eaf8624df93..d0b543137781f 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/conf/kubernetes-resource-staging-server.yaml b/conf/kubernetes-resource-staging-server.yaml
index 025b9b125d9e0..80d59b8091903 100644
--- a/conf/kubernetes-resource-staging-server.yaml
+++ b/conf/kubernetes-resource-staging-server.yaml
@@ -32,7 +32,7 @@ spec:
             name: spark-resource-staging-server-config
       containers:
         - name: spark-resource-staging-server
-          image: kubespark/spark-resource-staging-server:v2.1.0-kubernetes-0.2.0
+          image: kubespark/spark-resource-staging-server:v2.2.0-kubernetes-0.3.0
           resources:
             requests:
               cpu: 100m
diff --git a/conf/kubernetes-shuffle-service.yaml b/conf/kubernetes-shuffle-service.yaml
index 55c170b01a4f5..8ab0b362ea32e 100644
--- a/conf/kubernetes-shuffle-service.yaml
+++ b/conf/kubernetes-shuffle-service.yaml
@@ -20,14 +20,14 @@ kind: DaemonSet
 metadata:
   labels:
     app: spark-shuffle-service
-    spark-version: 2.1.0
+    spark-version: 2.2.0
   name: shuffle
 spec:
   template:
     metadata:
       labels:
         app: spark-shuffle-service
-        spark-version: 2.1.0
+        spark-version: 2.2.0
     spec:
       volumes:
         - name: temp-volume
@@ -38,7 +38,7 @@ spec:
           # This is an official image that is built
           # from the dockerfiles/shuffle directory
           # in the spark distribution.
-          image: kubespark/spark-shuffle:v2.1.0-kubernetes-0.2.0
+          image: kubespark/spark-shuffle:v2.2.0-kubernetes-0.3.0
           imagePullPolicy: IfNotPresent
           volumeMounts:
             - mountPath: '/tmp'
@@ -51,4 +51,4 @@ spec:
              requests:
                cpu: "1"
              limits:
-               cpu: "1"
\ No newline at end of file
+               cpu: "1"
diff --git a/core/pom.xml b/core/pom.xml
index d80b2591d80c3..a152b27284abf 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 1305aeb8c1faf..ed46adcbe9dfb 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -639,7 +639,9 @@ object SparkSubmit extends CommandLineUtils {
       if (args.isPython) {
         childArgs ++= Array("--primary-py-file", args.primaryResource)
         childArgs ++= Array("--main-class", "org.apache.spark.deploy.PythonRunner")
-        childArgs ++= Array("--other-py-files", args.pyFiles)
+        if (args.pyFiles != null) {
+          childArgs ++= Array("--other-py-files", args.pyFiles)
+        }
       } else {
         childArgs ++= Array("--primary-java-resource", args.primaryResource)
         childArgs ++= Array("--main-class", args.mainClass)
diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index 5e23801e15b10..5346e54e62aec 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -17,8 +17,10 @@ cluster, you may setup a test cluster on your local machine using
 * You must have appropriate permissions to create and list [pods](https://kubernetes.io/docs/user-guide/pods/),
 [ConfigMaps](https://kubernetes.io/docs/tasks/configure-pod-container/configmap/) and
 [secrets](https://kubernetes.io/docs/concepts/configuration/secret/) in your cluster. You can verify that
-you can list these resources by running `kubectl get pods` `kubectl get configmap`, and `kubectl get secrets` which
+you can list these resources by running `kubectl get pods`, `kubectl get configmap`, and `kubectl get secrets` which
 should give you a list of pods and configmaps (if any) respectively.
+  * The service account or credentials used by the driver pods must have appropriate permissions
+    as well for editing pod spec.
 * You must have a spark distribution with Kubernetes support. This may be obtained from the
 [release tarball](https://github.com/apache-spark-on-k8s/spark/releases) or by
 [building Spark with Kubernetes support](../resource-managers/kubernetes/README.md#building-spark-with-kubernetes-support).
@@ -36,15 +38,15 @@ If you wish to use pre-built docker images, you may use the images published in
 <tr><th>Component</th><th>Image</th></tr>
 <tr>
   <td>Spark Driver Image</td>
-  <td><code>kubespark/spark-driver:v2.1.0-kubernetes-0.2.0</code></td>
+  <td><code>kubespark/spark-driver:v2.2.0-kubernetes-0.3.0</code></td>
 </tr>
 <tr>
   <td>Spark Executor Image</td>
-  <td><code>kubespark/spark-executor:v2.1.0-kubernetes-0.2.0</code></td>
+  <td><code>kubespark/spark-executor:v2.2.0-kubernetes-0.3.0</code></td>
 </tr>
 <tr>
   <td>Spark Initialization Image</td>
-  <td><code>kubespark/spark-init:v2.1.0-kubernetes-0.2.0</code></td>
+  <td><code>kubespark/spark-init:v2.2.0-kubernetes-0.3.0</code></td>
 </tr>
 </table>
 
@@ -80,9 +82,9 @@ are set up as described above:
       --kubernetes-namespace default \
       --conf spark.executor.instances=5 \
       --conf spark.app.name=spark-pi \
-      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.1.0-kubernetes-0.2.0 \
+      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.2.0-kubernetes-0.3.0 \
       local:///opt/spark/examples/jars/spark_examples_2.11-2.2.0.jar
 
 The Spark master, specified either via passing the `--master` command line argument to `spark-submit` or by setting
@@ -107,6 +109,18 @@ Finally, notice that in the above example we specify a jar with a specific URI w
 the location of the example jar that is already in the Docker image. Using dependencies that are on your machine's local
 disk is discussed below.
 
+When Kubernetes [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) is enabled,
+the `default` service account used by the driver may not have appropriate pod `edit` permissions
+for launching executor pods. We recommend to add another service account, say `spark`, with
+the necessary privilege. For example:
+
+    kubectl create serviceaccount spark
+    kubectl create clusterrolebinding spark-edit --clusterrole edit  \
+        --serviceaccount default:spark --namespace default
+
+With this, one can add `--conf spark.kubernetes.authenticate.driver.serviceAccountName=spark` to
+the spark-submit command line above to specify the service account to use.
+
 ## Dependency Management
 
 Application dependencies that are being submitted from your machine need to be sent to a **resource staging server**
@@ -129,9 +143,9 @@ and then you can compute the value of Pi as follows:
       --kubernetes-namespace default \
       --conf spark.executor.instances=5 \
       --conf spark.app.name=spark-pi \
-      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.1.0-kubernetes-0.2.0 \
+      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.2.0-kubernetes-0.3.0 \
       --conf spark.kubernetes.resourceStagingServer.uri=http://<address-of-any-cluster-node>:31000 \
       examples/jars/spark_examples_2.11-2.2.0.jar
 
@@ -170,9 +184,9 @@ If our local proxy were listening on port 8001, we would have our submission loo
       --kubernetes-namespace default \
       --conf spark.executor.instances=5 \
       --conf spark.app.name=spark-pi \
-      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.1.0-kubernetes-0.2.0 \
+      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.2.0-kubernetes-0.3.0 \
       local:///opt/spark/examples/jars/spark_examples_2.11-2.2.0.jar
 
 Communication between Spark and Kubernetes clusters is performed using the fabric8 kubernetes-client library.
@@ -220,7 +234,7 @@ service because there may be multiple shuffle service instances running in a clu
 a way to target a particular shuffle service.
 
 For example, if the shuffle service we want to use is in the default namespace, and
-has pods with labels `app=spark-shuffle-service` and `spark-version=2.1.0`, we can
+has pods with labels `app=spark-shuffle-service` and `spark-version=2.2.0`, we can
 use those tags to target that particular shuffle service at job launch time. In order to run a job with dynamic allocation enabled,
 the command may then look like the following:
 
@@ -235,7 +249,7 @@ the command may then look like the following:
       --conf spark.dynamicAllocation.enabled=true \
       --conf spark.shuffle.service.enabled=true \
       --conf spark.kubernetes.shuffle.namespace=default \
-      --conf spark.kubernetes.shuffle.labels="app=spark-shuffle-service,spark-version=2.1.0" \
+      --conf spark.kubernetes.shuffle.labels="app=spark-shuffle-service,spark-version=2.2.0" \
       local:///opt/spark/examples/jars/spark_examples_2.11-2.2.0.jar 10 400000 2
 
 ## Advanced
@@ -312,9 +326,9 @@ communicate with the resource staging server over TLS. The trustStore can be set
       --kubernetes-namespace default \
       --conf spark.executor.instances=5 \
       --conf spark.app.name=spark-pi \
-      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.1.0-kubernetes-0.2.0 \
+      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.2.0-kubernetes-0.3.0 \
       --conf spark.kubernetes.resourceStagingServer.uri=https://<address-of-any-cluster-node>:31000 \
       --conf spark.ssl.kubernetes.resourceStagingServer.enabled=true \
       --conf spark.ssl.kubernetes.resourceStagingServer.clientCertPem=/home/myuser/cert.pem \
@@ -768,6 +782,22 @@ from the other deployment modes. See the [configuration page](configuration.html
     <code>myIdentifier</code>. Multiple node selector keys can be added by setting multiple configurations with this prefix.
   </td>
 </tr>
+<tr>
+  <td><code>spark.executorEnv.[EnvironmentVariableName]</code></td> 
+  <td>(none)</td>
+  <td>
+    Add the environment variable specified by <code>EnvironmentVariableName</code> to
+    the Executor process. The user can specify multiple of these to set multiple environment variables.
+  </td>
+</tr>
+<tr>
+  <td><code>spark.kubernetes.driverEnv.[EnvironmentVariableName]</code></td> 
+  <td>(none)</td>
+  <td>
+    Add the environment variable specified by <code>EnvironmentVariableName</code> to
+    the Driver process. The user can specify multiple of these to set multiple environment variables.
+  </td>
+</tr>
 </table>
 
 
diff --git a/examples/pom.xml b/examples/pom.xml
index d2135bf5ff192..3f7814d060526 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index fdf7611936346..f90eb57c64dfe 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml
index 3ae7d254c95a1..8592d85ccbd3f 100644
--- a/external/flume-assembly/pom.xml
+++ b/external/flume-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml
index 89e61be25d8c9..ef0231990c3c9 100644
--- a/external/flume-sink/pom.xml
+++ b/external/flume-sink/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/flume/pom.xml b/external/flume/pom.xml
index 897af93b6b8a2..606116ad0860c 100644
--- a/external/flume/pom.xml
+++ b/external/flume/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/java8-tests/pom.xml b/external/java8-tests/pom.xml
index 5513dfea0f281..586f87d312ec0 100644
--- a/external/java8-tests/pom.xml
+++ b/external/java8-tests/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index d836fffdb56b6..39e1339bbf2a1 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index 10419a1275f73..68ee5be2889b2 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index a0a7b26f2b71e..23e4eef8253b2 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-8-assembly/pom.xml b/external/kafka-0-8-assembly/pom.xml
index 4cd40dbe89689..aa36d4a713e1e 100644
--- a/external/kafka-0-8-assembly/pom.xml
+++ b/external/kafka-0-8-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml
index 57ad12dc70709..af7de9380e271 100644
--- a/external/kafka-0-8/pom.xml
+++ b/external/kafka-0-8/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index 240db2098d1f8..ced0e89bdff16 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index c6a7ceb80c465..0ae897f62dda3 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index cf23a63e23cbe..2744a9fdea489 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/graphx/pom.xml b/graphx/pom.xml
index 8fddc5bfee0f1..e59db233b54d8 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/launcher/pom.xml b/launcher/pom.xml
index 705675af7a01f..9df0c2d79a5fb 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index 155adae6afa61..d120f8f3fc4f0 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 25c38e2281eff..755f1cfe954f0 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/pom.xml b/pom.xml
index 7835481531216..d90ad23752f54 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.11</artifactId>
-  <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+  <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
@@ -2127,7 +2127,7 @@
             <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
             <junitxml>.</junitxml>
             <filereports>SparkTestSuite.txt</filereports>
-            <argLine>-ea -Xmx3g -XX:MaxPermSize=${MaxPermGen} -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraScalaTestArgs}</argLine>
+            <argLine>-ea -Xmx3g -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraScalaTestArgs}</argLine>
             <stderr/>
             <environmentVariables>
               <!--
diff --git a/repl/pom.xml b/repl/pom.xml
index c798170e81ba7..c042a69c14644 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/README.md b/resource-managers/kubernetes/README.md
index fd1ad29eb795d..31b721d193362 100644
--- a/resource-managers/kubernetes/README.md
+++ b/resource-managers/kubernetes/README.md
@@ -14,11 +14,14 @@ important matters to keep in mind when developing this feature.
 
 # Building Spark with Kubernetes Support
 
-To build Spark with Kubernetes support, use the `kubernetes` profile when invoking Maven. For example, to simply compile
-the Kubernetes core implementation module along with its dependencies:
+To build Spark with Kubernetes support, use the `kubernetes` profile when invoking Maven. For example, to simply compile the Kubernetes core implementation module along with its dependencies:
 
     build/mvn compile -Pkubernetes -pl resource-managers/kubernetes/core -am -DskipTests
 
+If this is the first time you compile the Kubernetes core implementation module, run the following command to install the dependencies and compile:
+
+    build/mvn install -Pkubernetes -pl resource-managers/kubernetes/core -am -DskipTests
+
 To build a distribution of Spark with Kubernetes support, use the `dev/make-distribution.sh` script, and add the
 `kubernetes` profile as part of the build arguments. Any other build arguments can be specified as one would expect when
 building Spark normally. For example, to build Spark against Hadoop 2.7 and Kubernetes:
@@ -64,10 +67,12 @@ build/mvn integration-test \
 # Running against an arbitrary cluster
 
 In order to run against any cluster, use the following:
+```sh
 build/mvn integration-test \
     -Pkubernetes -Pkubernetes-integration-tests \
     -pl resource-managers/kubernetes/integration-tests -am
     -DextraScalaTestArgs="-Dspark.kubernetes.test.master=k8s://https://<master> -Dspark.docker.test.driverImage=<driver-image> -Dspark.docker.test.executorImage=<executor-image>"
+```
 
 # Preserve the Minikube VM
 
diff --git a/resource-managers/kubernetes/architecture-docs/external-shuffle-service.md b/resource-managers/kubernetes/architecture-docs/external-shuffle-service.md
new file mode 100644
index 0000000000000..e303800943d42
--- /dev/null
+++ b/resource-managers/kubernetes/architecture-docs/external-shuffle-service.md
@@ -0,0 +1,29 @@
+---
+layout: global
+title: Kubernetes Implementation of the External Shuffle Service
+---
+
+# External Shuffle Service
+
+The `KubernetesExternalShuffleService` was added to allow Spark to use Dynamic Allocation Mode when
+running in Kubernetes. The shuffle service is responsible for persisting shuffle files beyond the
+lifetime of the executors, allowing the number of executors to scale up and down without losing
+computation. 
+
+The implementation of choice is as a DaemonSet that runs a shuffle-service pod on each node.
+Shuffle-service pods and executors pods that land on the same node share disk using hostpath
+volumes. Spark requires that each executor must know the IP address of the shuffle-service pod that
+shares disk with it. 
+
+The user specifies the shuffle service pods they want executors of a particular SparkJob to use
+through two new properties:
+
+* spark.kubernetes.shuffle.service.labels
+* spark.kubernetes.shuffle.namespace
+
+KubernetesClusterSchedulerBackend is aware of shuffle service pods and the node corresponding to
+them in a particular namespace. It uses this data to configure the executor pods to connect with the
+shuffle services that are co-located with them on the same node. 
+
+There is additional logic in the `KubernetesExternalShuffleService` to watch the Kubernetes API,
+detect failures, and proactively cleanup files in those error cases. 
diff --git a/resource-managers/kubernetes/architecture-docs/scheduler-backend.md b/resource-managers/kubernetes/architecture-docs/scheduler-backend.md
new file mode 100644
index 0000000000000..cc43bd7816a8b
--- /dev/null
+++ b/resource-managers/kubernetes/architecture-docs/scheduler-backend.md
@@ -0,0 +1,49 @@
+---
+layout: global
+title: Kubernetes Implementation of the Spark Scheduler Backend
+---
+
+# Scheduler Backend
+
+The general idea is to run Spark drivers and executors inside Kubernetes [Pods](https://kubernetes.io/docs/concepts/workloads/pods/pod/). 
+Pods are a co-located and co-scheduled group of one or more containers run in a shared context. The main component is KubernetesClusterSchedulerBackend, 
+an implementation of CoarseGrainedSchedulerBackend, which manages allocating and destroying executors via the Kubernetes API. 
+There are auxiliary and optional components: `ResourceStagingServer` and `KubernetesExternalShuffleService`, which serve specific purposes described further below.
+
+The scheduler backend is invoked in the driver associated with a particular job. The driver may run outside the cluster (client mode) or within (cluster mode). 
+The scheduler backend manages [pods](http://kubernetes.io/docs/user-guide/pods/) for each executor. 
+The executor code is running within a Kubernetes pod, but remains unmodified and unaware of the orchestration layer. 
+When a job is running, the scheduler backend configures and creates executor pods with the following properties:
+
+- The pod's container runs a pre-built Docker image containing a Spark distribution (with Kubernetes integration) and 
+invokes the Java runtime with the CoarseGrainedExecutorBackend main class.
+- The scheduler backend specifies environment variables on the executor pod to configure its runtime, p
+articularly for its JVM options, number of cores, heap size, and the driver's hostname.
+- The executor container has [resource limits and requests](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) 
+that are set in accordance to the resource limits specified in the Spark configuration (executor.cores and executor.memory in the application's SparkConf)
+- The executor pods may also be launched into a particular [Kubernetes namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/%5C), 
+or target a particular subset of nodes in the Kubernetes cluster, based on the Spark configuration supplied.
+
+## Requesting Executors
+
+Spark requests for new executors through the `doRequestTotalExecutors(numExecutors: Int)` method. 
+The scheduler backend keeps track of the request made by Spark core for the number of executors.
+
+A separate kubernetes-pod-allocator thread handles the creation of new executor pods with appropriate throttling and monitoring. 
+This indirection is required because the Kubernetes API Server accepts requests for new executor pods optimistically, with the 
+anticipation of being able to eventually run them. However, it is undesirable to have a very large number of pods that cannot be 
+scheduled and stay pending within the cluster. Hence, the kubernetes-pod-allocator uses the Kubernetes API to make a decision to 
+submit new requests for executors based on whether previous pod creation requests have completed. This gives us control over how 
+fast a job scales up (which can be configured), and helps prevent Spark jobs from DOS-ing the Kubernetes API server with pod creation requests.
+
+## Destroying Executors
+
+Spark requests deletion of executors through the `doKillExecutors(executorIds: List[String])`
+method.
+
+The inverse behavior is required in the implementation of doKillExecutors(). When the executor
+allocation manager desires to remove executors from the application, the scheduler should find the
+pods that are running the appropriate executors, and tell the API server to stop these pods.
+It's worth noting that this code does not have to decide on the executors that should be
+removed. When `doKillExecutors()` is called, the executors that are to be removed have already been
+selected by the CoarseGrainedSchedulerBackend and ExecutorAllocationManager.
diff --git a/resource-managers/kubernetes/architecture-docs/submission-client.md b/resource-managers/kubernetes/architecture-docs/submission-client.md
new file mode 100644
index 0000000000000..022cf15c60be7
--- /dev/null
+++ b/resource-managers/kubernetes/architecture-docs/submission-client.md
@@ -0,0 +1,231 @@
+---
+layout: global
+title: Implementation of Submitting Applications to Kubernetes
+---
+
+
+Similarly to YARN and Standalone mode, it is common for Spark applications to be deployed on Kubernetes through the
+`spark-submit` process. Applications are deployed on Kubernetes via sending YAML files to the Kubernetes API server.
+These YAML files declare the structure and behavior of the processes that will be run. However, such a declarative
+approach to application deployment differs considerably from how Spark applications are deployed via the `spark-submit`
+API. There are contracts provided by `spark-submit` that should work in Kubernetes in a consistent manner to the other
+cluster managers that `spark-submit` can deploy on.
+
+This document outlines the design of the **Kubernetes submission client**, which effectively serves as a *translation
+of options provided in spark-submit to a specification of one or more Kubernetes API resources that represent the
+Spark driver*.
+
+# Entry Point
+
+As with the other cluster managers, the user's invocation of `spark-submit` will eventually delegate to running
+`org.apache.spark.deploy.SparkSubmit#submit`. This method calls a main method that handles the submission logic
+for a specific type of cluster manager. The top level entry point for the Kubernetes submission logic is in
+`org.apache.spark.deploy.kubernetes.submit.Client#main()`.
+
+# Driver Configuration Steps
+
+In order to render submission parameters into the final Kubernetes driver pod specification, and do it in a scalable
+manner, the submission client breaks pod construction down into a
+series of configuration steps, each of which is responsible for handling some specific aspect of configuring the driver.
+A top level component then iterates through all of the steps to produce a final set of Kubernetes resources that are
+then deployed on the cluster.
+
+## Interface Definitions
+
+More formally, a configuration step must implement the following trait:
+
+```scala
+package org.apache.spark.deploy.kubernetes.submit.submitsteps
+
+/**
+ * Represents a step in preparing the Kubernetes driver.
+ */
+private[spark] trait DriverConfigurationStep {
+
+  /**
+   * Apply some transformation to the previous state of the driver to add a new feature to it.
+   */
+  def configureDriver(driverSpec: KubernetesDriverSpec): KubernetesDriverSpec
+}
+```
+
+A `DriverConfigurationStep` is thus a function that transforms a `KubernetesDriverSpec` into another
+`KubernetesDriverSpec`, by taking the original specification and making additions to the specification in accordance to
+the specific feature that step is responsible for. A `KubernetesDriverSpec` is a data structure with the following
+properties:
+
+```scala
+private[spark] case class KubernetesDriverSpec(
+    driverPod: Pod,
+    driverContainer: Container,
+    otherKubernetesResources: Seq[HasMetadata],
+    driverSparkConf: SparkConf)
+```
+
+The `Pod` and `Container` classes are Java representations of Kubernetes pods and containers respectively, and the
+`HasMetadata` type corresponds to an arbitrary Kubernetes API resource such as a `Secret` or a `ConfigMap`. Kubernetes
+primitives are represented using an [open-source Java Kubernetes client](https://github.com/fabric8io/kubernetes-client).
+The `otherKubernetesResources` field represents Kubernetes resources that are required by the Spark application. For
+example, the driver may require a `ConfigMap` or `Secret` resource to be created that will be mounted into the driver
+container.
+
+## Requirements for Configuration Steps
+
+Configuration steps must be *independent*. A given configuration step should not be opinionated about the other
+configuration steps that are executed before or after it. By extension, configuration steps should be *strictly
+additive*. A given configuration step should not attempt to mutate an existing field nor remove fields set in the
+input driver specification.
+
+## Composition of Configuration Steps
+
+Finally, configuration steps are wired together by an **orchestrator**. The orchestrator effectively translates the
+parameters sent to `spark-submit` into the set of steps required to configure the final `KubernetesDriverSpec`. The
+top level submission client takes the final `KubernetesDriverSpec` object and builds the final requests to the
+Kubernetes API server to deploy the Kubernetes resources that comprise the Spark driver. The top level submission
+process can thus be expressed as follows in pseudo-code with roughly Scala syntax:
+
+```scala
+def runApplication(sparkSubmitArguments: SparkSubmitArguments) {
+  val initialSpec = createEmptyDriverSpec()
+  val orchestrator = new DriverConfigurationStepsOrchestrator(sparkSubmitArguments)
+  val steps = orchestrator.getSubmissionSteps()
+  var currentSpec = initialSpec
+  // iteratively apply the configuration steps to build up the pod spec:
+  for (step <- steps) {
+    currentSpec = step.configureDriver(currentSpec)
+  }
+  // Put the container in the pod spec
+  val resolvedPod = attachContainer(currentSpec.driverPod, currentSpec.driverContainer)
+  kubernetes.create(resolvedPod + currentSpec.otherKubernetesResources)
+}
+```
+
+## Writing a New Configuration Step
+
+All configuration steps should be placed in the `org.apache.spark.deploy.kubernetes.submit.submitsteps` package.
+Examples of other configuration steps can be found in this package as well. Ensure that the new configuration step is
+returned in `org.apache.spark.deploy.kubernetes.submit.DriverConfigurationStepsOrchestrator#getAllConfigurationSteps()`.
+
+# Dependency Management
+
+Spark applications typically depend on binaries and various configuration files which are hosted in various locations.
+Kubernetes applications typically bundle binary dependencies such as jars inside Docker images. However, Spark's API
+fundamentally allows dependencies to be provided from many other locations, including the submitter's local disk.
+These dependencies have to be deployed into the driver and executor containers before they run. This is challenging
+because unlike Hadoop YARN which requires co-deployment with an HDFS cluster, Kubernetes clusters do not have a
+large-scale persistent storage layer that would be available across every Kubernetes cluster.
+
+## Resource Staging Server
+
+The *resource staging server* is a lightweight daemon that serves as a file store for application dependencies. It has
+two endpoints which effectively correspond to putting files into the server and getting files out of the server. When
+files are put into the server, the server returns a unique identifier and a secret token in the response to the client.
+This identifier and secret token must be provided when a client makes a request to retrieve the files that were uploaded
+to the server.
+
+### Resource Staging Server API Definition
+
+The resource staging server has the following Scala API which would then be translated into HTTP endpoints via Jetty and
+JAX-RS. Associated structures passed as input and output are also defined below:
+
+```scala
+private[spark] trait ResourceStagingService {
+
+  def uploadResources(resources: InputStream, resourcesOwner: StagedResourcesOwner): SubmittedResourceIdAndSecret
+  def downloadResources(resourceId: String, resourceSecret: String): StreamingOutput
+}
+
+case class StagedResourcesOwner(
+    ownerNamespace: String,
+    ownerLabels: Map[String, String],
+    ownerType: StagedResourcesOwnerType.OwnerType)
+    
+// Pseudo-code to represent an enum
+enum StagedResourcesOwnerType.OwnerType = { Pod }
+
+case class SubmittedResourceIdAndSecret(resourceId: String, resourceSecret: String)
+```
+
+Clients that send resources to the server do so in a streaming manner so that both the server and the client do not
+need to hold the entire resource bundle in memory. Aside from the notion of the `StagedResourcesOwner` that is provided
+on uploads and not for downloads, uploading is symmetrical to downloading. The significance of the
+`StagedResourcesOwner` is discussed below.
+
+### Cleaning Up Stale Resources
+
+The resource staging server is built to provide resources for the pods and containers in a Kubernetes cluster. These
+pods are ephemeral, so at some point there will be no need for the resources that were sent for a specific application.
+Clients indicate the set of resources that would be using a given resource bundle by providing a description of the
+resource's "owner". The `StagedResourceOwner` is this description, defining the owner as a Kubernetes API object in
+a given namespace and having a specific set of labels.
+
+The resource staging server keeps track of the resources that were sent to it. When the resource is first uploaded, it
+is marked as "unused". If the resource remains unused for a period of time, it is cleaned up. A resource is marked as
+"used" when a request is made to download it. After that, the server periodically checks the API server to see if any
+Kubernetes API objects exist that match the description of the owner. If no such objects exist, then resource staging
+server cleans up the uploaded resource. See `org.apache.spark.deploy.rest.kubernetes.StagedResourcesCleaner` for the
+code that manages the resource's lifecycle.
+
+A resource owner can currently only be a pod, but hypothetically one could want to tie the lifetime of a resource to the
+lifetime of many pods under a higher level Kubernetes object like a Deployment or a StatefulSet, all of which depend on
+the uploaded resource. The resource staging server's API can be extended to tie ownership of a resource to any
+Kubernetes API object type, as long as we update the `StagedResourcesOwnerType` enumeration accordingly.
+
+### Usage in Spark
+
+Spark-submit supports adding jars and files by passing `--jars` and `--files` to `spark-submit` respectively. The spark
+configurations `spark.jars` and `spark.files` can also be set to provide this information. The submission client
+determines the list of jars and files that the application needs, and it determines if any of them are files being sent
+from the submitter's local machine. If any files are being sent from the local machine, the user must have specified a
+URL for the resource staging server to send the files to.
+
+Local jars and files are compacted into a tarball which are then uploaded to the resource staging server. The submission
+client then knows the secret token that the driver and executors must use to download the files again. These secrets
+are mounted into an [init-container](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/)
+that runs before the driver and executor processes run, and the init-container
+downloads the uploaded resources from the resource staging server.
+
+### Other Considered Alternatives
+
+The resource staging server was considered the best option among other alternative solutions to this problem.
+
+A first implementation effectively included the resource staging server in the driver container itself. The driver
+container ran a custom command that opened an HTTP endpoint and waited for the submission client to send resources to
+it. The server would then run the driver application after it had received the resources from the user's local
+machine. The problem with this approach is that the submission client needs to deploy the driver in such a way that the
+driver itself would be reachable from outside of the cluster, but it is difficult for an automated framework which is
+not aware of the cluster's configuration to expose an arbitrary pod in a generic way. The resource staging server allows
+a cluster administrator to expose the resource staging server in a manner that makes sense for their cluster, such as
+with an Ingress or with a NodePort service.
+
+It is also impossible to use Kubernetes API objects like Secrets or ConfigMaps to store application binaries. These
+objects require their contents to be small so that they can fit in etcd.
+
+Finally, as mentioned before, the submission client should not be opinionated about storing dependencies in a
+distributed storage system like HDFS, because not all Kubernetes clusters will have the same types of persistent storage
+layers. Spark supports fetching jars directly from distributed storage layers though, so users can feel free to manually
+push their dependencies to their appropriate systems and refer to them by their remote URIs in the submission request.
+
+## Init-Containers
+
+The driver pod and executor pods both use [init-containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/)
+to localize resources before the driver and
+executor processes launch. As mentioned before, the init-container fetches dependencies from the resource staging
+server. However, even if the resource staging server is not being used, files still need to be localized from remote
+locations such as HDFS clusters or HTTP file servers. The init-container will fetch these dependencies accordingly as
+well.
+
+Init-containers were preferred over fetching the dependencies in the main container primarily because this allows the
+main container's runtime commands to be simplified. Using init-containers to fetch these remote dependencies allows the
+main image command to simply be an invocation of Java that runs the user's main class directly. The execution of the
+file localizer process can also be shared by both the driver and the executor images without needing to be copied
+into both image commands. Finally, it becomes easier to debug localization failures as they will be easily spotted as
+being a failure in the pod's initialization lifecycle phase.
+
+# Future Work
+
+* The driver's pod specification should be highly customizable, to the point where users may want to specify a template
+pod spec in a YAML file: https://github.com/apache-spark-on-k8s/spark/issues/38.
+* The resource staging server can be backed by a distributed file store like HDFS to improve robustness and scalability.
+* Additional driver bootstrap steps need to be added to support communication with Kerberized HDFS clusters:
+  * https://github.com/apache-spark-on-k8s/spark/pull/391
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index 3feecb09230bc..dfbf8adc5bc7b 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/config.scala
index e746410772f62..5bd61328f3a0b 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/config.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/config.scala
@@ -138,6 +138,8 @@ package object config extends Logging {
       .stringConf
       .createOptional
 
+  private[spark] val KUBERNETES_DRIVER_ENV_KEY = "spark.kubernetes.driverEnv."
+
   private[spark] val KUBERNETES_DRIVER_ANNOTATIONS =
     ConfigBuilder("spark.kubernetes.driver.annotations")
       .doc("Custom annotations that will be added to the driver pod. This should be a" +
@@ -460,6 +462,22 @@ package object config extends Logging {
       .timeConf(TimeUnit.MINUTES)
       .createWithDefault(5)
 
+  private[spark] val EXECUTOR_SUBMITTED_SMALL_FILES_SECRET =
+    ConfigBuilder("spark.kubernetes.mountdependencies.smallfiles.executor.secretName")
+      .doc("Name of the secret that should be mounted into the executor containers for" +
+        " distributing submitted small files without the resource staging server.")
+      .internal()
+      .stringConf
+      .createOptional
+
+  private[spark] val EXECUTOR_SUBMITTED_SMALL_FILES_SECRET_MOUNT_PATH =
+    ConfigBuilder("spark.kubernetes.mountdependencies.smallfiles.executor.secretMountPath")
+      .doc(s"Mount path in the executors for the secret given by" +
+        s" ${EXECUTOR_SUBMITTED_SMALL_FILES_SECRET.key}")
+      .internal()
+      .stringConf
+      .createOptional
+
   private[spark] val EXECUTOR_INIT_CONTAINER_CONFIG_MAP =
     ConfigBuilder("spark.kubernetes.initcontainer.executor.configmapname")
       .doc("Name of the config map to use in the init-container that retrieves submitted files" +
@@ -503,6 +521,19 @@ package object config extends Logging {
       .stringConf
       .createOptional
 
+  private[spark] val KUBERNETES_DRIVER_CLUSTER_NODENAME_DNS_LOOKUP_ENABLED =
+    ConfigBuilder("spark.kubernetes.driver.hdfslocality.clusterNodeNameDNSLookup.enabled")
+      .doc("Whether or not HDFS locality support code should look up DNS for full hostnames of" +
+        " cluster nodes. In some K8s clusters, notably GKE, cluster node names are short" +
+        " hostnames, and so comparing them against HDFS datanode hostnames always fail. To fix," +
+        " enable this flag. This is disabled by default because DNS lookup can be expensive." +
+        " The driver can slow down and fail to respond to executor heartbeats in time." +
+        " If enabling this flag, make sure your DNS server has enough capacity" +
+        " for the workload.")
+      .internal()
+      .booleanConf
+      .createWithDefault(false)
+
   private[spark] val KUBERNETES_EXECUTOR_LIMIT_CORES =
     ConfigBuilder("spark.kubernetes.executor.limit.cores")
       .doc("Specify the hard cpu limit for a single executor pod")
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/constants.scala
index f39a3fc2f4bb5..3faf387aadfc0 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/constants.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/constants.scala
@@ -76,6 +76,8 @@ package object constants {
   private[spark] val ENV_MOUNTED_FILES_DIR = "SPARK_MOUNTED_FILES_DIR"
   private[spark] val ENV_PYSPARK_FILES = "PYSPARK_FILES"
   private[spark] val ENV_PYSPARK_PRIMARY = "PYSPARK_PRIMARY"
+  private[spark] val ENV_JAVA_OPT_PREFIX = "SPARK_JAVA_OPT_"
+  private[spark] val ENV_MOUNTED_FILES_FROM_SECRET_DIR = "SPARK_MOUNTED_FILES_FROM_SECRET_DIR"
   private[spark] val ENV_HADOOP_TOKEN_FILE_LOCATION = "HADOOP_TOKEN_FILE_LOCATION"
 
   // Bootstrapping dependencies with the init-container
@@ -99,10 +101,13 @@ package object constants {
   private[spark] val DEFAULT_SHUFFLE_MOUNT_NAME = "shuffle"
   private[spark] val INIT_CONTAINER_SECRET_VOLUME_NAME = "spark-init-secret"
 
+  // Bootstrapping dependencies via a secret
+  private[spark] val MOUNTED_SMALL_FILES_SECRET_MOUNT_PATH = "/etc/spark-submitted-files"
+
   // Miscellaneous
   private[spark] val ANNOTATION_EXECUTOR_NODE_AFFINITY = "scheduler.alpha.kubernetes.io/affinity"
   private[spark] val DRIVER_CONTAINER_NAME = "spark-kubernetes-driver"
   private[spark] val KUBERNETES_MASTER_INTERNAL_URL = "https://kubernetes.default.svc"
   private[spark] val MEMORY_OVERHEAD_FACTOR = 0.10
-  private[spark] val MEMORY_OVERHEAD_MIN = 384L
+  private[spark] val MEMORY_OVERHEAD_MIN_MIB = 384L
 }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/Client.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/Client.scala
index 2fa9b416330e5..1901f1198a84f 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/Client.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/Client.scala
@@ -18,9 +18,10 @@ package org.apache.spark.deploy.kubernetes.submit
 
 import java.util.{Collections, UUID}
 
-import io.fabric8.kubernetes.api.model.{ContainerBuilder, OwnerReferenceBuilder, PodBuilder}
+import io.fabric8.kubernetes.api.model.{ContainerBuilder, EnvVar, EnvVarBuilder, OwnerReferenceBuilder, PodBuilder}
 import io.fabric8.kubernetes.client.KubernetesClient
 import scala.collection.mutable
+import scala.collection.JavaConverters._
 
 import org.apache.spark.SparkConf
 import org.apache.spark.deploy.kubernetes.config._
@@ -92,18 +93,21 @@ private[spark] class Client(
       currentDriverSpec = nextStep.configureDriver(currentDriverSpec)
     }
     val resolvedDriverJavaOpts = currentDriverSpec
-      .driverSparkConf
-      // We don't need this anymore since we just set the JVM options on the environment
-      .remove(org.apache.spark.internal.config.DRIVER_JAVA_OPTIONS)
-      .getAll
-      .map {
-        case (confKey, confValue) => s"-D$confKey=$confValue"
-      }.mkString(" ") + driverJavaOptions.map(" " + _).getOrElse("")
+        .driverSparkConf
+        // We don't need this anymore since we just set the JVM options on the environment
+        .remove(org.apache.spark.internal.config.DRIVER_JAVA_OPTIONS)
+        .getAll
+        .map {
+          case (confKey, confValue) => s"-D$confKey=$confValue"
+        } ++ driverJavaOptions.map(Utils.splitCommandString).getOrElse(Seq.empty)
+    val driverJavaOptsEnvs: Seq[EnvVar] = resolvedDriverJavaOpts.zipWithIndex.map {
+      case (option, index) => new EnvVarBuilder()
+          .withName(s"$ENV_JAVA_OPT_PREFIX$index")
+          .withValue(option)
+          .build()
+    }
     val resolvedDriverContainer = new ContainerBuilder(currentDriverSpec.driverContainer)
-      .addNewEnv()
-        .withName(ENV_DRIVER_JAVA_OPTS)
-        .withValue(resolvedDriverJavaOpts)
-        .endEnv()
+      .addAllToEnv(driverJavaOptsEnvs.asJava)
       .build()
     val resolvedDriverPod = new PodBuilder(currentDriverSpec.driverPod)
       .editSpec()
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/DriverConfigurationStepsOrchestrator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/DriverConfigurationStepsOrchestrator.scala
index 06f1547e6a5b1..1e9f5863a0d96 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/DriverConfigurationStepsOrchestrator.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/DriverConfigurationStepsOrchestrator.scala
@@ -100,41 +100,78 @@ private[spark] class DriverConfigurationStepsOrchestrator(
         Option(new PythonStep(mainPyResource, additionalPythonFiles, filesDownloadPath))
       case _ => Option.empty[DriverConfigurationStep]
     }
-    val initContainerBootstrapStep = if ((sparkJars ++ sparkFiles).exists { uri =>
-      Option(Utils.resolveURI(uri).getScheme).getOrElse("file") != "local"
-    }) {
-      val initContainerConfigurationStepsOrchestrator =
-          new InitContainerConfigurationStepsOrchestrator(
-              namespace,
-              kubernetesResourceNamePrefix,
-              sparkJars,
+
+    val (localFilesDownloadPath, submittedDependenciesBootstrapSteps) =
+      if (areAnyFilesNonContainerLocal(sparkJars ++ sparkFiles)) {
+        val (submittedLocalFilesDownloadPath,
+            sparkFilesResolvedFromInitContainer,
+            mountSmallFilesWithoutInitContainerStep) =
+          // If the resource staging server is specified, submit all local files through that.
+          submissionSparkConf.get(RESOURCE_STAGING_SERVER_URI).map { _ =>
+            (filesDownloadPath, sparkFiles, Option.empty[DriverConfigurationStep])
+          }.getOrElse {
+            // Else - use a small files bootstrap that submits the local files via a secret.
+            // Then, indicate to the outer block that the init-container should not handle
+            // those local files simply by filtering them out.
+            val sparkFilesWithoutLocal = KubernetesFileUtils.getNonSubmitterLocalFiles(sparkFiles)
+            val smallFilesSecretName = s"${kubernetesAppId}-submitted-files"
+            val mountSmallFilesBootstrap = new MountSmallFilesBootstrapImpl(
+                smallFilesSecretName, MOUNTED_SMALL_FILES_SECRET_MOUNT_PATH)
+            val mountSmallLocalFilesStep = new MountSmallLocalFilesStep(
               sparkFiles,
-              jarsDownloadPath,
-              filesDownloadPath,
-              dockerImagePullPolicy,
-              allDriverLabels,
+              smallFilesSecretName,
+              MOUNTED_SMALL_FILES_SECRET_MOUNT_PATH,
+              mountSmallFilesBootstrap)
+            (MOUNTED_SMALL_FILES_SECRET_MOUNT_PATH,
+              sparkFilesWithoutLocal.toArray,
+              Some(mountSmallLocalFilesStep))
+          }
+
+        val initContainerBootstrapStep =
+          if (areAnyFilesNonContainerLocal(sparkJars ++ sparkFilesResolvedFromInitContainer)) {
+            val initContainerConfigurationStepsOrchestrator =
+              new InitContainerConfigurationStepsOrchestrator(
+                namespace,
+                kubernetesResourceNamePrefix,
+                sparkJars,
+                sparkFilesResolvedFromInitContainer,
+                jarsDownloadPath,
+                filesDownloadPath,
+                dockerImagePullPolicy,
+                allDriverLabels,
+                initContainerConfigMapName,
+                INIT_CONTAINER_CONFIG_MAP_KEY,
+                submissionSparkConf)
+            val initContainerConfigurationSteps =
+              initContainerConfigurationStepsOrchestrator.getAllConfigurationSteps()
+            Some(new InitContainerBootstrapStep(initContainerConfigurationSteps,
               initContainerConfigMapName,
-              INIT_CONTAINER_CONFIG_MAP_KEY,
-              submissionSparkConf)
-      val initContainerConfigurationSteps =
-          initContainerConfigurationStepsOrchestrator.getAllConfigurationSteps()
-      Some(new InitContainerBootstrapStep(initContainerConfigurationSteps,
-        initContainerConfigMapName,
-        INIT_CONTAINER_CONFIG_MAP_KEY))
+              INIT_CONTAINER_CONFIG_MAP_KEY))
+        } else Option.empty[DriverConfigurationStep]
+        (submittedLocalFilesDownloadPath,
+          mountSmallFilesWithoutInitContainerStep.toSeq ++
+            initContainerBootstrapStep.toSeq)
     } else {
-      Option.empty[DriverConfigurationStep]
+      (filesDownloadPath, Seq.empty[DriverConfigurationStep])
     }
     val dependencyResolutionStep = new DependencyResolutionStep(
       sparkJars,
       sparkFiles,
       jarsDownloadPath,
-      filesDownloadPath)
+      localFilesDownloadPath)
     Seq(
       initialSubmissionStep,
       kubernetesCredentialsStep,
       hadoopCredentialsStep,
       dependencyResolutionStep) ++
-      initContainerBootstrapStep.toSeq ++
+      submittedDependenciesBootstrapSteps ++
       pythonStep.toSeq
   }
+
+  private def areAnyFilesNonContainerLocal(files: Seq[String]): Boolean = {
+    files.exists { uri =>
+      Option(Utils.resolveURI(uri).getScheme).getOrElse("file") != "local"
+    }
+  }
+
 }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/MountSmallFilesBootstrap.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/MountSmallFilesBootstrap.scala
new file mode 100644
index 0000000000000..79919b511ec37
--- /dev/null
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/MountSmallFilesBootstrap.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.deploy.kubernetes.submit
+
+import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder, Pod, PodBuilder}
+
+import org.apache.spark.deploy.kubernetes.constants._
+
+private[spark] trait MountSmallFilesBootstrap {
+  def mountSmallFilesSecret(pod: Pod, container: Container): (Pod, Container)
+}
+
+private[spark] class MountSmallFilesBootstrapImpl(
+    secretName: String, secretMountPath: String) extends MountSmallFilesBootstrap {
+  def mountSmallFilesSecret(pod: Pod, container: Container): (Pod, Container) = {
+    val resolvedPod = new PodBuilder(pod)
+      .editOrNewSpec()
+        .addNewVolume()
+          .withName("submitted-files")
+          .withNewSecret()
+            .withSecretName(secretName)
+            .endSecret()
+          .endVolume()
+        .endSpec()
+      .build()
+    val resolvedContainer = new ContainerBuilder(container)
+      .addNewEnv()
+        .withName(ENV_MOUNTED_FILES_FROM_SECRET_DIR)
+        .withValue(secretMountPath)
+        .endEnv()
+      .addNewVolumeMount()
+        .withName("submitted-files")
+        .withMountPath(secretMountPath)
+        .endVolumeMount()
+      .build()
+    (resolvedPod, resolvedContainer)
+  }
+}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStep.scala
index b3f509b44054e..19784e8a8074f 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStep.scala
@@ -46,13 +46,13 @@ private[spark] class BaseDriverConfigurationStep(
   private val driverLimitCores = submissionSparkConf.get(KUBERNETES_DRIVER_LIMIT_CORES)
 
   // Memory settings
-  private val driverMemoryMb = submissionSparkConf.get(
+  private val driverMemoryMiB = submissionSparkConf.get(
       org.apache.spark.internal.config.DRIVER_MEMORY)
-  private val memoryOverheadMb = submissionSparkConf
+  private val memoryOverheadMiB = submissionSparkConf
       .get(KUBERNETES_DRIVER_MEMORY_OVERHEAD)
-      .getOrElse(math.max((MEMORY_OVERHEAD_FACTOR * driverMemoryMb).toInt,
-          MEMORY_OVERHEAD_MIN))
-  private val driverContainerMemoryWithOverhead = driverMemoryMb + memoryOverheadMb
+      .getOrElse(math.max((MEMORY_OVERHEAD_FACTOR * driverMemoryMiB).toInt,
+          MEMORY_OVERHEAD_MIN_MIB))
+  private val driverContainerMemoryWithOverheadMiB = driverMemoryMiB + memoryOverheadMiB
   private val driverDockerImage = submissionSparkConf.get(DRIVER_DOCKER_IMAGE)
 
   override def configureDriver(
@@ -72,6 +72,13 @@ private[spark] class BaseDriverConfigurationStep(
     require(!driverCustomAnnotations.contains(SPARK_APP_NAME_ANNOTATION),
         s"Annotation with key $SPARK_APP_NAME_ANNOTATION is not allowed as it is reserved for" +
             s" Spark bookkeeping operations.")
+
+    val driverCustomEnvs = submissionSparkConf.getAllWithPrefix(KUBERNETES_DRIVER_ENV_KEY).toSeq
+      .map(env => new EnvVarBuilder()
+        .withName(env._1)
+        .withValue(env._2)
+        .build())
+
     val allDriverAnnotations = driverCustomAnnotations ++ Map(SPARK_APP_NAME_ANNOTATION -> appName)
     val nodeSelector = ConfigurationUtils.parsePrefixedKeyValuePairs(
       submissionSparkConf, KUBERNETES_NODE_SELECTOR_PREFIX, "node selector")
@@ -79,10 +86,10 @@ private[spark] class BaseDriverConfigurationStep(
       .withAmount(driverCpuCores)
       .build()
     val driverMemoryQuantity = new QuantityBuilder(false)
-      .withAmount(s"${driverMemoryMb}M")
+      .withAmount(s"${driverMemoryMiB}Mi")
       .build()
     val driverMemoryLimitQuantity = new QuantityBuilder(false)
-      .withAmount(s"${driverContainerMemoryWithOverhead}M")
+      .withAmount(s"${driverContainerMemoryWithOverheadMiB}Mi")
       .build()
     val maybeCpuLimitQuantity = driverLimitCores.map { limitCores =>
       ("cpu", new QuantityBuilder(false).withAmount(limitCores).build())
@@ -91,10 +98,11 @@ private[spark] class BaseDriverConfigurationStep(
       .withName(DRIVER_CONTAINER_NAME)
       .withImage(driverDockerImage)
       .withImagePullPolicy(dockerImagePullPolicy)
+      .addAllToEnv(driverCustomEnvs.asJava)
       .addToEnv(driverExtraClasspathEnv.toSeq: _*)
       .addNewEnv()
         .withName(ENV_DRIVER_MEMORY)
-        .withValue(driverContainerMemoryWithOverhead + "m")
+        .withValue(driverContainerMemoryWithOverheadMiB + "M") // JVM treats the "M" unit as "Mi"
         .endEnv()
       .addNewEnv()
         .withName(ENV_DRIVER_MAIN_CLASS)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/DependencyResolutionStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/DependencyResolutionStep.scala
index dddc62410d6c9..090240420119e 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/DependencyResolutionStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/DependencyResolutionStep.scala
@@ -23,6 +23,7 @@ import io.fabric8.kubernetes.api.model.ContainerBuilder
 
 import org.apache.spark.deploy.kubernetes.constants._
 import org.apache.spark.deploy.kubernetes.submit.KubernetesFileUtils
+import org.apache.spark.deploy.kubernetes.submit.submitsteps.initcontainer.InitContainerConfigurationStep
 import org.apache.spark.util.Utils
 
 /**
@@ -36,11 +37,12 @@ private[spark] class DependencyResolutionStep(
     sparkJars: Seq[String],
     sparkFiles: Seq[String],
     jarsDownloadPath: String,
-    filesDownloadPath: String) extends DriverConfigurationStep {
+    localFilesDownloadPath: String) extends DriverConfigurationStep {
 
   override def configureDriver(driverSpec: KubernetesDriverSpec): KubernetesDriverSpec = {
     val resolvedSparkJars = KubernetesFileUtils.resolveSubmittedUris(sparkJars, jarsDownloadPath)
-    val resolvedSparkFiles = KubernetesFileUtils.resolveSubmittedUris(sparkFiles, filesDownloadPath)
+    val resolvedSparkFiles = KubernetesFileUtils.resolveSubmittedUris(
+        sparkFiles, localFilesDownloadPath)
     val sparkConfResolvedSparkDependencies = driverSpec.driverSparkConf.clone()
     if (resolvedSparkJars.nonEmpty) {
       sparkConfResolvedSparkDependencies.set("spark.jars", resolvedSparkJars.mkString(","))
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/DriverKubernetesCredentialsStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/DriverKubernetesCredentialsStep.scala
index 70a108edc8678..74df2de480bf4 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/DriverKubernetesCredentialsStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/DriverKubernetesCredentialsStep.scala
@@ -44,6 +44,7 @@ private[spark] class DriverKubernetesCredentialsStep(
       s"$APISERVER_AUTH_DRIVER_MOUNTED_CONF_PREFIX.$CLIENT_CERT_FILE_CONF_SUFFIX")
   private val maybeMountedCaCertFile = submissionSparkConf.getOption(
       s"$APISERVER_AUTH_DRIVER_MOUNTED_CONF_PREFIX.$CA_CERT_FILE_CONF_SUFFIX")
+  private val driverServiceAccount = submissionSparkConf.get(KUBERNETES_SERVICE_ACCOUNT_NAME)
 
   override def configureDriver(driverSpec: KubernetesDriverSpec): KubernetesDriverSpec = {
     val driverSparkConf = driverSpec.driverSparkConf.clone()
@@ -81,7 +82,16 @@ private[spark] class DriverKubernetesCredentialsStep(
             .endVolume()
           .endSpec()
         .build()
-    }.getOrElse(driverSpec.driverPod)
+    }.getOrElse(
+      driverServiceAccount.map { account =>
+          new PodBuilder(driverSpec.driverPod)
+            .editOrNewSpec()
+              .withServiceAccount(account)
+              .withServiceAccountName(account)
+              .endSpec()
+            .build()
+      }.getOrElse(driverSpec.driverPod)
+    )
     val driverContainerWithMountedSecretVolume = kubernetesCredentialsSecret.map { secret =>
       new ContainerBuilder(driverSpec.driverContainer)
         .addNewVolumeMount()
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/InitContainerBootstrapStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/InitContainerBootstrapStep.scala
index 29cad18c484c0..b4248338cc8de 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/InitContainerBootstrapStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/InitContainerBootstrapStep.scala
@@ -33,32 +33,32 @@ private[spark] class InitContainerBootstrapStep(
 
   override def configureDriver(driverSpec: KubernetesDriverSpec): KubernetesDriverSpec = {
     var currentInitContainerSpec = InitContainerSpec(
-        initContainerProperties = Map.empty[String, String],
-        additionalDriverSparkConf = Map.empty[String, String],
-        initContainer = new ContainerBuilder().build(),
-        driverContainer = driverSpec.driverContainer,
-        podToInitialize = driverSpec.driverPod,
-        initContainerDependentResources = Seq.empty[HasMetadata])
+      initContainerProperties = Map.empty[String, String],
+      additionalDriverSparkConf = Map.empty[String, String],
+      initContainer = new ContainerBuilder().build(),
+      driverContainer = driverSpec.driverContainer,
+      podToInitialize = driverSpec.driverPod,
+      initContainerDependentResources = Seq.empty[HasMetadata])
     for (nextStep <- initContainerConfigurationSteps) {
       currentInitContainerSpec = nextStep.configureInitContainer(currentInitContainerSpec)
     }
     val configMap = PropertiesConfigMapFromScalaMapBuilder.buildConfigMap(
-        initContainerConfigMapName,
-        initContainerConfigMapKey,
-        currentInitContainerSpec.initContainerProperties)
+      initContainerConfigMapName,
+      initContainerConfigMapKey,
+      currentInitContainerSpec.initContainerProperties)
     val resolvedDriverSparkConf = driverSpec.driverSparkConf.clone()
-        .set(EXECUTOR_INIT_CONTAINER_CONFIG_MAP, initContainerConfigMapName)
-        .set(EXECUTOR_INIT_CONTAINER_CONFIG_MAP_KEY, initContainerConfigMapKey)
-        .setAll(currentInitContainerSpec.additionalDriverSparkConf)
+      .set(EXECUTOR_INIT_CONTAINER_CONFIG_MAP, initContainerConfigMapName)
+      .set(EXECUTOR_INIT_CONTAINER_CONFIG_MAP_KEY, initContainerConfigMapKey)
+      .setAll(currentInitContainerSpec.additionalDriverSparkConf)
     val resolvedDriverPod = InitContainerUtil.appendInitContainer(
-        currentInitContainerSpec.podToInitialize, currentInitContainerSpec.initContainer)
+      currentInitContainerSpec.podToInitialize, currentInitContainerSpec.initContainer)
     driverSpec.copy(
-        driverPod = resolvedDriverPod,
-        driverContainer = currentInitContainerSpec.driverContainer,
-        driverSparkConf = resolvedDriverSparkConf,
-        otherKubernetesResources =
-            driverSpec.otherKubernetesResources ++
-                currentInitContainerSpec.initContainerDependentResources ++
-                Seq(configMap))
+      driverPod = resolvedDriverPod,
+      driverContainer = currentInitContainerSpec.driverContainer,
+      driverSparkConf = resolvedDriverSparkConf,
+      otherKubernetesResources =
+        driverSpec.otherKubernetesResources ++
+          currentInitContainerSpec.initContainerDependentResources ++
+          Seq(configMap))
   }
 }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/MountSmallLocalFilesStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/MountSmallLocalFilesStep.scala
new file mode 100644
index 0000000000000..cd1b7f6b7eb7e
--- /dev/null
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/MountSmallLocalFilesStep.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.deploy.kubernetes.submit.submitsteps
+
+import java.io.File
+
+import com.google.common.io.{BaseEncoding, Files}
+import io.fabric8.kubernetes.api.model.SecretBuilder
+import scala.collection.JavaConverters._
+
+import org.apache.spark.deploy.kubernetes.config._
+import org.apache.spark.deploy.kubernetes.submit.{KubernetesFileUtils, MountSmallFilesBootstrap}
+import org.apache.spark.util.Utils
+
+private[spark] class MountSmallLocalFilesStep(
+    sparkFiles: Seq[String],
+    smallFilesSecretName: String,
+    smallFilesSecretMountPath: String,
+    mountSmallFilesBootstrap: MountSmallFilesBootstrap) extends DriverConfigurationStep {
+
+  import MountSmallLocalFilesStep._
+  override def configureDriver(driverSpec: KubernetesDriverSpec): KubernetesDriverSpec = {
+    val localFiles = KubernetesFileUtils.getOnlySubmitterLocalFiles(sparkFiles).map(new File(_))
+    val totalSizeBytes = localFiles.map(_.length()).sum
+    val totalSizeBytesString = Utils.bytesToString(totalSizeBytes)
+    require(totalSizeBytes < MAX_SECRET_BUNDLE_SIZE_BYTES,
+      s"Total size of all files submitted must be less than $MAX_SECRET_BUNDLE_SIZE_BYTES_STRING" +
+        s" if you do not use a resource staging server. The total size of all submitted local" +
+        s" files is $totalSizeBytesString. Please install a resource staging server and configure" +
+        s" your application to use it via ${RESOURCE_STAGING_SERVER_URI.key}")
+    val localFileBase64Contents = localFiles.map { file =>
+      val fileBase64 = BaseEncoding.base64().encode(Files.toByteArray(file))
+      (file.getName, fileBase64)
+    }.toMap
+    val localFilesSecret = new SecretBuilder()
+      .withNewMetadata()
+        .withName(smallFilesSecretName)
+        .endMetadata()
+      .withData(localFileBase64Contents.asJava)
+      .build()
+    val (resolvedDriverPod, resolvedDriverContainer) =
+      mountSmallFilesBootstrap.mountSmallFilesSecret(
+        driverSpec.driverPod, driverSpec.driverContainer)
+    val resolvedSparkConf = driverSpec.driverSparkConf.clone()
+      .set(EXECUTOR_SUBMITTED_SMALL_FILES_SECRET, smallFilesSecretName)
+      .set(EXECUTOR_SUBMITTED_SMALL_FILES_SECRET_MOUNT_PATH, smallFilesSecretMountPath)
+    driverSpec.copy(
+      driverPod = resolvedDriverPod,
+      driverContainer = resolvedDriverContainer,
+      driverSparkConf = resolvedSparkConf,
+      otherKubernetesResources = driverSpec.otherKubernetesResources ++ Seq(localFilesSecret))
+  }
+}
+
+private[spark] object MountSmallLocalFilesStep {
+  val MAX_SECRET_BUNDLE_SIZE_BYTES = 10240
+  val MAX_SECRET_BUNDLE_SIZE_BYTES_STRING =
+    Utils.bytesToString(MAX_SECRET_BUNDLE_SIZE_BYTES)
+}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/InetAddressUtil.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/InetAddressUtil.scala
new file mode 100644
index 0000000000000..e04ab9e541963
--- /dev/null
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/InetAddressUtil.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.scheduler.cluster.kubernetes
+
+import java.net.InetAddress
+
+/**
+ * Gets full host names of given IP addresses from DNS.
+ */
+private[kubernetes] trait InetAddressUtil {
+
+  def getFullHostName(ipAddress: String): String
+}
+
+private[kubernetes] object InetAddressUtilImpl extends InetAddressUtil {
+
+  // NOTE: This does issue a network call to DNS. Caching is done internally by the InetAddress
+  // class for both hits and misses.
+  override def getFullHostName(ipAddress: String): String = {
+    InetAddress.getByName(ipAddress).getCanonicalHostName
+  }
+}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesClusterManager.scala
index fa0ecca3b4ee6..b89e81bcb0be9 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesClusterManager.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesClusterManager.scala
@@ -24,6 +24,7 @@ import org.apache.spark.SparkContext
 import org.apache.spark.deploy.kubernetes.{InitContainerResourceStagingServerSecretPluginImpl, SparkKubernetesClientFactory, SparkPodInitContainerBootstrapImpl}
 import org.apache.spark.deploy.kubernetes.config._
 import org.apache.spark.deploy.kubernetes.constants._
+import org.apache.spark.deploy.kubernetes.submit.MountSmallFilesBootstrapImpl
 import org.apache.spark.internal.Logging
 import org.apache.spark.scheduler.{ExternalClusterManager, SchedulerBackend, TaskScheduler, TaskSchedulerImpl}
 
@@ -40,28 +41,30 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit
   override def createSchedulerBackend(sc: SparkContext, masterURL: String, scheduler: TaskScheduler)
       : SchedulerBackend = {
     val sparkConf = sc.getConf
-    val maybeConfigMap = sparkConf.get(EXECUTOR_INIT_CONTAINER_CONFIG_MAP)
-    val maybeConfigMapKey = sparkConf.get(EXECUTOR_INIT_CONTAINER_CONFIG_MAP_KEY)
+    val maybeInitContainerConfigMap = sparkConf.get(EXECUTOR_INIT_CONTAINER_CONFIG_MAP)
+    val maybeInitContainerConfigMapKey = sparkConf.get(EXECUTOR_INIT_CONTAINER_CONFIG_MAP_KEY)
+    val maybeSubmittedFilesSecret = sparkConf.get(EXECUTOR_SUBMITTED_SMALL_FILES_SECRET)
+    val maybeSubmittedFilesSecretMountPath = sparkConf.get(
+      EXECUTOR_SUBMITTED_SMALL_FILES_SECRET_MOUNT_PATH)
 
     val maybeExecutorInitContainerSecretName =
       sparkConf.get(EXECUTOR_INIT_CONTAINER_SECRET)
-    val maybeExecutorInitContainerSecretMount =
+    val maybeExecutorInitContainerSecretMountPath =
       sparkConf.get(EXECUTOR_INIT_CONTAINER_SECRET_MOUNT_DIR)
     val executorInitContainerSecretVolumePlugin = for {
       initContainerSecretName <- maybeExecutorInitContainerSecretName
-      initContainerSecretMountPath <- maybeExecutorInitContainerSecretMount
+      initContainerSecretMountPath <- maybeExecutorInitContainerSecretMountPath
     } yield {
       new InitContainerResourceStagingServerSecretPluginImpl(
         initContainerSecretName,
         initContainerSecretMountPath)
     }
     // Only set up the bootstrap if they've provided both the config map key and the config map
-    // name. Note that we generally expect both to have been set from spark-submit V2, but for
-    // testing developers may simply run the driver JVM locally, but the config map won't be set
-    // then.
-    val bootStrap = for {
-      configMap <- maybeConfigMap
-      configMapKey <- maybeConfigMapKey
+    // name. The config map might not be provided if init-containers aren't being used to
+    // bootstrap dependencies.
+    val executorInitContainerbootStrap = for {
+      configMap <- maybeInitContainerConfigMap
+      configMapKey <- maybeInitContainerConfigMapKey
     } yield {
       new SparkPodInitContainerBootstrapImpl(
         sparkConf.get(INIT_CONTAINER_DOCKER_IMAGE),
@@ -72,11 +75,17 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit
         configMap,
         configMapKey)
     }
-    if (maybeConfigMap.isEmpty) {
+    val mountSmallFilesBootstrap = for {
+      secretName <- maybeSubmittedFilesSecret
+      secretMountPath <- maybeSubmittedFilesSecretMountPath
+    } yield {
+      new MountSmallFilesBootstrapImpl(secretName, secretMountPath)
+    }
+    if (maybeInitContainerConfigMap.isEmpty) {
       logWarning("The executor's init-container config map was not specified. Executors will" +
         " therefore not attempt to fetch remote or submitted dependencies.")
     }
-    if (maybeConfigMapKey.isEmpty) {
+    if (maybeInitContainerConfigMapKey.isEmpty) {
       logWarning("The executor's init-container config map key was not specified. Executors will" +
         " therefore not attempt to fetch remote or submitted dependencies.")
     }
@@ -90,8 +99,9 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit
     new KubernetesClusterSchedulerBackend(
         sc.taskScheduler.asInstanceOf[TaskSchedulerImpl],
         sc,
-        bootStrap,
+        executorInitContainerbootStrap,
         executorInitContainerSecretVolumePlugin,
+        mountSmallFilesBootstrap,
         kubernetesClient)
   }
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesClusterSchedulerBackend.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesClusterSchedulerBackend.scala
index f6aecc402c0fe..415c1bda7b632 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesClusterSchedulerBackend.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesClusterSchedulerBackend.scala
@@ -22,22 +22,21 @@ import java.util.Collections
 import java.util.concurrent.{ConcurrentHashMap, TimeUnit}
 import java.util.concurrent.atomic.{AtomicInteger, AtomicLong, AtomicReference}
 
-import scala.collection.{concurrent, mutable}
-import scala.collection.JavaConverters._
-import scala.concurrent.{ExecutionContext, Future}
-
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
 import io.fabric8.kubernetes.api.model._
 import io.fabric8.kubernetes.client.{KubernetesClient, KubernetesClientException, Watcher}
 import io.fabric8.kubernetes.client.Watcher.Action
 import org.apache.commons.io.FilenameUtils
+import scala.collection.{concurrent, mutable}
+import scala.collection.JavaConverters._
+import scala.concurrent.{ExecutionContext, Future}
 
 import org.apache.spark.{SparkContext, SparkEnv, SparkException}
 import org.apache.spark.deploy.kubernetes.{ConfigurationUtils, InitContainerResourceStagingServerSecretPlugin, PodWithDetachedInitContainer, SparkPodInitContainerBootstrap}
 import org.apache.spark.deploy.kubernetes.config._
 import org.apache.spark.deploy.kubernetes.constants._
-import org.apache.spark.deploy.kubernetes.submit.{HadoopSecretUtil, InitContainerUtil}
+import org.apache.spark.deploy.kubernetes.submit.{HadoopSecretUtil, InitContainerUtil, MountSmallFilesBootstrap}
 import org.apache.spark.network.netty.SparkTransportConf
 import org.apache.spark.network.shuffle.kubernetes.KubernetesExternalShuffleClient
 import org.apache.spark.rpc.{RpcAddress, RpcCallContext, RpcEndpointAddress, RpcEnv}
@@ -51,6 +50,7 @@ private[spark] class KubernetesClusterSchedulerBackend(
     val sc: SparkContext,
     executorInitContainerBootstrap: Option[SparkPodInitContainerBootstrap],
     executorMountInitContainerSecretPlugin: Option[InitContainerResourceStagingServerSecretPlugin],
+    mountSmallFilesBootstrap: Option[MountSmallFilesBootstrap],
     kubernetesClient: KubernetesClient)
   extends CoarseGrainedSchedulerBackend(scheduler, sc.env.rpcEnv) {
 
@@ -113,16 +113,16 @@ private[spark] class KubernetesClusterSchedulerBackend(
       throw new SparkException("Must specify the driver pod name"))
   private val executorPodNamePrefix = conf.get(KUBERNETES_EXECUTOR_POD_NAME_PREFIX)
 
-  private val executorMemoryMb = conf.get(org.apache.spark.internal.config.EXECUTOR_MEMORY)
+  private val executorMemoryMiB = conf.get(org.apache.spark.internal.config.EXECUTOR_MEMORY)
   private val executorMemoryString = conf.get(
     org.apache.spark.internal.config.EXECUTOR_MEMORY.key,
     org.apache.spark.internal.config.EXECUTOR_MEMORY.defaultValueString)
 
-  private val memoryOverheadMb = conf
+  private val memoryOverheadMiB = conf
     .get(KUBERNETES_EXECUTOR_MEMORY_OVERHEAD)
-    .getOrElse(math.max((MEMORY_OVERHEAD_FACTOR * executorMemoryMb).toInt,
-      MEMORY_OVERHEAD_MIN))
-  private val executorMemoryWithOverhead = executorMemoryMb + memoryOverheadMb
+    .getOrElse(math.max((MEMORY_OVERHEAD_FACTOR * executorMemoryMiB).toInt,
+      MEMORY_OVERHEAD_MIN_MIB))
+  private val executorMemoryWithOverheadMiB = executorMemoryMiB + memoryOverheadMiB
 
   private val executorCores = conf.getDouble("spark.executor.cores", 1d)
   private val executorLimitCores = conf.getOption(KUBERNETES_EXECUTOR_LIMIT_CORES.key)
@@ -443,10 +443,10 @@ private[spark] class KubernetesClusterSchedulerBackend(
       SPARK_ROLE_LABEL -> SPARK_POD_EXECUTOR_ROLE) ++
       executorLabels
     val executorMemoryQuantity = new QuantityBuilder(false)
-      .withAmount(s"${executorMemoryMb}M")
+      .withAmount(s"${executorMemoryMiB}Mi")
       .build()
     val executorMemoryLimitQuantity = new QuantityBuilder(false)
-      .withAmount(s"${executorMemoryWithOverhead}M")
+      .withAmount(s"${executorMemoryWithOverheadMiB}Mi")
       .build()
     val executorCpuQuantity = new QuantityBuilder(false)
       .withAmount(executorCores.toString)
@@ -457,7 +457,16 @@ private[spark] class KubernetesClusterSchedulerBackend(
         .withValue(cp)
         .build()
     }
-    val requiredEnv = Seq(
+    val executorExtraJavaOptionsEnv = conf
+        .get(org.apache.spark.internal.config.EXECUTOR_JAVA_OPTIONS)
+        .map { opts =>
+          val delimitedOpts = Utils.splitCommandString(opts)
+          delimitedOpts.zipWithIndex.map {
+            case (opt, index) =>
+              new EnvVarBuilder().withName(s"$ENV_JAVA_OPT_PREFIX$index").withValue(opt).build()
+          }
+        }.getOrElse(Seq.empty[EnvVar])
+    val executorEnv = (Seq(
       (ENV_EXECUTOR_PORT, executorPort.toString),
       (ENV_DRIVER_URL, driverUrl),
       // Executor backend expects integral value for executor cores, so round it up to an int.
@@ -465,7 +474,7 @@ private[spark] class KubernetesClusterSchedulerBackend(
       (ENV_EXECUTOR_MEMORY, executorMemoryString),
       (ENV_APPLICATION_ID, applicationId()),
       (ENV_EXECUTOR_ID, executorId),
-      (ENV_MOUNTED_CLASSPATH, s"$executorJarsDownloadDir/*"))
+      (ENV_MOUNTED_CLASSPATH, s"$executorJarsDownloadDir/*")) ++ sc.executorEnvs.toSeq)
       .map(env => new EnvVarBuilder()
         .withName(env._1)
         .withValue(env._2)
@@ -477,7 +486,7 @@ private[spark] class KubernetesClusterSchedulerBackend(
           .withNewFieldRef("v1", "status.podIP")
           .build())
         .build()
-      )
+      ) ++ executorExtraJavaOptionsEnv ++ executorExtraClasspathEnv.toSeq
     val requiredPorts = Seq(
       (EXECUTOR_PORT_NAME, executorPort),
       (BLOCK_MANAGER_PORT_NAME, blockmanagerPort))
@@ -497,8 +506,7 @@ private[spark] class KubernetesClusterSchedulerBackend(
         .addToLimits("memory", executorMemoryLimitQuantity)
         .addToRequests("cpu", executorCpuQuantity)
       .endResources()
-      .addAllToEnv(requiredEnv.asJava)
-      .addToEnv(executorExtraClasspathEnv.toSeq: _*)
+      .addAllToEnv(executorEnv.asJava)
       .withPorts(requiredPorts.asJava)
       .build()
 
@@ -559,13 +567,18 @@ private[spark] class KubernetesClusterSchedulerBackend(
           .build()
       }
     }.getOrElse(executorPod)
+    val (withMaybeSmallFilesMountedPod, withMaybeSmallFilesMountedContainer) =
+        mountSmallFilesBootstrap.map { bootstrap =>
+          bootstrap.mountSmallFilesSecret(
+            withMaybeShuffleConfigPod, withMaybeShuffleConfigExecutorContainer)
+        }.getOrElse((withMaybeShuffleConfigPod, withMaybeShuffleConfigExecutorContainer))
     val (executorPodWithInitContainer, initBootstrappedExecutorContainer) =
         executorInitContainerBootstrap.map { bootstrap =>
           val podWithDetachedInitContainer = bootstrap.bootstrapInitContainerAndVolumes(
               PodWithDetachedInitContainer(
-                  withMaybeShuffleConfigPod,
+                  withMaybeSmallFilesMountedPod,
                   new ContainerBuilder().build(),
-                  withMaybeShuffleConfigExecutorContainer))
+                withMaybeSmallFilesMountedContainer))
 
           val resolvedInitContainer = executorMountInitContainerSecretPlugin.map { plugin =>
             plugin.mountResourceStagingServerSecretIntoInitContainer(
@@ -580,7 +593,7 @@ private[spark] class KubernetesClusterSchedulerBackend(
           }.getOrElse(podWithAttachedInitContainer)
 
           (resolvedPodWithMountedSecret, podWithDetachedInitContainer.mainContainer)
-      }.getOrElse((withMaybeShuffleConfigPod, withMaybeShuffleConfigExecutorContainer))
+      }.getOrElse((withMaybeSmallFilesMountedPod, withMaybeSmallFilesMountedContainer))
 
     val executorPodWithNodeAffinity = addNodeAffinityAnnotationIfUseful(
         executorPodWithInitContainer, nodeToLocalTaskCount)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSchedulerImpl.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSchedulerImpl.scala
index a5e126480b83d..ca40d46400ea2 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSchedulerImpl.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSchedulerImpl.scala
@@ -16,12 +16,59 @@
  */
 package org.apache.spark.scheduler.cluster.kubernetes
 
+import org.apache.spark.deploy.kubernetes.config._
+import org.apache.spark.scheduler.{SchedulerBackend, TaskSchedulerImpl, TaskSet, TaskSetManager}
+import org.apache.spark.util.Utils
 import org.apache.spark.SparkContext
-import org.apache.spark.scheduler.{TaskSchedulerImpl, TaskSet, TaskSetManager}
 
-private[spark] class KubernetesTaskSchedulerImpl(sc: SparkContext) extends TaskSchedulerImpl(sc) {
+private[spark] class KubernetesTaskSchedulerImpl(
+    sc: SparkContext,
+    rackResolverUtil: RackResolverUtil,
+    inetAddressUtil: InetAddressUtil = InetAddressUtilImpl) extends TaskSchedulerImpl(sc) {
 
+  var kubernetesSchedulerBackend: KubernetesClusterSchedulerBackend = null
+
+  def this(sc: SparkContext) = this(sc, new RackResolverUtilImpl(sc.hadoopConfiguration))
+
+  override def initialize(backend: SchedulerBackend): Unit = {
+    super.initialize(backend)
+    kubernetesSchedulerBackend = this.backend.asInstanceOf[KubernetesClusterSchedulerBackend]
+  }
   override def createTaskSetManager(taskSet: TaskSet, maxTaskFailures: Int): TaskSetManager = {
     new KubernetesTaskSetManager(this, taskSet, maxTaskFailures)
   }
+
+  override def getRackForHost(hostPort: String): Option[String] = {
+    if (!rackResolverUtil.isConfigured) {
+      // Only calls resolver when it is configured to avoid sending DNS queries for cluster nodes.
+      // See InetAddressUtil for details.
+      None
+    } else {
+      getRackForDatanodeOrExecutor(hostPort)
+    }
+  }
+
+  private def getRackForDatanodeOrExecutor(hostPort: String): Option[String] = {
+    val host = Utils.parseHostPort(hostPort)._1
+    val executorPod = kubernetesSchedulerBackend.getExecutorPodByIP(host)
+    val hadoopConfiguration = sc.hadoopConfiguration
+    executorPod.map(
+        pod => {
+          val clusterNodeName = pod.getSpec.getNodeName
+          val rackByNodeName = rackResolverUtil.resolveRack(hadoopConfiguration, clusterNodeName)
+          rackByNodeName.orElse({
+            val clusterNodeIP = pod.getStatus.getHostIP
+            val rackByNodeIP = rackResolverUtil.resolveRack(hadoopConfiguration, clusterNodeIP)
+            rackByNodeIP.orElse({
+              if (conf.get(KUBERNETES_DRIVER_CLUSTER_NODENAME_DNS_LOOKUP_ENABLED)) {
+                val clusterNodeFullName = inetAddressUtil.getFullHostName(clusterNodeIP)
+                rackResolverUtil.resolveRack(hadoopConfiguration, clusterNodeFullName)
+              } else {
+                Option.empty
+              }
+            })
+          })
+        }
+      ).getOrElse(rackResolverUtil.resolveRack(hadoopConfiguration, host))
+  }
 }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSetManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSetManager.scala
index 51566d03a7a6c..44d01a5d76b01 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSetManager.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSetManager.scala
@@ -16,19 +16,20 @@
  */
 package org.apache.spark.scheduler.cluster.kubernetes
 
-import java.net.InetAddress
-
 import scala.collection.mutable.ArrayBuffer
 
+import org.apache.spark.deploy.kubernetes.config._
 import org.apache.spark.scheduler.{TaskSchedulerImpl, TaskSet, TaskSetManager}
 
 private[spark] class KubernetesTaskSetManager(
     sched: TaskSchedulerImpl,
     taskSet: TaskSet,
     maxTaskFailures: Int,
-    inetAddressUtil: InetAddressUtil = new InetAddressUtil)
+    inetAddressUtil: InetAddressUtil = InetAddressUtilImpl)
   extends TaskSetManager(sched, taskSet, maxTaskFailures) {
 
+  private val conf = sched.sc.conf
+
   /**
    * Overrides the lookup to use not only the executor pod IP, but also the cluster node
    * name and host IP address that the pod is running on. The base class may have populated
@@ -58,13 +59,19 @@ private[spark] class KubernetesTaskSetManager(
               s"$executorIP using cluster node IP $clusterNodeIP")
             pendingTasksClusterNodeIP
           } else {
-            val clusterNodeFullName = inetAddressUtil.getFullHostName(clusterNodeIP)
-            val pendingTasksClusterNodeFullName = super.getPendingTasksForHost(clusterNodeFullName)
-            if (pendingTasksClusterNodeFullName.nonEmpty) {
-              logDebug(s"Got preferred task list $pendingTasksClusterNodeFullName " +
-                s"for executor host $executorIP using cluster node full name $clusterNodeFullName")
+            if (conf.get(KUBERNETES_DRIVER_CLUSTER_NODENAME_DNS_LOOKUP_ENABLED)) {
+              val clusterNodeFullName = inetAddressUtil.getFullHostName(clusterNodeIP)
+              val pendingTasksClusterNodeFullName = super.getPendingTasksForHost(
+                clusterNodeFullName)
+              if (pendingTasksClusterNodeFullName.nonEmpty) {
+                logDebug(s"Got preferred task list $pendingTasksClusterNodeFullName " +
+                  s"for executor host $executorIP using cluster node full name " +
+                  s"$clusterNodeFullName")
+              }
+              pendingTasksClusterNodeFullName
+            } else {
+              pendingTasksExecutorIP  // Empty
             }
-            pendingTasksClusterNodeFullName
           }
         }
       } else {
@@ -74,12 +81,3 @@ private[spark] class KubernetesTaskSetManager(
   }
 }
 
-// To support mocks in unit tests.
-private[kubernetes] class InetAddressUtil {
-
-  // NOTE: This does issue a network call to DNS. Caching is done internally by the InetAddress
-  // class for both hits and misses.
-  def getFullHostName(ipAddress: String): String = {
-    InetAddress.getByName(ipAddress).getCanonicalHostName
-  }
-}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/RackResolverUtil.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/RackResolverUtil.scala
new file mode 100644
index 0000000000000..29a7dc982a5af
--- /dev/null
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/RackResolverUtil.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.scheduler.cluster.kubernetes
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic
+import org.apache.hadoop.net.{NetworkTopology, ScriptBasedMapping, TableMapping}
+import org.apache.hadoop.yarn.util.RackResolver
+import org.apache.log4j.{Level, Logger}
+
+/**
+ * Finds rack names that cluster nodes belong to in order to support HDFS rack locality.
+ */
+private[kubernetes] trait RackResolverUtil {
+
+  def isConfigured() : Boolean
+
+  def resolveRack(hadoopConfiguration: Configuration, host: String): Option[String]
+}
+
+private[kubernetes] class RackResolverUtilImpl(hadoopConfiguration: Configuration)
+    extends RackResolverUtil {
+
+  val scriptPlugin : String = classOf[ScriptBasedMapping].getCanonicalName
+  val tablePlugin : String = classOf[TableMapping].getCanonicalName
+  val isResolverConfigured : Boolean = checkConfigured(hadoopConfiguration)
+
+  // RackResolver logs an INFO message whenever it resolves a rack, which is way too often.
+  if (Logger.getLogger(classOf[RackResolver]).getLevel == null) {
+    Logger.getLogger(classOf[RackResolver]).setLevel(Level.WARN)
+  }
+
+  override def isConfigured() : Boolean = isResolverConfigured
+
+  def checkConfigured(hadoopConfiguration: Configuration): Boolean = {
+    val plugin = hadoopConfiguration.get(
+      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, scriptPlugin)
+    val scriptName = hadoopConfiguration.get(
+      CommonConfigurationKeysPublic.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY, "")
+    val tableName = hadoopConfiguration.get(
+      CommonConfigurationKeysPublic.NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY, "")
+    plugin == scriptPlugin && scriptName.nonEmpty ||
+      plugin == tablePlugin && tableName.nonEmpty ||
+      plugin != scriptPlugin && plugin != tablePlugin
+  }
+
+  override def resolveRack(hadoopConfiguration: Configuration, host: String): Option[String] = {
+    val rack = Option(RackResolver.resolve(hadoopConfiguration, host).getNetworkLocation)
+    if (rack.nonEmpty && rack.get != NetworkTopology.DEFAULT_RACK) {
+      rack
+    } else {
+      None
+    }
+  }
+}
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/ClientSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/ClientSuite.scala
index 965ee75c248b8..0100dce454a3f 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/ClientSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/ClientSuite.scala
@@ -135,7 +135,7 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter {
         .set("spark.logConf", "true")
         .set(
           org.apache.spark.internal.config.DRIVER_JAVA_OPTIONS,
-          "-XX:+|-HeapDumpOnOutOfMemoryError")
+          "-XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails")
     val submissionClient = new Client(
         submissionSteps,
         sparkConf,
@@ -147,15 +147,22 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter {
     val createdPod = createdPodArgumentCaptor.getValue
     val driverContainer = Iterables.getOnlyElement(createdPod.getSpec.getContainers)
     assert(driverContainer.getName === SecondTestConfigurationStep.containerName)
-    val driverJvmOptsEnv = Iterables.getOnlyElement(driverContainer.getEnv)
-    assert(driverJvmOptsEnv.getName === ENV_DRIVER_JAVA_OPTS)
-    val driverJvmOpts = driverJvmOptsEnv.getValue.split(" ").toSet
-    assert(driverJvmOpts.contains("-Dspark.logConf=true"))
-    assert(driverJvmOpts.contains(
+    val driverJvmOptsEnvs = driverContainer.getEnv.asScala.filter { env =>
+      env.getName.startsWith(ENV_JAVA_OPT_PREFIX)
+    }.sortBy(_.getName)
+    assert(driverJvmOptsEnvs.size === 4)
+
+    val expectedJvmOptsValues = Seq(
+        "-Dspark.logConf=true",
         s"-D${SecondTestConfigurationStep.sparkConfKey}=" +
-          SecondTestConfigurationStep.sparkConfValue))
-    assert(driverJvmOpts.contains(
-        "-XX:+|-HeapDumpOnOutOfMemoryError"))
+            s"${SecondTestConfigurationStep.sparkConfValue}",
+        s"-XX:+HeapDumpOnOutOfMemoryError",
+        s"-XX:+PrintGCDetails")
+    driverJvmOptsEnvs.zip(expectedJvmOptsValues).zipWithIndex.foreach {
+      case ((resolvedEnv, expectedJvmOpt), index) =>
+        assert(resolvedEnv.getName === s"$ENV_JAVA_OPT_PREFIX$index")
+        assert(resolvedEnv.getValue === expectedJvmOpt)
+    }
   }
 
   test("Waiting for app completion should stall on the watcher") {
@@ -211,8 +218,8 @@ private object SecondTestConfigurationStep extends DriverConfigurationStep {
   override def configureDriver(driverSpec: KubernetesDriverSpec): KubernetesDriverSpec = {
     val modifiedPod = new PodBuilder(driverSpec.driverPod)
       .editMetadata()
-      .addToAnnotations(annotationKey, annotationValue)
-      .endMetadata()
+        .addToAnnotations(annotationKey, annotationValue)
+        .endMetadata()
       .build()
     val resolvedSparkConf = driverSpec.driverSparkConf.clone().set(sparkConfKey, sparkConfValue)
     val modifiedContainer = new ContainerBuilder(driverSpec.driverContainer)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/DriverConfigurationStepsOrchestratorSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/DriverConfigurationStepsOrchestratorSuite.scala
index e4f221ad99cc5..c168e7b5407ba 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/DriverConfigurationStepsOrchestratorSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/DriverConfigurationStepsOrchestratorSuite.scala
@@ -17,7 +17,8 @@
 package org.apache.spark.deploy.kubernetes.submit
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
-import org.apache.spark.deploy.kubernetes.submit.submitsteps.{BaseDriverConfigurationStep, DependencyResolutionStep, DriverKubernetesCredentialsStep, InitContainerBootstrapStep, PythonStep}
+import org.apache.spark.deploy.kubernetes.config._
+import org.apache.spark.deploy.kubernetes.submit.submitsteps.{BaseDriverConfigurationStep, DependencyResolutionStep, DriverConfigurationStep, DriverKubernetesCredentialsStep, InitContainerBootstrapStep, MountSmallLocalFilesStep, PythonStep}
 
 private[spark] class DriverConfigurationStepsOrchestratorSuite extends SparkFunSuite {
 
@@ -31,7 +32,7 @@ private[spark] class DriverConfigurationStepsOrchestratorSuite extends SparkFunS
 
   test("Base submission steps without an init-container or python files.") {
     val sparkConf = new SparkConf(false)
-      .set("spark.jars", "local:///var/apps/jars/jar1.jar")
+        .set("spark.jars", "local:///var/apps/jars/jar1.jar")
     val mainAppResource = JavaMainAppResource("local:///var/apps/jars/main.jar")
     val orchestrator = new DriverConfigurationStepsOrchestrator(
         NAMESPACE,
@@ -43,16 +44,17 @@ private[spark] class DriverConfigurationStepsOrchestratorSuite extends SparkFunS
         APP_ARGS,
         ADDITIONAL_PYTHON_FILES,
         sparkConf)
-    val steps = orchestrator.getAllConfigurationSteps()
-    assert(steps.size === 3)
-    assert(steps(0).isInstanceOf[BaseDriverConfigurationStep])
-    assert(steps(1).isInstanceOf[DriverKubernetesCredentialsStep])
-    assert(steps(2).isInstanceOf[DependencyResolutionStep])
+    validateStepTypes(
+        orchestrator,
+        classOf[BaseDriverConfigurationStep],
+        classOf[DriverKubernetesCredentialsStep],
+        classOf[DependencyResolutionStep])
   }
 
   test("Submission steps with an init-container.") {
     val sparkConf = new SparkConf(false)
-      .set("spark.jars", "hdfs://localhost:9000/var/apps/jars/jar1.jar")
+        .set("spark.jars", "hdfs://localhost:9000/var/apps/jars/jar1.jar")
+        .set(RESOURCE_STAGING_SERVER_URI, "https://localhost:8080")
     val mainAppResource = JavaMainAppResource("local:///var/apps/jars/main.jar")
     val orchestrator = new DriverConfigurationStepsOrchestrator(
         NAMESPACE,
@@ -64,12 +66,12 @@ private[spark] class DriverConfigurationStepsOrchestratorSuite extends SparkFunS
         APP_ARGS,
         ADDITIONAL_PYTHON_FILES,
         sparkConf)
-    val steps = orchestrator.getAllConfigurationSteps()
-    assert(steps.size === 4)
-    assert(steps(0).isInstanceOf[BaseDriverConfigurationStep])
-    assert(steps(1).isInstanceOf[DriverKubernetesCredentialsStep])
-    assert(steps(2).isInstanceOf[DependencyResolutionStep])
-    assert(steps(3).isInstanceOf[InitContainerBootstrapStep])
+    validateStepTypes(
+        orchestrator,
+        classOf[BaseDriverConfigurationStep],
+        classOf[DriverKubernetesCredentialsStep],
+        classOf[DependencyResolutionStep],
+        classOf[InitContainerBootstrapStep])
   }
 
   test("Submission steps with python files.") {
@@ -85,11 +87,40 @@ private[spark] class DriverConfigurationStepsOrchestratorSuite extends SparkFunS
         APP_ARGS,
         ADDITIONAL_PYTHON_FILES,
         sparkConf)
+    validateStepTypes(
+        orchestrator,
+        classOf[BaseDriverConfigurationStep],
+        classOf[DriverKubernetesCredentialsStep],
+        classOf[DependencyResolutionStep],
+        classOf[PythonStep])
+  }
+
+  test("Only local files without a resource staging server.") {
+    val sparkConf = new SparkConf(false).set("spark.files", "/var/spark/file1.txt")
+    val mainAppResource = JavaMainAppResource("local:///var/apps/jars/main.jar")
+    val orchestrator = new DriverConfigurationStepsOrchestrator(
+        NAMESPACE,
+        APP_ID,
+        LAUNCH_TIME,
+        mainAppResource,
+        APP_NAME,
+        MAIN_CLASS,
+        APP_ARGS,
+        ADDITIONAL_PYTHON_FILES,
+        sparkConf)
+    validateStepTypes(
+        orchestrator,
+        classOf[BaseDriverConfigurationStep],
+        classOf[DriverKubernetesCredentialsStep],
+        classOf[DependencyResolutionStep],
+        classOf[MountSmallLocalFilesStep])
+  }
+
+  private def validateStepTypes(
+      orchestrator: DriverConfigurationStepsOrchestrator,
+      types: Class[_ <: DriverConfigurationStep]*): Unit = {
     val steps = orchestrator.getAllConfigurationSteps()
-    assert(steps.size === 4)
-    assert(steps(0).isInstanceOf[BaseDriverConfigurationStep])
-    assert(steps(1).isInstanceOf[DriverKubernetesCredentialsStep])
-    assert(steps(2).isInstanceOf[DependencyResolutionStep])
-    assert(steps(3).isInstanceOf[PythonStep])
+    assert(steps.size === types.size)
+    assert(steps.map(_.getClass) === types)
   }
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStepSuite.scala
index c7d80a16a1532..53c4ea60afacd 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStepSuite.scala
@@ -36,6 +36,8 @@ private[spark] class BaseDriverConfigurationStepSuite extends SparkFunSuite {
   private val CUSTOM_ANNOTATION_VALUE = "customAnnotationValue"
   private val DEPRECATED_CUSTOM_ANNOTATION_KEY = "customAnnotationDeprecated"
   private val DEPRECATED_CUSTOM_ANNOTATION_VALUE = "customAnnotationDeprecatedValue"
+  private val DRIVER_CUSTOM_ENV_KEY1 = "customDriverEnv1"
+  private val DRIVER_CUSTOM_ENV_KEY2 = "customDriverEnv2"
 
   test("Set all possible configurations from the user.") {
     val sparkConf = new SparkConf()
@@ -49,6 +51,9 @@ private[spark] class BaseDriverConfigurationStepSuite extends SparkFunSuite {
         .set(s"spark.kubernetes.driver.annotation.$CUSTOM_ANNOTATION_KEY", CUSTOM_ANNOTATION_VALUE)
         .set("spark.kubernetes.driver.annotations",
             s"$DEPRECATED_CUSTOM_ANNOTATION_KEY=$DEPRECATED_CUSTOM_ANNOTATION_VALUE")
+        .set(s"$KUBERNETES_DRIVER_ENV_KEY$DRIVER_CUSTOM_ENV_KEY1", "customDriverEnv1")
+        .set(s"$KUBERNETES_DRIVER_ENV_KEY$DRIVER_CUSTOM_ENV_KEY2", "customDriverEnv2")
+
     val submissionStep = new BaseDriverConfigurationStep(
         APP_ID,
         RESOURCE_NAME_PREFIX,
@@ -74,17 +79,19 @@ private[spark] class BaseDriverConfigurationStepSuite extends SparkFunSuite {
         .asScala
         .map(env => (env.getName, env.getValue))
         .toMap
-    assert(envs.size === 4)
+    assert(envs.size === 6)
     assert(envs(ENV_SUBMIT_EXTRA_CLASSPATH) === "/opt/spark/spark-exmaples.jar")
-    assert(envs(ENV_DRIVER_MEMORY) === "456m")
+    assert(envs(ENV_DRIVER_MEMORY) === "456M")
     assert(envs(ENV_DRIVER_MAIN_CLASS) === MAIN_CLASS)
     assert(envs(ENV_DRIVER_ARGS) === "arg1 arg2")
+    assert(envs(DRIVER_CUSTOM_ENV_KEY1) === "customDriverEnv1")
+    assert(envs(DRIVER_CUSTOM_ENV_KEY2) === "customDriverEnv2")
     val resourceRequirements = preparedDriverSpec.driverContainer.getResources
     val requests = resourceRequirements.getRequests.asScala
     assert(requests("cpu").getAmount === "2")
-    assert(requests("memory").getAmount === "256M")
+    assert(requests("memory").getAmount === "256Mi")
     val limits = resourceRequirements.getLimits.asScala
-    assert(limits("memory").getAmount === "456M")
+    assert(limits("memory").getAmount === "456Mi")
     assert(limits("cpu").getAmount === "4")
     val driverPodMetadata = preparedDriverSpec.driverPod.getMetadata
     assert(driverPodMetadata.getName === "spark-driver-pod")
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/MountSmallLocalFilesStepTest.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/MountSmallLocalFilesStepTest.scala
new file mode 100644
index 0000000000000..11744ef409c20
--- /dev/null
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/MountSmallLocalFilesStepTest.scala
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.deploy.kubernetes.submit.submitsteps
+
+import java.io.{File, RandomAccessFile}
+
+import com.google.common.base.Charsets
+import com.google.common.io.{BaseEncoding, Files}
+import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder, HasMetadata, Pod, PodBuilder, Secret}
+import org.junit.Test
+import org.mockito.{Mock, MockitoAnnotations}
+import org.scalatest.BeforeAndAfter
+import scala.collection.JavaConverters._
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.deploy.kubernetes.config._
+import org.apache.spark.deploy.kubernetes.constants._
+import org.apache.spark.deploy.kubernetes.submit.MountSmallFilesBootstrap
+import org.apache.spark.util.Utils
+
+private[spark] class MountSmallLocalFilesStepTest extends SparkFunSuite with BeforeAndAfter {
+
+  private val FIRST_TEMP_FILE_NAME = "file1.txt"
+  private val SECOND_TEMP_FILE_NAME = "file2.txt"
+  private val FIRST_TEMP_FILE_CONTENTS = "123"
+  private val SECOND_TEMP_FILE_CONTENTS = "456"
+  private val REMOTE_FILE_URI = "hdfs://localhost:9000/file3.txt"
+  private val SECRET_NAME = "secret"
+
+  private var tempFolder: File = _
+
+  private val mountSmallFilesBootstrap = new DummyMountSmallFilesBootstrap
+
+  before {
+    MockitoAnnotations.initMocks(this)
+    tempFolder = Utils.createTempDir()
+  }
+
+  after {
+    tempFolder.delete()
+  }
+
+  test("Local files should be added to the secret.") {
+    val firstTempFile = createTempFileWithContents(
+      tempFolder, FIRST_TEMP_FILE_NAME, FIRST_TEMP_FILE_CONTENTS)
+    val secondTempFile = createTempFileWithContents(
+        tempFolder, SECOND_TEMP_FILE_NAME, SECOND_TEMP_FILE_CONTENTS)
+    val sparkFiles = Seq(
+        firstTempFile.getAbsolutePath,
+        secondTempFile.getAbsolutePath,
+        REMOTE_FILE_URI)
+    val configurationStep = new MountSmallLocalFilesStep(
+        sparkFiles,
+        SECRET_NAME,
+        MOUNTED_SMALL_FILES_SECRET_MOUNT_PATH,
+        mountSmallFilesBootstrap)
+    val baseDriverSpec = new KubernetesDriverSpec(
+        new PodBuilder().build(),
+        new ContainerBuilder().build(),
+        Seq.empty[HasMetadata],
+        new SparkConf(false))
+    val configuredDriverSpec = configurationStep.configureDriver(baseDriverSpec)
+    assert(configuredDriverSpec.otherKubernetesResources.size === 1)
+    assert(configuredDriverSpec.otherKubernetesResources(0).isInstanceOf[Secret])
+    val localFilesSecret = configuredDriverSpec.otherKubernetesResources(0).asInstanceOf[Secret]
+    assert(localFilesSecret.getMetadata.getName === SECRET_NAME)
+    val expectedSecretContents = Map(
+      FIRST_TEMP_FILE_NAME -> BaseEncoding.base64().encode(
+        FIRST_TEMP_FILE_CONTENTS.getBytes(Charsets.UTF_8)),
+      SECOND_TEMP_FILE_NAME -> BaseEncoding.base64().encode(
+        SECOND_TEMP_FILE_CONTENTS.getBytes(Charsets.UTF_8)))
+    assert(localFilesSecret.getData.asScala === expectedSecretContents)
+    assert(configuredDriverSpec.driverPod.getMetadata.getLabels.asScala ===
+        Map(mountSmallFilesBootstrap.LABEL_KEY -> mountSmallFilesBootstrap.LABEL_VALUE))
+    assert(configuredDriverSpec.driverContainer.getEnv.size() === 1)
+    assert(configuredDriverSpec.driverContainer.getEnv.get(0).getName ===
+        mountSmallFilesBootstrap.ENV_KEY)
+    assert(configuredDriverSpec.driverContainer.getEnv.get(0).getValue ===
+        mountSmallFilesBootstrap.ENV_VALUE)
+    assert(configuredDriverSpec.driverSparkConf.get(
+        EXECUTOR_SUBMITTED_SMALL_FILES_SECRET) ===
+        Some(SECRET_NAME))
+    assert(configuredDriverSpec.driverSparkConf.get(
+        EXECUTOR_SUBMITTED_SMALL_FILES_SECRET_MOUNT_PATH) ===
+        Some(MOUNTED_SMALL_FILES_SECRET_MOUNT_PATH))
+  }
+
+  test("Using large files should throw an exception.") {
+    val largeTempFileContents = BaseEncoding.base64().encode(new Array[Byte](10241))
+    val largeTempFile = createTempFileWithContents(tempFolder, "large.txt", largeTempFileContents)
+    val configurationStep = new MountSmallLocalFilesStep(
+        Seq(largeTempFile.getAbsolutePath),
+        SECRET_NAME,
+        MOUNTED_SMALL_FILES_SECRET_MOUNT_PATH,
+        mountSmallFilesBootstrap)
+    val baseDriverSpec = new KubernetesDriverSpec(
+        new PodBuilder().build(),
+        new ContainerBuilder().build(),
+        Seq.empty[HasMetadata],
+        new SparkConf(false))
+    try {
+      configurationStep.configureDriver(baseDriverSpec)
+      fail("Using the small local files mounter should not be allowed with big files.")
+    } catch {
+      case e: Throwable =>
+        assert(e.getMessage ===
+          s"requirement failed: Total size of all files submitted must be less than" +
+            s" ${MountSmallLocalFilesStep.MAX_SECRET_BUNDLE_SIZE_BYTES_STRING} if you do not" +
+            s" use a resource staging server. The total size of all submitted local" +
+            s" files is ${Utils.bytesToString(largeTempFile.length())}. Please install a" +
+            s" resource staging server and configure your application to use it via" +
+            s" ${RESOURCE_STAGING_SERVER_URI.key}"
+        )
+    }
+  }
+
+  private def createTempFileWithContents(
+      root: File,
+      fileName: String,
+      fileContents: String): File = {
+    val tempFile = new File(root, fileName)
+    tempFile.createNewFile()
+    Files.write(fileContents, tempFile, Charsets.UTF_8)
+    tempFile
+  }
+
+  private class DummyMountSmallFilesBootstrap extends MountSmallFilesBootstrap {
+    val LABEL_KEY = "smallFilesLabelKey"
+    val LABEL_VALUE = "smallFilesLabelValue"
+    val ENV_KEY = "smallFilesEnvKey"
+    val ENV_VALUE = "smallFilesEnvValue"
+
+    override def mountSmallFilesSecret(pod: Pod, container: Container): (Pod, Container) = {
+      val editedPod = new PodBuilder(pod)
+        .editOrNewMetadata()
+          .addToLabels(LABEL_KEY, LABEL_VALUE)
+          .endMetadata()
+        .build()
+      val editedContainer = new ContainerBuilder(container)
+        .addNewEnv()
+          .withName(ENV_KEY)
+          .withValue(ENV_VALUE)
+          .endEnv()
+        .build()
+      (editedPod, editedContainer)
+    }
+  }
+}
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSchedulerImplSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSchedulerImplSuite.scala
new file mode 100644
index 0000000000000..e10b9508530f7
--- /dev/null
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSchedulerImplSuite.scala
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.scheduler.cluster.kubernetes
+
+import io.fabric8.kubernetes.api.model.{Pod, PodSpec, PodStatus}
+import org.mockito.Matchers._
+import org.mockito.Mockito._
+import org.scalatest.BeforeAndAfter
+
+import org.apache.spark.{SparkContext, SparkFunSuite}
+import org.apache.spark.deploy.kubernetes.config._
+import org.apache.spark.scheduler.FakeTask
+
+class KubernetesTaskSchedulerImplSuite extends SparkFunSuite with BeforeAndAfter {
+
+  SparkContext.clearActiveContext()
+  val sc = new SparkContext("local", "test")
+  val backend = mock(classOf[KubernetesClusterSchedulerBackend])
+
+  before {
+    sc.conf.remove(KUBERNETES_DRIVER_CLUSTER_NODENAME_DNS_LOOKUP_ENABLED)
+  }
+
+  test("Create a k8s task set manager") {
+    val sched = new KubernetesTaskSchedulerImpl(sc)
+    sched.kubernetesSchedulerBackend = backend
+    val taskSet = FakeTask.createTaskSet(0)
+
+    val manager = sched.createTaskSetManager(taskSet, maxTaskFailures = 3)
+    assert(manager.isInstanceOf[KubernetesTaskSetManager])
+  }
+
+  test("Gets racks for datanodes") {
+    val rackResolverUtil = mock(classOf[RackResolverUtil])
+    when(rackResolverUtil.isConfigured).thenReturn(true)
+    when(rackResolverUtil.resolveRack(sc.hadoopConfiguration, "kube-node1"))
+      .thenReturn(Option("/rack1"))
+    when(rackResolverUtil.resolveRack(sc.hadoopConfiguration, "kube-node2"))
+      .thenReturn(Option("/rack2"))
+    val sched = new KubernetesTaskSchedulerImpl(sc, rackResolverUtil)
+    sched.kubernetesSchedulerBackend = backend
+    when(backend.getExecutorPodByIP("kube-node1")).thenReturn(None)
+    when(backend.getExecutorPodByIP("kube-node2")).thenReturn(None)
+
+    assert(sched.getRackForHost("kube-node1:60010") == Option("/rack1"))
+    assert(sched.getRackForHost("kube-node2:60010") == Option("/rack2"))
+  }
+
+  test("Gets racks for executor pods") {
+    sc.conf.set(KUBERNETES_DRIVER_CLUSTER_NODENAME_DNS_LOOKUP_ENABLED, true)
+    val rackResolverUtil = mock(classOf[RackResolverUtil])
+    when(rackResolverUtil.isConfigured).thenReturn(true)
+    when(rackResolverUtil.resolveRack(sc.hadoopConfiguration, "kube-node1"))
+      .thenReturn(Option("/rack1"))
+    when(rackResolverUtil.resolveRack(sc.hadoopConfiguration, "kube-node2.mydomain.com"))
+      .thenReturn(Option("/rack2"))
+    when(rackResolverUtil.resolveRack(sc.hadoopConfiguration, "kube-node2"))
+      .thenReturn(None)
+    when(rackResolverUtil.resolveRack(sc.hadoopConfiguration, "192.168.1.5"))
+      .thenReturn(None)
+    val inetAddressUtil = mock(classOf[InetAddressUtil])
+    val sched = new KubernetesTaskSchedulerImpl(sc, rackResolverUtil, inetAddressUtil)
+    sched.kubernetesSchedulerBackend = backend
+
+    val spec1 = mock(classOf[PodSpec])
+    when(spec1.getNodeName).thenReturn("kube-node1")
+    val status1 = mock(classOf[PodStatus])
+    when(status1.getHostIP).thenReturn("192.168.1.4")
+    val pod1 = mock(classOf[Pod])
+    when(pod1.getSpec).thenReturn(spec1)
+    when(pod1.getStatus).thenReturn(status1)
+    when(backend.getExecutorPodByIP("10.0.0.1")).thenReturn(Some(pod1))
+
+    val spec2 = mock(classOf[PodSpec])
+    when(spec2.getNodeName).thenReturn("kube-node2")
+    val status2 = mock(classOf[PodStatus])
+    when(status2.getHostIP).thenReturn("192.168.1.5")
+    val pod2 = mock(classOf[Pod])
+    when(pod2.getSpec).thenReturn(spec2)
+    when(pod2.getStatus).thenReturn(status2)
+    when(inetAddressUtil.getFullHostName("192.168.1.5")).thenReturn("kube-node2.mydomain.com")
+    when(backend.getExecutorPodByIP("10.0.1.1")).thenReturn(Some(pod2))
+
+    assert(sched.getRackForHost("10.0.0.1:7079") == Option("/rack1"))
+    assert(sched.getRackForHost("10.0.1.1:7079") == Option("/rack2"))
+
+    verify(inetAddressUtil, times(1)).getFullHostName(anyString())
+  }
+
+  test("Gets racks for executor pods while disabling DNS lookup ") {
+    sc.conf.set(KUBERNETES_DRIVER_CLUSTER_NODENAME_DNS_LOOKUP_ENABLED, false)
+    val rackResolverUtil = mock(classOf[RackResolverUtil])
+    when(rackResolverUtil.isConfigured).thenReturn(true)
+    when(rackResolverUtil.resolveRack(sc.hadoopConfiguration, "kube-node1"))
+      .thenReturn(Option("/rack1"))
+    when(rackResolverUtil.resolveRack(sc.hadoopConfiguration, "kube-node2.mydomain.com"))
+      .thenReturn(Option("/rack2"))
+    when(rackResolverUtil.resolveRack(sc.hadoopConfiguration, "kube-node2"))
+      .thenReturn(None)
+    when(rackResolverUtil.resolveRack(sc.hadoopConfiguration, "192.168.1.5"))
+      .thenReturn(None)
+    val inetAddressUtil = mock(classOf[InetAddressUtil])
+    val sched = new KubernetesTaskSchedulerImpl(sc, rackResolverUtil, inetAddressUtil)
+    sched.kubernetesSchedulerBackend = backend
+
+    val spec1 = mock(classOf[PodSpec])
+    when(spec1.getNodeName).thenReturn("kube-node1")
+    val status1 = mock(classOf[PodStatus])
+    when(status1.getHostIP).thenReturn("192.168.1.4")
+    val pod1 = mock(classOf[Pod])
+    when(pod1.getSpec).thenReturn(spec1)
+    when(pod1.getStatus).thenReturn(status1)
+    when(backend.getExecutorPodByIP("10.0.0.1")).thenReturn(Some(pod1))
+
+    val spec2 = mock(classOf[PodSpec])
+    when(spec2.getNodeName).thenReturn("kube-node2")
+    val status2 = mock(classOf[PodStatus])
+    when(status2.getHostIP).thenReturn("192.168.1.5")
+    val pod2 = mock(classOf[Pod])
+    when(pod2.getSpec).thenReturn(spec2)
+    when(pod2.getStatus).thenReturn(status2)
+    when(inetAddressUtil.getFullHostName("192.168.1.5")).thenReturn("kube-node2.mydomain.com")
+    when(backend.getExecutorPodByIP("10.0.1.1")).thenReturn(Some(pod2))
+
+    assert(sched.getRackForHost("10.0.0.1:7079") == Option("/rack1"))
+    assert(sched.getRackForHost("10.0.1.1:7079") == None)
+
+    verify(inetAddressUtil, never).getFullHostName(anyString())
+  }
+
+  test("Does not get racks if plugin is not configured") {
+    val rackResolverUtil = mock(classOf[RackResolverUtil])
+    when(rackResolverUtil.isConfigured()).thenReturn(false)
+    val sched = new KubernetesTaskSchedulerImpl(sc, rackResolverUtil)
+    sched.kubernetesSchedulerBackend = backend
+    when(backend.getExecutorPodByIP("kube-node1")).thenReturn(None)
+
+    val spec1 = mock(classOf[PodSpec])
+    when(spec1.getNodeName).thenReturn("kube-node1")
+    val status1 = mock(classOf[PodStatus])
+    when(status1.getHostIP).thenReturn("192.168.1.4")
+    val pod1 = mock(classOf[Pod])
+    when(pod1.getSpec).thenReturn(spec1)
+    when(pod1.getStatus).thenReturn(status1)
+    when(backend.getExecutorPodByIP("10.0.0.1")).thenReturn(Some(pod1))
+
+    assert(sched.getRackForHost("kube-node1:60010").isEmpty)
+    assert(sched.getRackForHost("10.0.0.1:7079").isEmpty)
+  }
+}
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSetManagerSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSetManagerSuite.scala
index 7618c137ab22b..889758731a6c1 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSetManagerSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesTaskSetManagerSuite.scala
@@ -20,18 +20,25 @@ import scala.collection.mutable.ArrayBuffer
 
 import io.fabric8.kubernetes.api.model.{Pod, PodSpec, PodStatus}
 import org.mockito.Mockito._
+import org.scalatest.BeforeAndAfter
 
 import org.apache.spark.{SparkContext, SparkFunSuite}
+import org.apache.spark.deploy.kubernetes.config._
 import org.apache.spark.scheduler.{FakeTask, FakeTaskScheduler, HostTaskLocation, TaskLocation}
 
-class KubernetesTaskSetManagerSuite extends SparkFunSuite {
+class KubernetesTaskSetManagerSuite extends SparkFunSuite with BeforeAndAfter {
 
+  SparkContext.clearActiveContext()
   val sc = new SparkContext("local", "test")
   val sched = new FakeTaskScheduler(sc,
     ("execA", "10.0.0.1"), ("execB", "10.0.0.2"), ("execC", "10.0.0.3"))
   val backend = mock(classOf[KubernetesClusterSchedulerBackend])
   sched.backend = backend
 
+  before {
+    sc.conf.remove(KUBERNETES_DRIVER_CLUSTER_NODENAME_DNS_LOOKUP_ENABLED)
+  }
+
   test("Find pending tasks for executors using executor pod IP addresses") {
     val taskSet = FakeTask.createTaskSet(3,
       Seq(TaskLocation("10.0.0.1", "execA")),  // Task 0 runs on executor pod 10.0.0.1.
@@ -76,7 +83,33 @@ class KubernetesTaskSetManagerSuite extends SparkFunSuite {
     assert(manager.getPendingTasksForHost("10.0.0.1") == ArrayBuffer(1, 0))
   }
 
+  test("Test DNS lookup is disabled by default for cluster node full hostnames") {
+    assert(!sc.conf.get(KUBERNETES_DRIVER_CLUSTER_NODENAME_DNS_LOOKUP_ENABLED))
+  }
+
+  test("Find pending tasks for executors, but avoid looking up cluster node FQDNs from DNS") {
+    sc.conf.set(KUBERNETES_DRIVER_CLUSTER_NODENAME_DNS_LOOKUP_ENABLED, false)
+    val taskSet = FakeTask.createTaskSet(2,
+      Seq(HostTaskLocation("kube-node1.domain1")),  // Task 0's partition belongs to datanode here.
+      Seq(HostTaskLocation("kube-node1.domain1"))   // task 1's partition belongs to datanode here.
+    )
+    val spec1 = mock(classOf[PodSpec])
+    when(spec1.getNodeName).thenReturn("kube-node1")
+    val pod1 = mock(classOf[Pod])
+    when(pod1.getSpec).thenReturn(spec1)
+    val status1 = mock(classOf[PodStatus])
+    when(status1.getHostIP).thenReturn("196.0.0.5")
+    when(pod1.getStatus).thenReturn(status1)
+    val inetAddressUtil = mock(classOf[InetAddressUtil])
+    when(inetAddressUtil.getFullHostName("196.0.0.5")).thenReturn("kube-node1.domain1")
+    when(backend.getExecutorPodByIP("10.0.0.1")).thenReturn(Some(pod1))
+
+    val manager = new KubernetesTaskSetManager(sched, taskSet, maxTaskFailures = 2, inetAddressUtil)
+    assert(manager.getPendingTasksForHost("10.0.0.1") == ArrayBuffer())
+  }
+
   test("Find pending tasks for executors using cluster node FQDNs that executor pods run on") {
+    sc.conf.set(KUBERNETES_DRIVER_CLUSTER_NODENAME_DNS_LOOKUP_ENABLED, true)
     val taskSet = FakeTask.createTaskSet(2,
       Seq(HostTaskLocation("kube-node1.domain1")),  // Task 0's partition belongs to datanode here.
       Seq(HostTaskLocation("kube-node1.domain1"))   // task 1's partition belongs to datanode here.
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/kubernetes/RackResolverUtilImplSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/kubernetes/RackResolverUtilImplSuite.scala
new file mode 100644
index 0000000000000..ee671ce7d6590
--- /dev/null
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/kubernetes/RackResolverUtilImplSuite.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.scheduler.cluster.kubernetes
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic
+
+import org.apache.spark.SparkFunSuite
+
+class RackResolverUtilImplSuite extends SparkFunSuite {
+
+  test("Detects if topology plugin is configured") {
+    val hadoopConfiguration = new Configuration
+    val rackResolverUtil = new RackResolverUtilImpl(hadoopConfiguration)
+
+    assert(!rackResolverUtil.checkConfigured(hadoopConfiguration))
+    hadoopConfiguration.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
+      rackResolverUtil.scriptPlugin)
+    assert(!rackResolverUtil.checkConfigured(hadoopConfiguration))
+    hadoopConfiguration.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY,
+      "my-script")
+    assert(rackResolverUtil.checkConfigured(hadoopConfiguration))
+
+    hadoopConfiguration.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
+      rackResolverUtil.tablePlugin)
+    assert(!rackResolverUtil.checkConfigured(hadoopConfiguration))
+    hadoopConfiguration.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY,
+      "my-table")
+    assert(rackResolverUtil.checkConfigured(hadoopConfiguration))
+
+    hadoopConfiguration.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
+      "my.Plugin")
+    assert(rackResolverUtil.checkConfigured(hadoopConfiguration))
+  }
+}
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/pom.xml b/resource-managers/kubernetes/docker-minimal-bundle/pom.xml
index 5f7683280cccf..585d8d474bba5 100644
--- a/resource-managers/kubernetes/docker-minimal-bundle/pom.xml
+++ b/resource-managers/kubernetes/docker-minimal-bundle/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/assembly/docker-assembly.xml b/resource-managers/kubernetes/docker-minimal-bundle/src/main/assembly/docker-assembly.xml
index 2b48d366256fe..e6de2c11a0ecf 100644
--- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/assembly/docker-assembly.xml
+++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/assembly/docker-assembly.xml
@@ -69,6 +69,7 @@
       <excludes>
         <exclude>org.apache.spark:spark-assembly_${scala.binary.version}:pom</exclude>
         <exclude>org.spark-project.spark:unused</exclude>
+        <exclude>com.sun.jersey:*</exclude>
       </excludes>
     </dependencySet>
   </dependencySets>
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile
index 6dcc7511c0dd9..7b1effa911f19 100644
--- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile
+++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile
@@ -39,10 +39,11 @@ ENV PYSPARK_DRIVER_PYTHON python
 ENV PYTHONPATH ${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${PYTHONPATH}
 
 CMD SPARK_CLASSPATH="${SPARK_HOME}/jars/*" && \
+    env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt && \
+    readarray -t SPARK_DRIVER_JAVA_OPTS < /tmp/java_opts.txt && \
     if ! [ -z ${SPARK_MOUNTED_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH"; fi && \
     if ! [ -z ${SPARK_SUBMIT_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_SUBMIT_EXTRA_CLASSPATH:$SPARK_CLASSPATH"; fi && \
     if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH"; fi && \
-    if ! [ -z ${SPARK_MOUNTED_FILES_DIR} ]; then cp -R "$SPARK_MOUNTED_FILES_DIR/." .; fi && \
-    exec /sbin/tini -- ${JAVA_HOME}/bin/java $SPARK_DRIVER_JAVA_OPTS -cp $SPARK_CLASSPATH \
-    -Xms$SPARK_DRIVER_MEMORY -Xmx$SPARK_DRIVER_MEMORY \
-    $SPARK_DRIVER_CLASS $PYSPARK_PRIMARY $PYSPARK_FILES $SPARK_DRIVER_ARGS
+    if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R "$SPARK_MOUNTED_FILES_DIR/." .; fi && \
+    if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR+x} ]; then cp -R "$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/." .; fi && \
+    ${JAVA_HOME}/bin/java "${SPARK_DRIVER_JAVA_OPTS[@]}" -cp $SPARK_CLASSPATH -Xms$SPARK_DRIVER_MEMORY -Xmx$SPARK_DRIVER_MEMORY $SPARK_DRIVER_CLASS $PYSPARK_PRIMARY $PYSPARK_FILES $SPARK_DRIVER_ARGS
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver/Dockerfile
index 6bbff8ef64a0f..26d1d805fde2b 100644
--- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver/Dockerfile
+++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver/Dockerfile
@@ -24,8 +24,11 @@ FROM spark-base
 COPY examples /opt/spark/examples
 
 CMD SPARK_CLASSPATH="${SPARK_HOME}/jars/*" && \
+    env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt && \
+    readarray -t SPARK_DRIVER_JAVA_OPTS < /tmp/java_opts.txt && \
     if ! [ -z ${SPARK_MOUNTED_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH"; fi && \
     if ! [ -z ${SPARK_SUBMIT_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_SUBMIT_EXTRA_CLASSPATH:$SPARK_CLASSPATH"; fi && \
     if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH"; fi && \
-    if ! [ -z ${SPARK_MOUNTED_FILES_DIR} ]; then cp -R "$SPARK_MOUNTED_FILES_DIR/." .; fi && \
-    exec /sbin/tini -- ${JAVA_HOME}/bin/java $SPARK_DRIVER_JAVA_OPTS -cp $SPARK_CLASSPATH -Xms$SPARK_DRIVER_MEMORY -Xmx$SPARK_DRIVER_MEMORY $SPARK_DRIVER_CLASS $SPARK_DRIVER_ARGS
+    if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R "$SPARK_MOUNTED_FILES_DIR/." .; fi && \
+    if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR} ]; then cp -R "$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/." .; fi && \
+    ${JAVA_HOME}/bin/java "${SPARK_DRIVER_JAVA_OPTS[@]}" -cp $SPARK_CLASSPATH -Xms$SPARK_DRIVER_MEMORY -Xmx$SPARK_DRIVER_MEMORY $SPARK_DRIVER_CLASS $SPARK_DRIVER_ARGS
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile
index 7a65a4f879376..a8bb5b362ab52 100644
--- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile
+++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile
@@ -38,9 +38,11 @@ ENV PYSPARK_PYTHON python
 ENV PYSPARK_DRIVER_PYTHON python
 ENV PYTHONPATH ${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${PYTHONPATH}
 
-# TODO support spark.executor.extraClassPath
 CMD SPARK_CLASSPATH="${SPARK_HOME}/jars/*" && \
+    env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt && \
+    readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt && \
     if ! [ -z ${SPARK_MOUNTED_CLASSPATH}+x} ]; then SPARK_CLASSPATH="$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH"; fi && \
     if ! [ -z ${SPARK_EXECUTOR_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_EXECUTOR_EXTRA_CLASSPATH:$SPARK_CLASSPATH"; fi && \
-    if ! [ -z ${SPARK_MOUNTED_FILES_DIR} ]; then cp -R "$SPARK_MOUNTED_FILES_DIR/." .; fi && \
-    exec /sbin/tini -- ${JAVA_HOME}/bin/java -Dspark.executor.port=$SPARK_EXECUTOR_PORT -Xms$SPARK_EXECUTOR_MEMORY -Xmx$SPARK_EXECUTOR_MEMORY -cp $SPARK_CLASSPATH org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url $SPARK_DRIVER_URL --executor-id $SPARK_EXECUTOR_ID --cores $SPARK_EXECUTOR_CORES --app-id $SPARK_APPLICATION_ID --hostname $SPARK_EXECUTOR_POD_IP
\ No newline at end of file
+    if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R "$SPARK_MOUNTED_FILES_DIR/." .; fi && \
+    if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR+x} ]; then cp -R "$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/." .; fi && \
+    ${JAVA_HOME}/bin/java "${SPARK_EXECUTOR_JAVA_OPTS[@]}" -Dspark.executor.port=$SPARK_EXECUTOR_PORT -Xms$SPARK_EXECUTOR_MEMORY -Xmx$SPARK_EXECUTOR_MEMORY -cp $SPARK_CLASSPATH org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url $SPARK_DRIVER_URL --executor-id $SPARK_EXECUTOR_ID --cores $SPARK_EXECUTOR_CORES --app-id $SPARK_APPLICATION_ID --hostname $SPARK_EXECUTOR_POD_IP
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor/Dockerfile
index b3b0acc3b64b8..ab9f67e95a8e5 100644
--- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor/Dockerfile
+++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor/Dockerfile
@@ -23,10 +23,12 @@ FROM spark-base
 
 COPY examples /opt/spark/examples
 
-# TODO support spark.executor.extraClassPath
 CMD SPARK_CLASSPATH="${SPARK_HOME}/jars/*" && \
+    env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt && \
+    readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt && \
     if ! [ -z ${SPARK_MOUNTED_CLASSPATH}+x} ]; then SPARK_CLASSPATH="$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH"; fi && \
     if ! [ -z ${SPARK_EXECUTOR_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_EXECUTOR_EXTRA_CLASSPATH:$SPARK_CLASSPATH"; fi && \
     if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH"; fi && \
-    if ! [ -z ${SPARK_MOUNTED_FILES_DIR} ]; then cp -R "$SPARK_MOUNTED_FILES_DIR/." .; fi && \
-    exec /sbin/tini -- ${JAVA_HOME}/bin/java -Dspark.executor.port=$SPARK_EXECUTOR_PORT -Xms$SPARK_EXECUTOR_MEMORY -Xmx$SPARK_EXECUTOR_MEMORY -cp $SPARK_CLASSPATH org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url $SPARK_DRIVER_URL --executor-id $SPARK_EXECUTOR_ID --cores $SPARK_EXECUTOR_CORES --app-id $SPARK_APPLICATION_ID --hostname $SPARK_EXECUTOR_POD_IP
+    if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R "$SPARK_MOUNTED_FILES_DIR/." .; fi && \
+    if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR+x} ]; then cp -R "$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/." .; fi && \
+    ${JAVA_HOME}/bin/java "${SPARK_EXECUTOR_JAVA_OPTS[@]}" -Dspark.executor.port=$SPARK_EXECUTOR_PORT -Xms$SPARK_EXECUTOR_MEMORY -Xmx$SPARK_EXECUTOR_MEMORY -cp $SPARK_CLASSPATH org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url $SPARK_DRIVER_URL --executor-id $SPARK_EXECUTOR_ID --cores $SPARK_EXECUTOR_CORES --app-id $SPARK_APPLICATION_ID --hostname $SPARK_EXECUTOR_POD_IP
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/init-container/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/init-container/Dockerfile
index 4bafe25e2608f..1178dd2428448 100644
--- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/init-container/Dockerfile
+++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/init-container/Dockerfile
@@ -21,4 +21,4 @@ FROM spark-base
 # command should be invoked from the top level directory of the Spark distribution. E.g.:
 # docker build -t spark-init:latest -f dockerfiles/init-container/Dockerfile .
 
-ENTRYPOINT [ "/sbin/tini", "--", "bin/spark-class", "org.apache.spark.deploy.rest.kubernetes.KubernetesSparkDependencyDownloadInitContainer" ]
+ENTRYPOINT [ "/opt/entrypoint.sh", "/opt/spark/bin/spark-class", "org.apache.spark.deploy.rest.kubernetes.KubernetesSparkDependencyDownloadInitContainer" ]
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/resource-staging-server/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/resource-staging-server/Dockerfile
index 9ca96be0f1a88..87ed7d10f3eb3 100644
--- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/resource-staging-server/Dockerfile
+++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/resource-staging-server/Dockerfile
@@ -22,4 +22,4 @@ FROM spark-base
 # command should be invoked from the top level directory of the Spark distribution. E.g.:
 # docker build -t spark-resource-staging-server:latest -f dockerfiles/resource-staging-server/Dockerfile .
 
-ENTRYPOINT [ "/sbin/tini", "--", "bin/spark-class", "org.apache.spark.deploy.rest.kubernetes.ResourceStagingServer" ]
+ENTRYPOINT [ "/opt/entrypoint.sh", "/opt/spark/bin/spark-class", "org.apache.spark.deploy.rest.kubernetes.ResourceStagingServer" ]
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/shuffle-service/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/shuffle-service/Dockerfile
index ccb2f1a03d88c..b76e66d316c5c 100644
--- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/shuffle-service/Dockerfile
+++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/shuffle-service/Dockerfile
@@ -21,6 +21,4 @@ FROM spark-base
 # command should be invoked from the top level directory of the Spark distribution. E.g.:
 # docker build -t spark-shuffle:latest -f dockerfiles/shuffle-service/Dockerfile .
 
-COPY examples /opt/spark/examples
-
-ENTRYPOINT [ "/sbin/tini", "--", "bin/spark-class", "org.apache.spark.deploy.kubernetes.KubernetesExternalShuffleService", "1" ]
+ENTRYPOINT [ "/opt/entrypoint.sh", "/opt/spark/bin/spark-class", "org.apache.spark.deploy.kubernetes.KubernetesExternalShuffleService", "1" ]
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/spark-base/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/spark-base/Dockerfile
index b0925e3bb0416..83d1efa7f3a5f 100644
--- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/spark-base/Dockerfile
+++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/spark-base/Dockerfile
@@ -23,13 +23,20 @@ FROM openjdk:8-alpine
 RUN apk upgrade --no-cache && \
     apk add --no-cache bash tini && \
     mkdir -p /opt/spark && \
-    touch /opt/spark/RELEASE
+    mkdir -p /opt/spark/work-dir \
+    touch /opt/spark/RELEASE && \
+    rm /bin/sh && \
+    ln -sv /bin/bash /bin/sh && \
+    chgrp root /etc/passwd && chmod ug+rw /etc/passwd
 
 COPY jars /opt/spark/jars
 COPY bin /opt/spark/bin
 COPY sbin /opt/spark/sbin
 COPY conf /opt/spark/conf
+COPY dockerfiles/spark-base/entrypoint.sh /opt/
 
 ENV SPARK_HOME /opt/spark
 
-WORKDIR /opt/spark
+WORKDIR /opt/spark/work-dir
+
+ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/spark-base/entrypoint.sh b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/spark-base/entrypoint.sh
new file mode 100755
index 0000000000000..10690ec8de623
--- /dev/null
+++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/spark-base/entrypoint.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# echo commands to the terminal output
+set -x
+
+# Check whether there is a passwd entry for the container UID
+myuid=$(id -u)
+mygid=$(id -g)
+uidentry=$(getent passwd $myuid)
+
+# If there is no passwd entry for the container UID, attempt to create one
+if [ -z "$uidentry" ] ; then
+    if [ -w /etc/passwd ] ; then
+        echo "$myuid:x:$myuid:$mygid:anonymous uid:$SPARK_HOME:/bin/false" >> /etc/passwd
+    else
+        echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
+    fi
+fi
+
+# Execute the container CMD under tini for better hygiene
+/sbin/tini -s -- "$@"
diff --git a/resource-managers/kubernetes/integration-tests-spark-jobs-helpers/pom.xml b/resource-managers/kubernetes/integration-tests-spark-jobs-helpers/pom.xml
index 9672f49448c0c..2e877daf95c63 100644
--- a/resource-managers/kubernetes/integration-tests-spark-jobs-helpers/pom.xml
+++ b/resource-managers/kubernetes/integration-tests-spark-jobs-helpers/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/integration-tests-spark-jobs/pom.xml b/resource-managers/kubernetes/integration-tests-spark-jobs/pom.xml
index 80b0f57a2fcb9..276bbabdf32e1 100644
--- a/resource-managers/kubernetes/integration-tests-spark-jobs/pom.xml
+++ b/resource-managers/kubernetes/integration-tests-spark-jobs/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/integration-tests-spark-jobs/src/main/scala/org/apache/spark/deploy/kubernetes/integrationtest/jobs/FileExistenceTest.scala b/resource-managers/kubernetes/integration-tests-spark-jobs/src/main/scala/org/apache/spark/deploy/kubernetes/integrationtest/jobs/FileExistenceTest.scala
index 8994c998bffee..a9e328f4ff248 100644
--- a/resource-managers/kubernetes/integration-tests-spark-jobs/src/main/scala/org/apache/spark/deploy/kubernetes/integrationtest/jobs/FileExistenceTest.scala
+++ b/resource-managers/kubernetes/integration-tests-spark-jobs/src/main/scala/org/apache/spark/deploy/kubernetes/integrationtest/jobs/FileExistenceTest.scala
@@ -16,6 +16,7 @@
  */
 package org.apache.spark.deploy.kubernetes.integrationtest.jobs
 
+import java.io.File
 import java.nio.file.Paths
 
 import com.google.common.base.Charsets
@@ -45,6 +46,20 @@ private[spark] object FileExistenceTest {
       } else {
         println(s"File found at ${file.getAbsolutePath} with correct contents.")
       }
+      val spark = SparkSession.builder().getOrCreate().sparkContext
+      val fileNamesRdd = spark.parallelize(Seq(args(0)))
+      if (fileNamesRdd.filter(fileName => new File(fileName).isFile()).count() != 1) {
+        throw new SparkException(s"Executors do not have the file ${args(0)}.")
+      }
+      val matchingContents = fileNamesRdd.map { fileName =>
+        Files.toString(new File(fileName), Charsets.UTF_8)
+      }.filter(_.equals(args(1)))
+      if (matchingContents.count() != 1) {
+        throw new SparkException(s"The file on the executors at ${args(0)} did not have" +
+          s" the correct contents.")
+      }
+      println(s"File found on the executors at the relative path ${args(0)} with the" +
+        s" correct contents.")
       // scalastyle:on println
     }
     while (true) {
diff --git a/resource-managers/kubernetes/integration-tests-spark-jobs/src/main/scala/org/apache/spark/deploy/kubernetes/integrationtest/jobs/JavaOptionsTest.scala b/resource-managers/kubernetes/integration-tests-spark-jobs/src/main/scala/org/apache/spark/deploy/kubernetes/integrationtest/jobs/JavaOptionsTest.scala
new file mode 100644
index 0000000000000..7d457fc044227
--- /dev/null
+++ b/resource-managers/kubernetes/integration-tests-spark-jobs/src/main/scala/org/apache/spark/deploy/kubernetes/integrationtest/jobs/JavaOptionsTest.scala
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.deploy.kubernetes.integrationtest.jobs
+
+import java.io.{File, FileInputStream}
+import java.util.Properties
+
+import com.google.common.collect.Maps
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.util.Utils
+
+private[spark] object JavaOptionsTest {
+
+  def main(args: Array[String]): Unit = {
+    // scalastyle:off println
+    if (args.length != 2) {
+      println(s"Invalid arguments: ${args.mkString(",")}." +
+        s"Usage: JavaOptionsTest <driver-java-options-list-file> <executor-java-options-list-file>")
+      System.exit(1)
+    }
+    val expectedDriverJavaOptions = loadPropertiesFromFile(args(0))
+    val expectedExecutorJavaOptions = loadPropertiesFromFile(args(1))
+    val nonMatchingDriverOptions = expectedDriverJavaOptions.filter {
+      case (optKey, optValue) => System.getProperty(optKey) != optValue
+    }
+    if (nonMatchingDriverOptions.nonEmpty) {
+      println(s"The driver's JVM options did not match. Expected $expectedDriverJavaOptions." +
+        s" But these options did not match: $nonMatchingDriverOptions.")
+      val sysProps = Maps.fromProperties(System.getProperties).asScala
+      println("Driver system properties are:")
+      for (prop <- sysProps) {
+        println(s"Key: ${prop._1}, Value: ${prop._2}")
+      }
+      System.exit(1)
+    }
+
+    val spark = SparkSession.builder().getOrCreate().sparkContext
+    val nonMatchingExecutorOptions = try {
+      spark.parallelize(Seq(0)).flatMap { _ =>
+        expectedExecutorJavaOptions.filter {
+          case (optKey, optValue) => System.getProperty(optKey) != optValue
+        }
+      }.collectAsMap()
+    } finally {
+      spark.stop()
+    }
+    if (nonMatchingExecutorOptions.nonEmpty) {
+      val executorSysProps = spark.parallelize(Seq(0)).flatMap { _ =>
+        Maps.fromProperties(System.getProperties).asScala
+      }.collectAsMap()
+      println(s"The executor's JVM options did not match. Expected" +
+        s" $expectedExecutorJavaOptions. But these options did not" +
+        s" match: $nonMatchingExecutorOptions.")
+      println("Executor system properties are:")
+      for (prop <- executorSysProps) {
+        println(s"Key: ${prop._1}, Value: ${prop._2}")
+      }
+      System.exit(1)
+    } else {
+      println("All expected JVM options were present on the driver and executors.")
+    }
+    // scalastyle:on println
+  }
+
+  private def loadPropertiesFromFile(filePath: String): Map[String, String] = {
+    val file = new File(filePath)
+    if (!file.isFile) {
+      throw new IllegalArgumentException(s"File not found at $filePath or is not a file.")
+    }
+    val properties = new Properties()
+    Utils.tryWithResource(new FileInputStream(file)) { is =>
+      properties.load(is)
+    }
+    Maps.fromProperties(properties).asScala.toMap
+  }
+}
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 0da1e38d8c211..fc28026de1e43 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesSuite.scala
index c6cd6a74c88d1..e204d0173aff8 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesSuite.scala
@@ -16,9 +16,9 @@
  */
 package org.apache.spark.deploy.kubernetes.integrationtest
 
-import java.io.File
+import java.io.{File, FileOutputStream}
 import java.nio.file.Paths
-import java.util.UUID
+import java.util.{Properties, UUID}
 
 import com.google.common.base.Charsets
 import com.google.common.io.Files
@@ -213,19 +213,70 @@ private[spark] class KubernetesSuite extends SparkFunSuite with BeforeAndAfter {
 
   test("Added files should be placed in the driver's working directory.") {
     assume(testBackend.name == MINIKUBE_TEST_BACKEND)
+    launchStagingServer(SSLOptions(), None)
     val testExistenceFileTempDir = Utils.createTempDir(namePrefix = "test-existence-file-temp-dir")
     val testExistenceFile = new File(testExistenceFileTempDir, "input.txt")
     Files.write(TEST_EXISTENCE_FILE_CONTENTS, testExistenceFile, Charsets.UTF_8)
-    launchStagingServer(SSLOptions(), None)
     sparkConf.set("spark.files", testExistenceFile.getAbsolutePath)
     runSparkApplicationAndVerifyCompletion(
         JavaMainAppResource(SUBMITTER_LOCAL_MAIN_APP_RESOURCE),
         FILE_EXISTENCE_MAIN_CLASS,
-        Seq(s"File found at /opt/spark/${testExistenceFile.getName} with correct contents."),
+        Seq(
+          s"File found at /opt/spark/work-dir/${testExistenceFile.getName} with correct contents.",
+          s"File found on the executors at the relative path ${testExistenceFile.getName} with" +
+            s" the correct contents."),
         Array(testExistenceFile.getName, TEST_EXISTENCE_FILE_CONTENTS),
         Seq.empty[String])
   }
 
+  test("Setting JVM options on the driver and executors with spaces.") {
+    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
+    launchStagingServer(SSLOptions(), None)
+    val driverJvmOptionsFile = storeJvmOptionsInTempFile(
+        Map("simpleDriverConf" -> "simpleDriverConfValue",
+            "driverconfwithspaces" -> "driver conf with spaces value"),
+        "driver-jvm-options.properties",
+        "JVM options that should be set on the driver.")
+    val executorJvmOptionsFile = storeJvmOptionsInTempFile(
+        Map("simpleExecutorConf" -> "simpleExecutorConfValue",
+            "executor conf with spaces" -> "executor conf with spaces value"),
+        "executor-jvm-options.properties",
+        "JVM options that should be set on the executors.")
+    sparkConf.set(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS,
+        "-DsimpleDriverConf=simpleDriverConfValue" +
+            " -Ddriverconfwithspaces='driver conf with spaces value'")
+    sparkConf.set(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS,
+        "-DsimpleExecutorConf=simpleExecutorConfValue" +
+            " -D\'executor conf with spaces\'=\'executor conf with spaces value\'")
+    sparkConf.set("spark.files",
+        Seq(driverJvmOptionsFile.getAbsolutePath, executorJvmOptionsFile.getAbsolutePath)
+            .mkString(","))
+    runSparkApplicationAndVerifyCompletion(
+        JavaMainAppResource(SUBMITTER_LOCAL_MAIN_APP_RESOURCE),
+        JAVA_OPTIONS_MAIN_CLASS,
+        Seq(s"All expected JVM options were present on the driver and executors."),
+        Array(driverJvmOptionsFile.getName, executorJvmOptionsFile.getName),
+        Seq.empty[String])
+  }
+
+  test("Submit small local files without the resource staging server.") {
+    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
+    sparkConf.setJars(Seq(CONTAINER_LOCAL_HELPER_JAR_PATH))
+    val testExistenceFileTempDir = Utils.createTempDir(namePrefix = "test-existence-file-temp-dir")
+    val testExistenceFile = new File(testExistenceFileTempDir, "input.txt")
+    Files.write(TEST_EXISTENCE_FILE_CONTENTS, testExistenceFile, Charsets.UTF_8)
+    sparkConf.set("spark.files", testExistenceFile.getAbsolutePath)
+    runSparkApplicationAndVerifyCompletion(
+      JavaMainAppResource(CONTAINER_LOCAL_MAIN_APP_RESOURCE),
+      FILE_EXISTENCE_MAIN_CLASS,
+      Seq(
+        s"File found at /opt/spark/work-dir/${testExistenceFile.getName} with correct contents.",
+        s"File found on the executors at the relative path ${testExistenceFile.getName} with" +
+          s" the correct contents."),
+      Array(testExistenceFile.getName, TEST_EXISTENCE_FILE_CONTENTS),
+      Seq.empty[String])
+  }
+
   test("Use a very long application name.") {
     assume(testBackend.name == MINIKUBE_TEST_BACKEND)
 
@@ -339,6 +390,20 @@ private[spark] class KubernetesSuite extends SparkFunSuite with BeforeAndAfter {
       }
     }
   }
+
+  private def storeJvmOptionsInTempFile(
+      options: Map[String, String],
+      propertiesFileName: String,
+      comments: String): File = {
+    val tempDir = Utils.createTempDir()
+    val propertiesFile = new File(tempDir, propertiesFileName)
+    val properties = new Properties()
+    options.foreach { case (propKey, propValue) => properties.setProperty(propKey, propValue) }
+    Utils.tryWithResource(new FileOutputStream(propertiesFile)) { os =>
+      properties.store(os, comments)
+    }
+    propertiesFile
+  }
 }
 
 private[spark] object KubernetesSuite {
@@ -368,6 +433,8 @@ private[spark] object KubernetesSuite {
     ".integrationtest.jobs.FileExistenceTest"
   val GROUP_BY_MAIN_CLASS = "org.apache.spark.deploy.kubernetes" +
     ".integrationtest.jobs.GroupByTest"
+  val JAVA_OPTIONS_MAIN_CLASS = "org.apache.spark.deploy.kubernetes" +
+    ".integrationtest.jobs.JavaOptionsTest"
   val TEST_EXISTENCE_FILE_CONTENTS = "contents"
 
   case object ShuffleNotReadyException extends Exception
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/SparkReadinessWatcher.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/SparkReadinessWatcher.scala
index 20517eb2fc2a6..bd604ab94b936 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/SparkReadinessWatcher.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/SparkReadinessWatcher.scala
@@ -37,5 +37,5 @@ private[spark] class SparkReadinessWatcher[T <: HasMetadata] extends Watcher[T]
 
   override def onClose(cause: KubernetesClientException): Unit = {}
 
-  def waitUntilReady(): Boolean = signal.get(30, TimeUnit.SECONDS)
+  def waitUntilReady(): Boolean = signal.get(60, TimeUnit.SECONDS)
 }
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index 72ba1417bd1e7..db492f57120e5 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index c65806e293f25..a59cf8c5eb5ac 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 0aec944d693dd..60fed70731bfd 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 5943112510b0b..4b55cc859005c 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 65c8f520b0f8d..caa5da8b4d8b2 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 73a68b32d8168..df9535de6677b 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 51789b826b673..e5ffe7af8c044 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/tools/pom.xml b/tools/pom.xml
index 066d74c1e286d..7ca0a9bb2d3a3 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>