Sync hdfs-kerberos-support to branch-2.2-kubernetes

README.md

@@ -10,6 +10,7 @@ This is a collaboratively maintained project working on [SPARK-18278](https://is
 
 - [Usage guide](https://apache-spark-on-k8s.github.io/userdocs/) shows how to run the code
 - [Development docs](resource-managers/kubernetes/README.md) shows how to get set up for development
+- [Architecture docs](resource-managers/kubernetes/architecture-docs/) shows the high level architecture of Spark on Kubernetes
 - Code is primarily located in the [resource-managers/kubernetes](resource-managers/kubernetes) folder
 
 ## Why does this fork exist?

assembly/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

common/network-common/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/network-shuffle/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/network-yarn/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/sketch/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/tags/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/unsafe/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

conf/kubernetes-resource-staging-server.yaml

@@ -32,7 +32,7 @@ spec:
             name: spark-resource-staging-server-config
       containers:
         - name: spark-resource-staging-server
-          image: kubespark/spark-resource-staging-server:v2.1.0-kubernetes-0.2.0
+          image: kubespark/spark-resource-staging-server:v2.2.0-kubernetes-0.3.0
           resources:
             requests:
               cpu: 100m

conf/kubernetes-shuffle-service.yaml

@@ -20,14 +20,14 @@ kind: DaemonSet
 metadata:
   labels:
     app: spark-shuffle-service
-    spark-version: 2.1.0
+    spark-version: 2.2.0
   name: shuffle
 spec:
   template:
     metadata:
       labels:
         app: spark-shuffle-service
-        spark-version: 2.1.0
+        spark-version: 2.2.0
     spec:
       volumes:
         - name: temp-volume
@@ -38,7 +38,7 @@ spec:
           # This is an official image that is built
           # from the dockerfiles/shuffle directory
           # in the spark distribution.
-          image: kubespark/spark-shuffle:v2.1.0-kubernetes-0.2.0
+          image: kubespark/spark-shuffle:v2.2.0-kubernetes-0.3.0
           imagePullPolicy: IfNotPresent
           volumeMounts:
             - mountPath: '/tmp'
@@ -51,4 +51,4 @@ spec:
              requests:
                cpu: "1"
              limits:
-               cpu: "1"
+               cpu: "1"

core/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala

@@ -639,7 +639,9 @@ object SparkSubmit extends CommandLineUtils {
       if (args.isPython) {
         childArgs ++= Array("--primary-py-file", args.primaryResource)
         childArgs ++= Array("--main-class", "org.apache.spark.deploy.PythonRunner")
-        childArgs ++= Array("--other-py-files", args.pyFiles)
+        if (args.pyFiles != null) {
+          childArgs ++= Array("--other-py-files", args.pyFiles)
+        }
       } else {
         childArgs ++= Array("--primary-java-resource", args.primaryResource)
         childArgs ++= Array("--main-class", args.mainClass)

docs/running-on-kubernetes.md

@@ -17,8 +17,10 @@ cluster, you may setup a test cluster on your local machine using
 * You must have appropriate permissions to create and list [pods](https://kubernetes.io/docs/user-guide/pods/),
 [ConfigMaps](https://kubernetes.io/docs/tasks/configure-pod-container/configmap/) and
 [secrets](https://kubernetes.io/docs/concepts/configuration/secret/) in your cluster. You can verify that
-you can list these resources by running `kubectl get pods` `kubectl get configmap`, and `kubectl get secrets` which
+you can list these resources by running `kubectl get pods`, `kubectl get configmap`, and `kubectl get secrets` which
 should give you a list of pods and configmaps (if any) respectively.
+  * The service account or credentials used by the driver pods must have appropriate permissions
+    as well for editing pod spec.
 * You must have a spark distribution with Kubernetes support. This may be obtained from the
 [release tarball](https://github.com/apache-spark-on-k8s/spark/releases) or by
 [building Spark with Kubernetes support](../resource-managers/kubernetes/README.md#building-spark-with-kubernetes-support).
@@ -36,15 +38,15 @@ If you wish to use pre-built docker images, you may use the images published in
 <tr><th>Component</th><th>Image</th></tr>
 <tr>
   <td>Spark Driver Image</td>
-  <td><code>kubespark/spark-driver:v2.1.0-kubernetes-0.2.0</code></td>
+  <td><code>kubespark/spark-driver:v2.2.0-kubernetes-0.3.0</code></td>
 </tr>
 <tr>
   <td>Spark Executor Image</td>
-  <td><code>kubespark/spark-executor:v2.1.0-kubernetes-0.2.0</code></td>
+  <td><code>kubespark/spark-executor:v2.2.0-kubernetes-0.3.0</code></td>
 </tr>
 <tr>
   <td>Spark Initialization Image</td>
-  <td><code>kubespark/spark-init:v2.1.0-kubernetes-0.2.0</code></td>
+  <td><code>kubespark/spark-init:v2.2.0-kubernetes-0.3.0</code></td>
 </tr>
 </table>
 
@@ -80,9 +82,9 @@ are set up as described above:
       --kubernetes-namespace default \
       --conf spark.executor.instances=5 \
       --conf spark.app.name=spark-pi \
-      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.1.0-kubernetes-0.2.0 \
+      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.2.0-kubernetes-0.3.0 \
       local:///opt/spark/examples/jars/spark_examples_2.11-2.2.0.jar
 
 The Spark master, specified either via passing the `--master` command line argument to `spark-submit` or by setting
@@ -107,6 +109,18 @@ Finally, notice that in the above example we specify a jar with a specific URI w
 the location of the example jar that is already in the Docker image. Using dependencies that are on your machine's local
 disk is discussed below.
 
+When Kubernetes [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) is enabled,
+the `default` service account used by the driver may not have appropriate pod `edit` permissions
+for launching executor pods. We recommend to add another service account, say `spark`, with
+the necessary privilege. For example:
+
+    kubectl create serviceaccount spark
+    kubectl create clusterrolebinding spark-edit --clusterrole edit  \
+        --serviceaccount default:spark --namespace default
+
+With this, one can add `--conf spark.kubernetes.authenticate.driver.serviceAccountName=spark` to
+the spark-submit command line above to specify the service account to use.
+
 ## Dependency Management
 
 Application dependencies that are being submitted from your machine need to be sent to a **resource staging server**
@@ -129,9 +143,9 @@ and then you can compute the value of Pi as follows:
       --kubernetes-namespace default \
       --conf spark.executor.instances=5 \
       --conf spark.app.name=spark-pi \
-      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.1.0-kubernetes-0.2.0 \
+      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.2.0-kubernetes-0.3.0 \
       --conf spark.kubernetes.resourceStagingServer.uri=http://<address-of-any-cluster-node>:31000 \
       examples/jars/spark_examples_2.11-2.2.0.jar
 
@@ -170,9 +184,9 @@ If our local proxy were listening on port 8001, we would have our submission loo
       --kubernetes-namespace default \
       --conf spark.executor.instances=5 \
       --conf spark.app.name=spark-pi \
-      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.1.0-kubernetes-0.2.0 \
+      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.2.0-kubernetes-0.3.0 \
       local:///opt/spark/examples/jars/spark_examples_2.11-2.2.0.jar
 
 Communication between Spark and Kubernetes clusters is performed using the fabric8 kubernetes-client library.
@@ -220,7 +234,7 @@ service because there may be multiple shuffle service instances running in a clu
 a way to target a particular shuffle service.
 
 For example, if the shuffle service we want to use is in the default namespace, and
-has pods with labels `app=spark-shuffle-service` and `spark-version=2.1.0`, we can
+has pods with labels `app=spark-shuffle-service` and `spark-version=2.2.0`, we can
 use those tags to target that particular shuffle service at job launch time. In order to run a job with dynamic allocation enabled,
 the command may then look like the following:
 
@@ -235,7 +249,7 @@ the command may then look like the following:
       --conf spark.dynamicAllocation.enabled=true \
       --conf spark.shuffle.service.enabled=true \
       --conf spark.kubernetes.shuffle.namespace=default \
-      --conf spark.kubernetes.shuffle.labels="app=spark-shuffle-service,spark-version=2.1.0" \
+      --conf spark.kubernetes.shuffle.labels="app=spark-shuffle-service,spark-version=2.2.0" \
       local:///opt/spark/examples/jars/spark_examples_2.11-2.2.0.jar 10 400000 2
 
 ## Advanced
@@ -312,9 +326,9 @@ communicate with the resource staging server over TLS. The trustStore can be set
       --kubernetes-namespace default \
       --conf spark.executor.instances=5 \
       --conf spark.app.name=spark-pi \
-      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.1.0-kubernetes-0.2.0 \
-      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.1.0-kubernetes-0.2.0 \
+      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.2.0-kubernetes-0.3.0 \
+      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.2.0-kubernetes-0.3.0 \
       --conf spark.kubernetes.resourceStagingServer.uri=https://<address-of-any-cluster-node>:31000 \
       --conf spark.ssl.kubernetes.resourceStagingServer.enabled=true \
       --conf spark.ssl.kubernetes.resourceStagingServer.clientCertPem=/home/myuser/cert.pem \
@@ -768,6 +782,22 @@ from the other deployment modes. See the [configuration page](configuration.html
     <code>myIdentifier</code>. Multiple node selector keys can be added by setting multiple configurations with this prefix.
   </td>
 </tr>
+<tr>
+  <td><code>spark.executorEnv.[EnvironmentVariableName]</code></td> 
+  <td>(none)</td>
+  <td>
+    Add the environment variable specified by <code>EnvironmentVariableName</code> to
+    the Executor process. The user can specify multiple of these to set multiple environment variables.
+  </td>
+</tr>
+<tr>
+  <td><code>spark.kubernetes.driverEnv.[EnvironmentVariableName]</code></td> 
+  <td>(none)</td>
+  <td>
+    Add the environment variable specified by <code>EnvironmentVariableName</code> to
+    the Driver process. The user can specify multiple of these to set multiple environment variables.
+  </td>
+</tr>
 </table>
 
 

examples/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

external/docker-integration-tests/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/flume-assembly/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/flume-sink/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/flume/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/java8-tests/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-10-assembly/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-10-sql/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-10/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-8-assembly/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-8/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kinesis-asl-assembly/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kinesis-asl/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.0-k8s-0.3.0-SNAPSHOT</version>
+    <version>2.2.0-k8s-0.4.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>