Merge pull request apache-spark-on-k8s#254 from palantir/resync-kube

[NOSQUASH] Resync kube

Author: ash211

docs/running-on-kubernetes.md

@@ -85,7 +85,7 @@ are set up as described above:
       --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.2.0-kubernetes-0.3.0 \
       --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.2.0-kubernetes-0.3.0 \
       --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.2.0-kubernetes-0.3.0 \
-      local:///opt/spark/examples/jars/spark_examples_2.11-2.2.0.jar
+      local:///opt/spark/examples/jars/spark-examples_2.11-2.2.0-k8s-0.3.0.jar
 
 The Spark master, specified either via passing the `--master` command line argument to `spark-submit` or by setting
 `spark.master` in the application's configuration, must be a URL with the format `k8s://<api_server_url>`. Prefixing the
@@ -147,7 +147,7 @@ and then you can compute the value of Pi as follows:
       --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.2.0-kubernetes-0.3.0 \
       --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.2.0-kubernetes-0.3.0 \
       --conf spark.kubernetes.resourceStagingServer.uri=http://<address-of-any-cluster-node>:31000 \
-      examples/jars/spark_examples_2.11-2.2.0.jar
+      examples/jars/spark-examples_2.11-2.2.0-k8s-0.3.0.jar
 
 The Docker image for the resource staging server may also be built from source, in a similar manner to the driver
 and executor images. The Dockerfile is provided in `dockerfiles/resource-staging-server/Dockerfile`.
@@ -189,7 +189,7 @@ If our local proxy were listening on port 8001, we would have our submission loo
       --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.2.0-kubernetes-0.3.0 \
       --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.2.0-kubernetes-0.3.0 \
       --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.2.0-kubernetes-0.3.0 \
-      local:///opt/spark/examples/jars/spark_examples_2.11-2.2.0.jar
+      local:///opt/spark/examples/jars/spark-examples_2.11-2.2.0-k8s-0.3.0.jar
 
 Communication between Spark and Kubernetes clusters is performed using the fabric8 kubernetes-client library.
 The above mechanism using `kubectl proxy` can be used when we have authentication providers that the fabric8
@@ -252,7 +252,7 @@ the command may then look like the following:
       --conf spark.shuffle.service.enabled=true \
       --conf spark.kubernetes.shuffle.namespace=default \
       --conf spark.kubernetes.shuffle.labels="app=spark-shuffle-service,spark-version=2.2.0" \
-      local:///opt/spark/examples/jars/spark_examples_2.11-2.2.0.jar 10 400000 2
+      local:///opt/spark/examples/jars/spark-examples_2.11-2.2.0-k8s-0.3.0.jar 10 400000 2
 
 ## Advanced
 
@@ -334,7 +334,7 @@ communicate with the resource staging server over TLS. The trustStore can be set
       --conf spark.kubernetes.resourceStagingServer.uri=https://<address-of-any-cluster-node>:31000 \
       --conf spark.ssl.kubernetes.resourceStagingServer.enabled=true \
       --conf spark.ssl.kubernetes.resourceStagingServer.clientCertPem=/home/myuser/cert.pem \
-      examples/jars/spark_examples_2.11-2.2.0.jar
+      examples/jars/spark-examples_2.11-2.2.0-k8s-0.3.0.jar
 
 ### Spark Properties
 

resource-managers/kubernetes/architecture-docs/scheduler-backend.md

@@ -17,8 +17,7 @@ When a job is running, the scheduler backend configures and creates executor pod
 
 - The pod's container runs a pre-built Docker image containing a Spark distribution (with Kubernetes integration) and 
 invokes the Java runtime with the CoarseGrainedExecutorBackend main class.
-- The scheduler backend specifies environment variables on the executor pod to configure its runtime, p
-articularly for its JVM options, number of cores, heap size, and the driver's hostname.
+- The scheduler backend specifies environment variables on the executor pod to configure its runtime, particularly for its JVM options, number of cores, heap size, and the driver's hostname.
 - The executor container has [resource limits and requests](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) 
 that are set in accordance to the resource limits specified in the Spark configuration (executor.cores and executor.memory in the application's SparkConf)
 - The executor pods may also be launched into a particular [Kubernetes namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/%5C), 

resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStep.scala

@@ -48,6 +48,9 @@ private[spark] class BaseDriverConfigurationStep(
   // Memory settings
   private val driverMemoryMiB = submissionSparkConf.get(
       org.apache.spark.internal.config.DRIVER_MEMORY)
+  private val driverMemoryString = submissionSparkConf.get(
+      org.apache.spark.internal.config.DRIVER_MEMORY.key,
+      org.apache.spark.internal.config.DRIVER_MEMORY.defaultValueString)
   private val memoryOverheadMiB = submissionSparkConf
       .get(KUBERNETES_DRIVER_MEMORY_OVERHEAD)
       .getOrElse(math.max((MEMORY_OVERHEAD_FACTOR * driverMemoryMiB).toInt,
@@ -102,7 +105,7 @@ private[spark] class BaseDriverConfigurationStep(
       .addToEnv(driverExtraClasspathEnv.toSeq: _*)
       .addNewEnv()
         .withName(ENV_DRIVER_MEMORY)
-        .withValue(driverContainerMemoryWithOverheadMiB + "M") // JVM treats the "M" unit as "Mi"
+        .withValue(driverMemoryString)
         .endEnv()
       .addNewEnv()
         .withName(ENV_DRIVER_MAIN_CLASS)

resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStepSuite.scala

@@ -42,10 +42,10 @@ private[spark] class BaseDriverConfigurationStepSuite extends SparkFunSuite {
   test("Set all possible configurations from the user.") {
     val sparkConf = new SparkConf()
         .set(KUBERNETES_DRIVER_POD_NAME, "spark-driver-pod")
-        .set(org.apache.spark.internal.config.DRIVER_CLASS_PATH, "/opt/spark/spark-exmaples.jar")
+        .set(org.apache.spark.internal.config.DRIVER_CLASS_PATH, "/opt/spark/spark-examples.jar")
         .set("spark.driver.cores", "2")
         .set(KUBERNETES_DRIVER_LIMIT_CORES, "4")
-        .set(org.apache.spark.internal.config.DRIVER_MEMORY, 256L)
+        .set(org.apache.spark.internal.config.DRIVER_MEMORY.key, "256M")
         .set(KUBERNETES_DRIVER_MEMORY_OVERHEAD, 200L)
         .set(DRIVER_DOCKER_IMAGE, "spark-driver:latest")
         .set(s"spark.kubernetes.driver.annotation.$CUSTOM_ANNOTATION_KEY", CUSTOM_ANNOTATION_VALUE)
@@ -80,8 +80,8 @@ private[spark] class BaseDriverConfigurationStepSuite extends SparkFunSuite {
         .map(env => (env.getName, env.getValue))
         .toMap
     assert(envs.size === 6)
-    assert(envs(ENV_SUBMIT_EXTRA_CLASSPATH) === "/opt/spark/spark-exmaples.jar")
-    assert(envs(ENV_DRIVER_MEMORY) === "456M")
+    assert(envs(ENV_SUBMIT_EXTRA_CLASSPATH) === "/opt/spark/spark-examples.jar")
+    assert(envs(ENV_DRIVER_MEMORY) === "256M")
     assert(envs(ENV_DRIVER_MAIN_CLASS) === MAIN_CLASS)
     assert(envs(ENV_DRIVER_ARGS) === "arg1 arg2")
     assert(envs(DRIVER_CUSTOM_ENV_KEY1) === "customDriverEnv1")