Set ENV_DRIVER_MEMORY to memory instead of memory+overhead (apache-spark-on-k8s#475)

* Set ENV_DRIVER_MEMORY to memory instead of memory+overhead

Signed-off-by: duyanghao <[email protected]>

* Restore test

Author: duyanghao

resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStep.scala

@@ -48,6 +48,9 @@ private[spark] class BaseDriverConfigurationStep(
   // Memory settings
   private val driverMemoryMiB = submissionSparkConf.get(
       org.apache.spark.internal.config.DRIVER_MEMORY)
+  private val driverMemoryString = submissionSparkConf.get(
+      org.apache.spark.internal.config.DRIVER_MEMORY.key,
+      org.apache.spark.internal.config.DRIVER_MEMORY.defaultValueString)
   private val memoryOverheadMiB = submissionSparkConf
       .get(KUBERNETES_DRIVER_MEMORY_OVERHEAD)
       .getOrElse(math.max((MEMORY_OVERHEAD_FACTOR * driverMemoryMiB).toInt,
@@ -102,7 +105,7 @@ private[spark] class BaseDriverConfigurationStep(
       .addToEnv(driverExtraClasspathEnv.toSeq: _*)
       .addNewEnv()
         .withName(ENV_DRIVER_MEMORY)
-        .withValue(driverContainerMemoryWithOverheadMiB + "M") // JVM treats the "M" unit as "Mi"
+        .withValue(driverMemoryString)
         .endEnv()
       .addNewEnv()
         .withName(ENV_DRIVER_MAIN_CLASS)

resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStepSuite.scala

@@ -42,10 +42,10 @@ private[spark] class BaseDriverConfigurationStepSuite extends SparkFunSuite {
   test("Set all possible configurations from the user.") {
     val sparkConf = new SparkConf()
         .set(KUBERNETES_DRIVER_POD_NAME, "spark-driver-pod")
-        .set(org.apache.spark.internal.config.DRIVER_CLASS_PATH, "/opt/spark/spark-exmaples.jar")
+        .set(org.apache.spark.internal.config.DRIVER_CLASS_PATH, "/opt/spark/spark-examples.jar")
         .set("spark.driver.cores", "2")
         .set(KUBERNETES_DRIVER_LIMIT_CORES, "4")
-        .set(org.apache.spark.internal.config.DRIVER_MEMORY, 256L)
+        .set(org.apache.spark.internal.config.DRIVER_MEMORY.key, "256M")
         .set(KUBERNETES_DRIVER_MEMORY_OVERHEAD, 200L)
         .set(DRIVER_DOCKER_IMAGE, "spark-driver:latest")
         .set(s"spark.kubernetes.driver.annotation.$CUSTOM_ANNOTATION_KEY", CUSTOM_ANNOTATION_VALUE)
@@ -80,8 +80,8 @@ private[spark] class BaseDriverConfigurationStepSuite extends SparkFunSuite {
         .map(env => (env.getName, env.getValue))
         .toMap
     assert(envs.size === 6)
-    assert(envs(ENV_SUBMIT_EXTRA_CLASSPATH) === "/opt/spark/spark-exmaples.jar")
-    assert(envs(ENV_DRIVER_MEMORY) === "456M")
+    assert(envs(ENV_SUBMIT_EXTRA_CLASSPATH) === "/opt/spark/spark-examples.jar")
+    assert(envs(ENV_DRIVER_MEMORY) === "256M")
     assert(envs(ENV_DRIVER_MAIN_CLASS) === MAIN_CLASS)
     assert(envs(ENV_DRIVER_ARGS) === "arg1 arg2")
     assert(envs(DRIVER_CUSTOM_ENV_KEY1) === "customDriverEnv1")