Document the experimental config option

Author: kimoonkim

resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/config.scala

@@ -69,7 +69,17 @@ package object config extends Logging {
   private[spark] val CLIENT_CERT_FILE_CONF_SUFFIX = "clientCertFile"
   private[spark] val CA_CERT_FILE_CONF_SUFFIX = "caCertFile"
 
-  private[spark] val MOUNTED_HADOOP_SECRET_CONF = "spark.kubernetes.mounted.hadoopSecret"
+  // TODO: This option is intended to be used for internal prototype only until the submission
+  // client automatically creates the secret file. Remove this option afterward
+  // unless other use is found.
+  private[spark] val MOUNTED_HADOOP_SECRET_CONF =
+    ConfigBuilder("spark.kubernetes.mounted.hadoopSecret")
+        .doc("Use a Kubernetes secret containing Hadoop tokens such as an HDFS delegation token." +
+          " The secret should have an entry named 'hadoop-token-file' under the data section," +
+          " which contains binary dumps of Hadoop tokens.")
+        .internal()
+        .stringConf
+        .createOptional
 
   private[spark] val RESOURCE_STAGING_SERVER_USE_SERVICE_ACCOUNT_CREDENTIALS =
     ConfigBuilder(

resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/DriverHadoopCredentialsStep.scala

@@ -23,7 +23,7 @@ import org.apache.spark.deploy.kubernetes.submit.HadoopSecretUtil
 private[spark] class DriverHadoopCredentialsStep(submissionSparkConf: SparkConf)
   extends DriverConfigurationStep {
 
-  private val maybeMountedHadoopSecret = submissionSparkConf.getOption(MOUNTED_HADOOP_SECRET_CONF)
+  private val maybeMountedHadoopSecret = submissionSparkConf.get(MOUNTED_HADOOP_SECRET_CONF)
 
   override def configureDriver(driverSpec: KubernetesDriverSpec): KubernetesDriverSpec = {
     val podWithMountedHadoopToken = HadoopSecretUtil.configurePod(maybeMountedHadoopSecret,

resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/kubernetes/KubernetesClusterSchedulerBackend.scala

@@ -130,7 +130,7 @@ private[spark] class KubernetesClusterSchedulerBackend(
   private implicit val requestExecutorContext = ExecutionContext.fromExecutorService(
     ThreadUtils.newDaemonCachedThreadPool("kubernetes-executor-requests"))
 
-  private val maybeMountedHadoopSecret = conf.getOption(MOUNTED_HADOOP_SECRET_CONF)
+  private val maybeMountedHadoopSecret = conf.get(MOUNTED_HADOOP_SECRET_CONF)
 
   private val driverPod = try {
     kubernetesClient.pods().inNamespace(kubernetesNamespace).