cleanup and various small fixes

Author: ifilonenko

resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/KerberosTokenConfBootstrapSuite.scala

@@ -16,16 +16,13 @@
  */
 package org.apache.spark.deploy.kubernetes.submit
 
-import java.io.File
-import java.util.UUID
-
 import scala.collection.JavaConverters._
-import com.google.common.io.Files
+
 import io.fabric8.kubernetes.api.model._
+
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.deploy.kubernetes.{HadoopConfBootstrapImpl, KerberosTokenConfBootstrapImpl, PodWithMainContainer}
+import org.apache.spark.deploy.kubernetes.{KerberosTokenConfBootstrapImpl, PodWithMainContainer}
 import org.apache.spark.deploy.kubernetes.constants._
-import org.apache.spark.util.Utils
 
 
 private[spark] class KerberosTokenConfBootstrapSuite extends SparkFunSuite {

resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/kerberos-test/Dockerfile

@@ -24,5 +24,4 @@ FROM spark-base
 RUN apk add --no-cache --update krb5 krb5-libs
 COPY examples /opt/spark/examples
 COPY test-scripts/test-env.sh /opt/spark/
-COPY test-scripts/test-env2.sh /opt/spark/
 COPY hconf /opt/spark/hconf

resource-managers/kubernetes/integration-tests/kerberos-yml/namenode-hadoop-pv.yml

@@ -253,7 +253,6 @@
                   <directory>kerberos-yml</directory>
                   <includes>
                     <include>test-env.sh</include>
-                    <include>test-env2.sh</include>
                   </includes>
                 </resource>
               </resources>

resource-managers/kubernetes/integration-tests/kerberos-yml/namenode-hadoop.yml renamed to resource-managers/kubernetes/integration-tests/kerberos-yml/nn-hadoop.yml

@@ -128,39 +128,6 @@ private[spark] class KubernetesSuite extends SparkFunSuite with BeforeAndAfter {
       }
     }
   }
-//  test("Secure HDFS test with kinit") {
-//    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-//    launchKerberizedCluster()
-//    createKerberosTestPod(
-//      CONTAINER_LOCAL_MAIN_APP_RESOURCE,
-//      HDFS_TEST_CLASS,
-//      APP_LOCATOR_LABEL,
-//      "kerberos-yml/kerberos-test2.yml")
-//    val kubernetesClient = kubernetesTestComponents.kubernetesClient
-//    val driverWatcherCache = new KerberosDriverWatcherCache(
-//      kubernetesClient,
-//      Map("spark-app-locator" -> APP_LOCATOR_LABEL))
-//    driverWatcherCache.start()
-//    driverWatcherCache.stop()
-//    val expectedLogOnCompletion = Seq(
-//      "Returned length(s) of: 1",
-//      "File contents: [This is an awesome word count file]")
-//    val driverPod = kubernetesClient
-//      .pods()
-//      .withLabel("spark-app-locator", APP_LOCATOR_LABEL)
-//      .list()
-//      .getItems
-//      .get(0)
-//    Eventually.eventually(TIMEOUT, INTERVAL) {
-//      expectedLogOnCompletion.foreach { e =>
-//        assert(kubernetesClient
-//          .pods()
-//          .withName(driverPod.getMetadata.getName)
-//          .getLog
-//          .contains(e), "The application did not complete.")
-//      }
-//    }
-//  }
 
   test("Run PySpark Job on file from SUBMITTER with --py-files") {
     assume(testBackend.name == MINIKUBE_TEST_BACKEND)

resource-managers/kubernetes/integration-tests/kerberos-yml/server-keytab-pv.yml

@@ -87,16 +87,16 @@ private[spark] class SparkDockerImageBuilder
     if (exitCode != 0) {
       logInfo(s"exitCode: $exitCode")
     }
-    buildImage("spark-base", BASE_DOCKER_FILE)
-    buildImage("spark-driver", DRIVER_DOCKER_FILE)
-    buildImage("spark-driver-py", DRIVERPY_DOCKER_FILE)
-    buildImage("spark-executor", EXECUTOR_DOCKER_FILE)
-    buildImage("spark-executor-py", EXECUTORPY_DOCKER_FILE)
-    buildImage("spark-shuffle", SHUFFLE_SERVICE_DOCKER_FILE)
-    buildImage("spark-resource-staging-server", STAGING_SERVER_DOCKER_FILE)
-    buildImage("spark-init", INIT_CONTAINER_DOCKER_FILE)
-    buildImage("spark-integration-test-asset-server", STATIC_ASSET_SERVER_DOCKER_FILE)
-    buildImage("kerberos-test", KERBEROS_DOCKER_FILE)
+//    buildImage("spark-base", BASE_DOCKER_FILE)
+//    buildImage("spark-driver", DRIVER_DOCKER_FILE)
+//    buildImage("spark-driver-py", DRIVERPY_DOCKER_FILE)
+//    buildImage("spark-executor", EXECUTOR_DOCKER_FILE)
+//    buildImage("spark-executor-py", EXECUTORPY_DOCKER_FILE)
+//    buildImage("spark-shuffle", SHUFFLE_SERVICE_DOCKER_FILE)
+//    buildImage("spark-resource-staging-server", STAGING_SERVER_DOCKER_FILE)
+//    buildImage("spark-init", INIT_CONTAINER_DOCKER_FILE)
+//    buildImage("spark-integration-test-asset-server", STATIC_ASSET_SERVER_DOCKER_FILE)
+      buildImage("kerberos-test", KERBEROS_DOCKER_FILE)
   }
 
   private def buildImage(name: String, dockerFile: String): Unit = {

resource-managers/kubernetes/integration-tests/kerberos-yml/test-env2.sh

@@ -58,19 +58,35 @@ private[spark] class KerberosUtils(
       .withKey(file)
       .withPath(file)
       .build()).toList
-  private val pvNN = Seq("namenode-hadoop", "namenode-hadoop-pv")
-  private val pvKT = Seq("server-keytab", "server-keytab-pv")
-  private def buildKerberosPV(seqPair: Seq[String]) = {
+  private def createPVTemplate(name: String, pathType: String) : PersistentVolume =
+    new PersistentVolumeBuilder()
+      .withNewMetadata()
+        .withName(name)
+        .withLabels(Map(
+          "type" -> "local",
+          "job" -> "kerberostest").asJava)
+        .endMetadata()
+      .withNewSpec()
+        .withCapacity(Map("storage" -> new Quantity("1Gi")).asJava)
+        .withAccessModes("ReadWriteOnce")
+        .withHostPath(
+          new HostPathVolumeSource(s"/tmp/$namespace/$pathType"))
+        .endSpec()
+      .build()
+   private val pvNN = "nn-hadoop"
+   private val pvKT = "server-keytab"
+   private val persistentVolumeMap: Map[String, PersistentVolume] = Map(
+     pvNN -> createPVTemplate(pvNN, "nn"),
+     pvKT -> createPVTemplate(pvKT, "keytab"))
+  private def buildKerberosPV(pvType: String) = {
     KerberosStorage(
-      kubernetesClient.load(loadFromYaml(seqPair.head))
+      kubernetesClient.load(loadFromYaml(pvType))
         .get().get(0).asInstanceOf[PersistentVolumeClaim],
-      kubernetesClient.load(loadFromYaml(seqPair(1)))
-        .get().get(0).asInstanceOf[PersistentVolume])
+      persistentVolumeMap(pvType))
   }
   def getNNStorage: KerberosStorage = buildKerberosPV(pvNN)
   def getKTStorage: KerberosStorage = buildKerberosPV(pvKT)
   def getLabels: Map[String, String] = PV_LABELS
-  def getPVNN: Seq[String] = pvNN
   def getKeyPaths: Seq[KeyToPath] = keyPaths
   def getConfigMap: ConfigMap = new ConfigMapBuilder()
     .withNewMetadata()