handling flags, readmes, and POM changes

Author: ifilonenko

README.md

@@ -53,34 +53,17 @@ $ mvn clean integration-test  \
     -DextraScalaTestArgs="-Dspark.kubernetes.test.master=k8s://https://<master> -Dspark.docker.test.driverImage=<driver-image> -Dspark.docker.test.executorImage=<executor-image>"
 ```
 
-# Preserve the Minikube VM
-
-The integration tests make use of
-[Minikube](https://github.com/kubernetes/minikube), which fires up a virtual
-machine and setup a single-node kubernetes cluster within it. By default the vm
-is destroyed after the tests are finished.  If you want to preserve the vm, e.g.
-to reduce the running time of tests during development, you can pass the
-property `spark.docker.test.persistMinikube` to the test process:
-
-```
-$ mvn clean integration-test  \
-    -Dspark-distro-tgz=spark/spark-2.3.0-SNAPSHOT-bin.tgz  \
-    -DextraScalaTestArgs=-Dspark.docker.test.persistMinikube=true
-```
-
 # Reuse the previous Docker images
 
 The integration tests build a number of Docker images, which takes some time.
 By default, the images are built every time the tests run.  You may want to skip
 re-building those images during development, if the distribution package did not
 change since the last run. You can pass the property
-`spark.docker.test.skipBuildImages` to the test process. This will work only if
-you have been setting the property `spark.docker.test.persistMinikube`, in the
-previous run since the docker daemon run inside the minikube environment.  Here
-is an example:
+`spark.docker.test.skipBuildImages` to the test process.
+Here is an example:
 
 ```
 $ mvn clean integration-test  \
     -Dspark-distro-tgz=spark/spark-2.3.0-SNAPSHOT-bin.tgz  \
-    "-DextraScalaTestArgs=-Dspark.docker.test.persistMinikube=true -Dspark.docker.test.skipBuildImages=true"
+    "-Dspark.docker.test.skipBuildImages=true"
 ```

integration-test/pom.xml

@@ -40,7 +40,6 @@
     <slf4j-log4j12.version>1.7.24</slf4j-log4j12.version>
     <sbt.project.name>kubernetes-integration-tests</sbt.project.name>
     <spark-distro-tgz>YOUR-SPARK-DISTRO-TARBALL-HERE</spark-distro-tgz>
-    <spark-dockerfiles-dir>YOUR-DOCKERFILES-DIR-HERE</spark-dockerfiles-dir>
     <test.exclude.tags></test.exclude.tags>
   </properties>
   <packaging>jar</packaging>

integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala

@@ -21,38 +21,37 @@ import java.util.UUID
 import io.fabric8.kubernetes.client.DefaultKubernetesClient
 
 import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend
-import org.apache.spark.deploy.k8s.integrationtest.config._
 import org.apache.spark.deploy.k8s.integrationtest.docker.KubernetesSuiteDockerManager
 
 private[spark] object MinikubeTestBackend extends IntegrationTestBackend {
   private var defaultClient: DefaultKubernetesClient = _
-  private val userProvidedDockerImageTag = Option(
-    System.getProperty(KUBERNETES_TEST_DOCKER_TAG_SYSTEM_PROPERTY))
+  private val userSkipBuildImages =
+    System.getProperty("spark.docker.test.skipBuildImages", "false").toBoolean
   private val resolvedDockerImageTag =
-    userProvidedDockerImageTag.getOrElse(UUID.randomUUID().toString.replaceAll("-", ""))
+    UUID.randomUUID().toString.replaceAll("-", "")
   private val dockerManager = new KubernetesSuiteDockerManager(
     Minikube.getDockerEnv, resolvedDockerImageTag)
   override def initialize(): Unit = {
     val minikubeStatus = Minikube.getMinikubeStatus
     require(minikubeStatus == MinikubeStatus.RUNNING,
       s"Minikube must be running before integration tests can execute. Current status" +
         s" is: $minikubeStatus")
-    if (userProvidedDockerImageTag.isEmpty) {
+    if (!userSkipBuildImages) {
       dockerManager.buildSparkDockerImages()
     }
     defaultClient = Minikube.getKubernetesClient
   }
 
-
   override def cleanUp(): Unit = {
     super.cleanUp()
-    if (userProvidedDockerImageTag.isEmpty) {
+    if (!userSkipBuildImages) {
       dockerManager.deleteImages()
     }
   }
 
   override def getKubernetesClient(): DefaultKubernetesClient = {
     defaultClient
   }
+
   override def dockerImageTag(): String = resolvedDockerImageTag
 }

integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala

@@ -17,7 +17,6 @@
 package org.apache.spark.deploy.k8s.integrationtest
 
 package object config {
-  val KUBERNETES_TEST_DOCKER_TAG_SYSTEM_PROPERTY = "spark.kubernetes.test.imageDockerTag"
   val DRIVER_DOCKER_IMAGE = "spark.kubernetes.driver.docker.image"
   val EXECUTOR_DOCKER_IMAGE = "spark.kubernetes.executor.docker.image"
   val INIT_CONTAINER_DOCKER_IMAGE = "spark.kubernetes.initcontainer.docker.image"