Remove using nginx file server

Author: mccheah

README.md

@@ -11,7 +11,7 @@ is subject to change. Note that currently the integration tests only run with Ja
 The simplest way to run the integration tests is to install and run Minikube, then run the following:
 
     build/mvn integration-test
-    
+
 The minimum tested version of Minikube is 0.23.0. The kube-dns addon must be enabled. Minikube should
 run with a minimum of 3 CPUs and 4G of memory:
 
@@ -39,7 +39,7 @@ Therefore the command looks like this:
       -Dspark.kubernetes.test.deployMode=cloud \
       -Dspark.kubernetes.test.master=https://example.com:8443/apiserver \
       -Dspark.kubernetes.test.repo=docker.example.com/spark-images
-      
+
 ## Re-using Docker Images
 
 By default, the test framework will build new Docker images on every test execution. A unique image tag is generated,
@@ -75,4 +75,4 @@ source code has to be compiled.
 
 When the tests are cloning a repository and building it, the Spark distribution is placed in
 `target/spark/spark-<VERSION>.tgz`. Reuse this tarball to save a significant amount of time if you are iterating on
-the development of these integration tests.
+the development of these integration tests.

docker-file-server/.gitignore

@@ -33,27 +33,18 @@ then
   echo "No unpacked distribution was found at $UNPACKED_SPARK_TGZ. Please run clone-spark.sh and build-spark.sh first." && exit 1;
 fi
 
-FILE_SERVER_IMAGE="$IMAGE_REPO/spark-examples-file-server:$IMAGE_TAG"
-FILE_SERVER_BUILD_DIR="$TEST_ROOT_DIR/docker-file-server"
-rm -rf $FILE_SERVER_BUILD_DIR/jars
-mkdir -p $FILE_SERVER_BUILD_DIR/jars
-cp $UNPACKED_SPARK_TGZ/examples/jars/spark-examples*.jar $FILE_SERVER_BUILD_DIR/jars/.
 cd $UNPACKED_SPARK_TGZ
 if  [[ $DEPLOY_MODE == cloud ]] ;
 then
-  docker build -t $FILE_SERVER_IMAGE "$FILE_SERVER_BUILD_DIR"
   $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG build
   if  [[ $IMAGE_REPO == gcr.io* ]] ;
   then
     gcloud docker -- push $IMAGE_REPO/spark:$IMAGE_TAG && \
-    gcloud docker -- push $FILE_SERVER_IMAGE
   else
     $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG push
-    docker push $FILE_SERVER_IMAGE
   fi
 else
   # -m option for minikube.
   eval $(minikube docker-env)
-  docker build -t $FILE_SERVER_IMAGE $FILE_SERVER_BUILD_DIR
   $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t $IMAGE_TAG build
 fi

docker-file-server/Dockerfile

@@ -41,7 +41,6 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
   private var sparkHomeDir: Path = _
   private var kubernetesTestComponents: KubernetesTestComponents = _
   private var sparkAppConf: SparkAppConf = _
-  private var remoteExamplesJarUri: URI = _
   private var image: String = _
   private var containerLocalSparkDistroExamplesJar: String = _
 
@@ -74,11 +73,6 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
       .set("spark.kubernetes.driver.label.spark-app-locator", APP_LOCATOR_LABEL)
       .set("spark.kubernetes.executor.label.spark-app-locator", APP_LOCATOR_LABEL)
     kubernetesTestComponents.createNamespace()
-    remoteExamplesJarUri = SparkExamplesFileServerRunner
-      .launchServerAndGetUriForExamplesJar(
-        kubernetesTestComponents,
-        getTestImageTag,
-        getTestImageRepo)
   }
 
   after {
@@ -109,10 +103,6 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
     runSparkPiAndVerifyCompletion(appArgs = Array("5"))
   }
 
-  test("Run SparkPi using the remote example jar.") {
-    runSparkPiAndVerifyCompletion(appResource = remoteExamplesJarUri.toString)
-  }
-
   test("Run SparkPi with custom driver pod name, labels, annotations, and environment variables.") {
     sparkAppConf
       .set("spark.kubernetes.driver.pod.name", "spark-integration-spark-pi")
@@ -177,8 +167,8 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
 
     createTestSecret()
 
-    runSparkPiAndVerifyCompletion(
-      appResource = remoteExamplesJarUri.toString,
+    runSparkPageRankAndVerifyCompletion(
+      appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE),
       driverPodChecker = (driverPod: Pod) => {
         doBasicDriverPodCheck(driverPod)
         checkTestSecret(driverPod, withInitContainer = true)

docker-file-server/nginx.conf

@@ -18,9 +18,6 @@ package org.apache.spark.deploy.k8s.integrationtest
 
 import java.io.Closeable
 import java.net.URI
-import java.io.{IOException, InputStream, OutputStream}
-
-import com.google.common.io.ByteStreams
 
 object Utils extends Logging {