diff --git a/.gitignore b/.gitignore index dc020f2..b0e7abb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,11 @@ .idea/ -spark/ -integration-test/target/ +target/ +build/*.jar +build/apache-maven* +build/scala* +build/zinc* +build/run-mvn *.class *.log *.iml +*.swp diff --git a/README.md b/README.md index 8221265..54d2a94 100644 --- a/README.md +++ b/README.md @@ -8,98 +8,67 @@ title: Spark on Kubernetes Integration Tests Note that the integration test framework is currently being heavily revised and is subject to change. Note that currently the integration tests only run with Java 8. -As shorthand to run the tests against any given cluster, you can use the `e2e/runner.sh` script. -The script assumes that you have a functioning Kubernetes cluster (1.6+) with kubectl -configured to access it. The master URL of the currently configured cluster on your -machine can be discovered as follows: - -``` -$ kubectl cluster-info - -Kubernetes master is running at https://xyz -``` - -If you want to use a local [minikube](https://github.com/kubernetes/minikube) cluster, -the minimum tested version is 0.23.0, with the kube-dns addon enabled -and the recommended configuration is 3 CPUs and 4G of memory. There is also a wrapper -script for running on minikube, `e2e/e2e-minikube.sh` for testing the master branch -of the apache/spark repository in specific. - -``` -$ minikube start --memory 4000 --cpus 3 -``` - -If you're using a non-local cluster, you must provide an image repository -which you have write access to, using the `-i` option, in order to store docker images -generated during the test. - -Example usages of the script: - -``` -$ ./e2e/runner.sh -m https://xyz -i docker.io/foxish -d cloud -$ ./e2e/runner.sh -m https://xyz -i test -d minikube -$ ./e2e/runner.sh -m https://xyz -i test -r https://github.com/my-spark/spark -d minikube -$ ./e2e/runner.sh -m https://xyz -i test -r https://github.com/my-spark/spark -b my-branch -d minikube -``` - -# Detailed Documentation - -## Running the tests using maven - -Integration tests firstly require installing [Minikube](https://kubernetes.io/docs/getting-started-guides/minikube/) on -your machine, and for the `Minikube` binary to be on your `PATH`.. Refer to the Minikube documentation for instructions -on how to install it. It is recommended to allocate at least 8 CPUs and 8GB of memory to the Minikube cluster. - -Running the integration tests requires a Spark distribution package tarball that -contains Spark jars, submission clients, etc. You can download a tarball from -http://spark.apache.org/downloads.html. Or, you can create a distribution from -source code using `make-distribution.sh`. For example: - -``` -$ git clone git@github.com:apache/spark.git -$ cd spark -$ ./dev/make-distribution.sh --tgz \ - -Phadoop-2.7 -Pkubernetes -Pkinesis-asl -Phive -Phive-thriftserver -``` - -The above command will create a tarball like spark-2.3.0-SNAPSHOT-bin.tgz in the -top-level dir. For more details, see the related section in -[building-spark.md](https://github.com/apache/spark/blob/master/docs/building-spark.md#building-a-runnable-distribution) - - -Once you prepare the tarball, the integration tests can be executed with Maven or -your IDE. Note that when running tests from an IDE, the `pre-integration-test` -phase must be run every time the Spark main code changes. When running tests -from the command line, the `pre-integration-test` phase should automatically be -invoked if the `integration-test` phase is run. - -With Maven, the integration test can be run using the following command: - -``` -$ mvn clean integration-test \ - -Dspark-distro-tgz=spark/spark-2.3.0-SNAPSHOT-bin.tgz -``` - -## Running against an arbitrary cluster - -In order to run against any cluster, use the following: -```sh -$ mvn clean integration-test \ - -Dspark-distro-tgz=spark/spark-2.3.0-SNAPSHOT-bin.tgz \ - -DextraScalaTestArgs="-Dspark.kubernetes.test.master=k8s://https:// - -## Reuse the previous Docker images - -The integration tests build a number of Docker images, which takes some time. -By default, the images are built every time the tests run. You may want to skip -re-building those images during development, if the distribution package did not -change since the last run. You can pass the property -`spark.kubernetes.test.imageDockerTag` to the test process and specify the Docker -image tag that is appropriate. -Here is an example: - -``` -$ mvn clean integration-test \ - -Dspark-distro-tgz=spark/spark-2.3.0-SNAPSHOT-bin.tgz \ - -Dspark.kubernetes.test.imageDockerTag=latest -``` +The simplest way to run the integration tests is to install and run Minikube, then run the following: + + dev/dev-run-integration-tests.sh + +The minimum tested version of Minikube is 0.23.0. The kube-dns addon must be enabled. Minikube should +run with a minimum of 3 CPUs and 4G of memory: + + minikube start --cpus 3 --memory 4096 + +You can download Minikube [here](https://github.com/kubernetes/minikube/releases). + +# Integration test customization + +Configuration of the integration test runtime is done through passing different arguments to the test script. The main useful options are outlined below. + +## Use a non-local cluster + +To use your own cluster running in the cloud, set the following: + +* `--deploy-mode cloud` to indicate that the test is connecting to a remote cluster instead of Minikube, +* `--spark-master ` - set `` to the externally accessible Kubernetes cluster URL, +* `--image-repo ` - set `` to a write-accessible Docker image repository that provides the images for your cluster. The framework assumes your local Docker client can push to this repository. + +Therefore the command looks like this: + + dev/dev-run-integration-tests.sh \ + --deploy-mode cloud \ + --spark-master https://example.com:8443/apiserver \ + --image-repo docker.example.com/spark-images + +## Re-using Docker Images + +By default, the test framework will build new Docker images on every test execution. A unique image tag is generated, +and it is written to file at `target/imageTag.txt`. To reuse the images built in a previous run, or to use a Docker image tag +that you have built by other means already, pass the tag to the test script: + + dev/dev-run-integration-tests.sh --image-tag + +where if you still want to use images that were built before by the test framework: + + dev/dev-run-integration-tests.sh --image-tag $(cat target/imageTag.txt) + +## Customizing the Spark Source Code to Test + +By default, the test framework will test the master branch of Spark from [here](https://github.com/apache/spark). You +can specify the following options to test against different source versions of Spark: + +* `--spark-repo ` - set `` to the git or http URI of the Spark git repository to clone +* `--spark-branch ` - set `` to the branch of the repository to build. + + +An example: + + dev/dev-run-integration-tests.sh \ + --spark-repo https://github.com/apache-spark-on-k8s/spark \ + --spark-branch new-feature + +Additionally, you can use a pre-built Spark distribution. In this case, the repository is not cloned at all, and no +source code has to be compiled. + +* `--spark-tgz ` - set `` to point to a tarball containing the Spark distribution to test. + +When the tests are cloning a repository and building it, the Spark distribution is placed in `target/spark/spark-.tgz`. +Reuse this tarball to save a significant amount of time if you are iterating on the development of these integration tests. diff --git a/e2e/e2e-minikube.sh b/build/mvn similarity index 54% rename from e2e/e2e-minikube.sh rename to build/mvn index 3d2aef2..87e5b58 100755 --- a/e2e/e2e-minikube.sh +++ b/build/mvn @@ -1,5 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash +# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,23 +15,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# -### This script can be used to run integration tests locally on minikube. -### Requirements: minikube v0.23+ with the DNS addon enabled, and kubectl configured to point to it. +BUILD_DIR=$(dirname $0) -set -ex +MVN_RUNNER=$BUILD_DIR/run-mvn -### Basic Validation ### -if [ ! -d "integration-test" ]; then - echo "This script must be invoked from the top-level directory of the integration-tests repository" - usage - exit 1 +if [ ! -f $MVN_RUNNER ]; +then + curl -s --progress-bar https://raw.githubusercontent.com/apache/spark/master/build/mvn > $MVN_RUNNER + chmod +x $MVN_RUNNER fi - -# Set up config. -master=$(kubectl cluster-info | head -n 1 | grep -oE "https?://[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?") -repo="https://github.com/apache/spark" -image_repo=test - -# Run tests in minikube mode. -./e2e/runner.sh -m $master -r $repo -i $image_repo -d minikube +source $MVN_RUNNER diff --git a/dev/dev-run-integration-tests.sh b/dev/dev-run-integration-tests.sh new file mode 100755 index 0000000..2eb8327 --- /dev/null +++ b/dev/dev-run-integration-tests.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +TEST_ROOT_DIR=$(git rev-parse --show-toplevel) +BRANCH="master" +SPARK_REPO="https://github.com/apache/spark" +SPARK_REPO_LOCAL_DIR="$TEST_ROOT_DIR/target/spark" +DEPLOY_MODE="minikube" +IMAGE_REPO="docker.io/kubespark" +SPARK_TGZ="N/A" +IMAGE_TAG="N/A" +SPARK_MASTER= + +# Parse arguments +while (( "$#" )); do + case $1 in + --spark-branch) + BRANCH="$2" + shift + ;; + --spark-repo) + SPARK_REPO="$2" + shift + ;; + --image-repo) + IMAGE_REPO="$2" + shift + ;; + --image-tag) + IMAGE_TAG="$2" + shift + ;; + --deploy-mode) + DEPLOY_MODE="$2" + shift + ;; + --spark-tgz) + SPARK_TGZ="$2" + shift + ;; + *) + break + ;; + esac + shift +done + +if [[ $SPARK_TGZ == "N/A" ]]; +then + echo "Cloning $SPARK_REPO into $SPARK_REPO_LOCAL_DIR and checking out $BRANCH." + + # clone spark distribution if needed. + if [ -d "$SPARK_REPO_LOCAL_DIR" ]; + then + (cd $SPARK_REPO_LOCAL_DIR && git fetch origin $branch); + else + mkdir -p $SPARK_REPO_LOCAL_DIR; + git clone -b $BRANCH --single-branch $SPARK_REPO $SPARK_REPO_LOCAL_DIR; + fi + cd $SPARK_REPO_LOCAL_DIR + git checkout -B $BRANCH origin/$branch + ./dev/make-distribution.sh --tgz -Phadoop-2.7 -Pkubernetes -DskipTests; + SPARK_TGZ=$(find $SPARK_REPO_LOCAL_DIR -name spark-*.tgz) + echo "Built Spark TGZ at $SPARK_TGZ". + cd - +fi + +cd $TEST_ROOT_DIR + +if [ -z $SPARK_MASTER ]; +then + build/mvn integration-test \ + -Dspark.kubernetes.test.sparkTgz=$SPARK_TGZ \ + -Dspark.kubernetes.test.imageTag=$IMAGE_TAG \ + -Dspark.kubernetes.test.imageRepo=$IMAGE_REPO \ + -Dspark.kubernetes.test.deployMode=$DEPLOY_MODE; +else + build/mvn integration-test \ + -Dspark.kubernetes.test.sparkTgz=$SPARK_TGZ \ + -Dspark.kubernetes.test.imageTag=$IMAGE_TAG \ + -Dspark.kubernetes.test.imageRepo=$IMAGE_REPO \ + -Dspark.kubernetes.test.deployMode=$DEPLOY_MODE \ + -Dspark.kubernetes.test.master=$SPARK_MASTER; +fi diff --git a/e2e/e2e-prow.sh b/e2e/e2e-prow.sh deleted file mode 100755 index 45a8c2b..0000000 --- a/e2e/e2e-prow.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -### This script is used by Kubernetes Test Infrastructure to run integration tests. -### See documenation at https://github.com/kubernetes/test-infra/tree/master/prow -### To run the integration tests yourself, use e2e/runner.sh. - -set -ex - -# Install basic dependencies -echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list -apt-get update && apt-get install -y curl wget git tar -apt-get install -t jessie-backports -y openjdk-8-jdk - -# Set up config. -master=$(kubectl cluster-info | head -n 1 | grep -oE "https?://[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?") -repo="https://github.com/apache/spark" - -# Special GCP project for publishing docker images built by test. -image_repo="gcr.io/spark-testing-191023" -cd "$(dirname "$0")"/../ -./e2e/runner.sh -m $master -r $repo -i $image_repo -d cloud - -# Copy out the junit xml files for consumption by k8s test-infra. -ls -1 ./integration-test/target/surefire-reports/*.xml | cat -n | while read n f; do cp "$f" "/workspace/_artifacts/junit_0$n.xml"; done diff --git a/e2e/runner.sh b/e2e/runner.sh deleted file mode 100755 index aded856..0000000 --- a/e2e/runner.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -usage () { - echo "Usage:" - echo " ./e2e/runner.sh -h Display this help message." - echo " ./e2e/runner.sh -m -r -b -i -d [minikube|cloud]" - echo " note that you must have kubectl configured to access the specified" - echo " . Also you must have access to the . " - echo " The deployment mode can be specified using the 'd' flag." -} - -cd "$(dirname "$0")" - -### Set sensible defaults ### -REPO="https://github.com/apache/spark" -IMAGE_REPO="docker.io/kubespark" -DEPLOY_MODE="minikube" -BRANCH="master" - -### Parse options ### -while getopts h:m:r:i:d:b: option -do - case "${option}" - in - h) - usage - exit 0 - ;; - m) MASTER=${OPTARG};; - r) REPO=${OPTARG};; - b) BRANCH=${OPTARG};; - i) IMAGE_REPO=${OPTARG};; - d) DEPLOY_MODE=${OPTARG};; - \? ) - echo "Invalid Option: -$OPTARG" 1>&2 - exit 1 - ;; - esac -done - -### Ensure cluster is set. -if [ -z "$MASTER" ] -then - echo "Missing master-url (-m) argument." - echo "" - usage - exit -fi - -### Ensure deployment mode is minikube/cloud. -if [[ $DEPLOY_MODE != minikube && $DEPLOY_MODE != cloud ]]; -then - echo "Invalid deployment mode $DEPLOY_MODE" - usage - exit 1 -fi - -echo "Running tests on cluster $MASTER against $REPO." -echo "Spark images will be created in $IMAGE_REPO" - -set -ex -TEST_ROOT=$(git rev-parse --show-toplevel) -SPARK_REPO_ROOT="$TEST_ROOT/spark" -# clone spark distribution if needed. -if [ -d "$SPARK_REPO_ROOT" ]; -then - (cd $SPARK_REPO_ROOT && git pull origin $BRANCH); -else - git clone $REPO $SPARK_REPO_ROOT -fi - -cd $SPARK_REPO_ROOT -git checkout -B $BRANCH origin/$BRANCH -./dev/make-distribution.sh --tgz -Phadoop-2.7 -Pkubernetes -DskipTests -TAG=$(git rev-parse HEAD | cut -c -6) -echo "Spark distribution built at SHA $TAG" - -cd $SPARK_REPO_ROOT/dist - -if [[ $DEPLOY_MODE == cloud ]] ; -then - ./sbin/build-push-docker-images.sh -r $IMAGE_REPO -t $TAG build - if [[ $IMAGE_REPO == gcr.io* ]] ; - then - gcloud docker -- push $IMAGE_REPO/spark-driver:$TAG && \ - gcloud docker -- push $IMAGE_REPO/spark-executor:$TAG && \ - gcloud docker -- push $IMAGE_REPO/spark-init:$TAG - else - ./sbin/build-push-docker-images.sh -r $IMAGE_REPO -t $TAG push - fi -else - # -m option for minikube. - ./sbin/build-push-docker-images.sh -m -r $IMAGE_REPO -t $TAG build -fi - -cd $TEST_ROOT/integration-test -$SPARK_REPO_ROOT/build/mvn clean -Ddownload.plugin.skip=true integration-test \ - -Dspark-distro-tgz=$SPARK_REPO_ROOT/*.tgz \ - -DextraScalaTestArgs="-Dspark.kubernetes.test.master=k8s://$MASTER \ - -Dspark.docker.test.driverImage=$IMAGE_REPO/spark-driver:$TAG \ - -Dspark.docker.test.executorImage=$IMAGE_REPO/spark-executor:$TAG \ - -Dspark.docker.test.initContainerImage=$IMAGE_REPO/spark-init:$TAG" || : - -echo "TEST SUITE FINISHED" diff --git a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/docker/KubernetesSuiteDockerManager.scala b/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/docker/KubernetesSuiteDockerManager.scala deleted file mode 100644 index 0163d33..0000000 --- a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/docker/KubernetesSuiteDockerManager.scala +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.deploy.k8s.integrationtest.docker - -import java.io.{File, PrintWriter} -import java.net.URI -import java.nio.file.Paths -import java.util.UUID - -import com.google.common.base.Charsets -import com.google.common.io.Files -import com.spotify.docker.client.{DefaultDockerClient, DockerCertificates, LoggingBuildHandler} -import com.spotify.docker.client.DockerClient.{ListContainersParam, ListImagesParam, RemoveContainerParam} -import com.spotify.docker.client.messages.Container -import org.apache.http.client.utils.URIBuilder -import org.scalatest.concurrent.{Eventually, PatienceConfiguration} -import org.scalatest.time.{Minutes, Seconds, Span} -import scala.collection.JavaConverters._ - -import org.apache.spark.deploy.k8s.integrationtest.constants._ -import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite -import org.apache.spark.deploy.k8s.integrationtest.Logging -import org.apache.spark.deploy.k8s.integrationtest.Utils.tryWithResource - -private[spark] class KubernetesSuiteDockerManager( - dockerEnv: Map[String, String], userProvidedDockerImageTag: Option[String]) extends Logging { - - private val DOCKER_BUILD_PATH = SPARK_DISTRO_PATH - // Dockerfile paths must be relative to the build path. - private val DOCKERFILES_DIR = "kubernetes/dockerfiles/" - private val BASE_DOCKER_FILE = DOCKERFILES_DIR + "spark-base/Dockerfile" - private val DRIVER_DOCKER_FILE = DOCKERFILES_DIR + "driver/Dockerfile" - private val EXECUTOR_DOCKER_FILE = DOCKERFILES_DIR + "executor/Dockerfile" - private val INIT_CONTAINER_DOCKER_FILE = DOCKERFILES_DIR + "init-container/Dockerfile" - private val TIMEOUT = PatienceConfiguration.Timeout(Span(2, Minutes)) - private val INTERVAL = PatienceConfiguration.Interval(Span(2, Seconds)) - - private val resolvedDockerImageTag = - userProvidedDockerImageTag.getOrElse(UUID.randomUUID().toString.replaceAll("-", "")) - private val dockerHost = dockerEnv.getOrElse("DOCKER_HOST", - throw new IllegalStateException("DOCKER_HOST env not found.")) - private val originalDockerUri = URI.create(dockerHost) - private val httpsDockerUri = new URIBuilder() - .setHost(originalDockerUri.getHost) - .setPort(originalDockerUri.getPort) - .setScheme("https") - .build() - - private val dockerCerts = dockerEnv.getOrElse("DOCKER_CERT_PATH", - throw new IllegalStateException("DOCKER_CERT_PATH env not found.")) - - private val dockerClient = new DefaultDockerClient.Builder() - .uri(httpsDockerUri) - .dockerCertificates(DockerCertificates - .builder() - .dockerCertPath(Paths.get(dockerCerts)) - .build().get()) - .build() - - def buildSparkDockerImages(): Unit = { - if (userProvidedDockerImageTag.isEmpty) { - Eventually.eventually(TIMEOUT, INTERVAL) { - dockerClient.ping() - } - buildImage("spark-base", BASE_DOCKER_FILE) - buildImage("spark-driver", DRIVER_DOCKER_FILE) - buildImage("spark-executor", EXECUTOR_DOCKER_FILE) - buildImage("spark-init", INIT_CONTAINER_DOCKER_FILE) - } - } - - def deleteImages(): Unit = { - if (userProvidedDockerImageTag.isEmpty) { - removeRunningContainers() - deleteImage("spark-base") - deleteImage("spark-driver") - deleteImage("spark-executor") - deleteImage("spark-init") - } - } - - def dockerImageTag(): String = resolvedDockerImageTag - - private def buildImage(name: String, dockerFile: String): Unit = { - logInfo(s"Building Docker image - $name:$resolvedDockerImageTag") - val dockerFileWithBaseTag = new File(DOCKER_BUILD_PATH.resolve( - s"$dockerFile-$resolvedDockerImageTag").toAbsolutePath.toString) - dockerFileWithBaseTag.deleteOnExit() - try { - val originalDockerFileText = Files.readLines( - DOCKER_BUILD_PATH.resolve(dockerFile).toFile, Charsets.UTF_8).asScala - val dockerFileTextWithProperBaseImage = originalDockerFileText.map( - _.replace("FROM spark-base", s"FROM spark-base:$resolvedDockerImageTag")) - tryWithResource(Files.newWriter(dockerFileWithBaseTag, Charsets.UTF_8)) { fileWriter => - tryWithResource(new PrintWriter(fileWriter)) { printWriter => - for (line <- dockerFileTextWithProperBaseImage) { - // scalastyle:off println - printWriter.println(line) - // scalastyle:on println - } - } - } - dockerClient.build( - DOCKER_BUILD_PATH, - s"$name:$resolvedDockerImageTag", - s"$dockerFile-$resolvedDockerImageTag", - new LoggingBuildHandler()) - } finally { - dockerFileWithBaseTag.delete() - } - } - - /** - * Forces all containers running an image with the configured tag to halt and be removed. - */ - private def removeRunningContainers(): Unit = { - val imageIds = dockerClient.listImages(ListImagesParam.allImages()) - .asScala - .filter(image => image.repoTags().asScala.exists(_.endsWith(s":$resolvedDockerImageTag"))) - .map(_.id()) - .toSet - Eventually.eventually(KubernetesSuite.TIMEOUT, KubernetesSuite.INTERVAL) { - val runningContainersWithImageTag = stopRunningContainers(imageIds) - require( - runningContainersWithImageTag.isEmpty, - s"${runningContainersWithImageTag.size} containers found still running" + - s" with the image tag $resolvedDockerImageTag") - } - dockerClient.listContainers(ListContainersParam.allContainers()) - .asScala - .filter(container => imageIds.contains(container.imageId())) - .foreach(container => dockerClient.removeContainer( - container.id(), RemoveContainerParam.forceKill(true))) - Eventually.eventually(KubernetesSuite.TIMEOUT, KubernetesSuite.INTERVAL) { - val containersWithImageTag = dockerClient.listContainers(ListContainersParam.allContainers()) - .asScala - .filter(container => imageIds.contains(container.imageId())) - require(containersWithImageTag.isEmpty, s"${containersWithImageTag.size} containers still" + - s" found with image tag $resolvedDockerImageTag.") - } - - } - - private def stopRunningContainers(imageIds: Set[String]): Iterable[Container] = { - val runningContainersWithImageTag = getRunningContainersWithImageIds(imageIds) - if (runningContainersWithImageTag.nonEmpty) { - logInfo(s"Found ${runningContainersWithImageTag.size} containers running with" + - s" an image with the tag $resolvedDockerImageTag. Attempting to remove these containers," + - s" and then will stall for 2 seconds.") - runningContainersWithImageTag.foreach { container => - dockerClient.stopContainer(container.id(), 5) - } - } - runningContainersWithImageTag - } - - private def getRunningContainersWithImageIds(imageIds: Set[String]): Iterable[Container] = { - dockerClient - .listContainers( - ListContainersParam.allContainers(), - ListContainersParam.withStatusRunning()) - .asScala - .filter(container => imageIds.contains(container.imageId())) - } - - private def deleteImage(name: String): Unit = { - try { - dockerClient.removeImage(s"$name:$resolvedDockerImageTag") - } catch { - case e: RuntimeException => - logWarning(s"Failed to delete image $name:$resolvedDockerImageTag. There may be images leaking in the" + - s" docker environment which are now stale and unused.", e) - } - } -} diff --git a/integration-test/pom.xml b/pom.xml similarity index 79% rename from integration-test/pom.xml rename to pom.xml index 9375d91..cdc987a 100644 --- a/integration-test/pom.xml +++ b/pom.xml @@ -22,6 +22,7 @@ spark-kubernetes-integration-tests 0.1-SNAPSHOT + 3.3.9 3.5 1.1.1 5.0.2 @@ -39,7 +40,11 @@ 1.0 1.7.24 kubernetes-integration-tests - YOUR-SPARK-DISTRO-TARBALL-HERE + ${project.build.directory}/spark-dist-unpacked + N/A + ${project.build.directory}/imageTag.txt + minikube + docker.io/kubespark jar @@ -124,17 +129,31 @@ ${exec-maven-plugin.version} - unpack-spark-distro + setup-integration-test-env pre-integration-test exec - ${project.build.directory} - /bin/sh + scripts/setup-integration-test-env.sh - -c - rm -rf spark-distro; mkdir spark-distro-tmp; cd spark-distro-tmp; tar xfz ${spark-distro-tgz}; mv * ../spark-distro; cd ..; rm -rf spark-distro-tmp + --unpacked-spark-tgz + ${spark.kubernetes.test.unpackSparkDir} + + --image-repo + ${spark.kubernetes.test.imageRepo} + + --image-tag + ${spark.kubernetes.test.imageTag} + + --image-tag-output-file + ${spark.kubernetes.test.imageTagFile} + + --deploy-mode + ${spark.kubernetes.test.deployMode} + + --spark-tgz + ${spark.kubernetes.test.sparkTgz} @@ -155,6 +174,9 @@ file:src/test/resources/log4j.properties true + ${spark.kubernetes.test.imageTagFile} + ${spark.kubernetes.test.unpackSparkDir} + ${spark.kubernetes.test.imageRepo} ${test.exclude.tags} diff --git a/scripts/setup-integration-test-env.sh b/scripts/setup-integration-test-env.sh new file mode 100755 index 0000000..ccfb8e7 --- /dev/null +++ b/scripts/setup-integration-test-env.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +TEST_ROOT_DIR=$(git rev-parse --show-toplevel) +UNPACKED_SPARK_TGZ="$TEST_ROOT_DIR/target/spark-dist-unpacked" +IMAGE_TAG_OUTPUT_FILE="$TEST_ROOT_DIR/target/image-tag.txt" +DEPLOY_MODE="minikube" +IMAGE_REPO="docker.io/kubespark" +IMAGE_TAG="N/A" +SPARK_TGZ="N/A" + +# Parse arguments +while (( "$#" )); do + case $1 in + --unpacked-spark-tgz) + UNPACKED_SPARK_TGZ="$2" + shift + ;; + --image-repo) + IMAGE_REPO="$2" + shift + ;; + --image-tag) + IMAGE_TAG="$2" + shift + ;; + --image-tag-output-file) + IMAGE_TAG_OUTPUT_FILE="$2" + shift + ;; + --deploy-mode) + DEPLOY_MODE="$2" + shift + ;; + --spark-tgz) + SPARK_TGZ="$2" + shift + ;; + *) + break + ;; + esac + shift +done + +if [[ $SPARK_TGZ == "N/A" ]]; +then + echo "Must specify a Spark tarball to build Docker images against with --spark-tgz." && exit 1; +fi + +rm -rf $UNPACKED_SPARK_TGZ +mkdir -p $UNPACKED_SPARK_TGZ +tar -xzvf $SPARK_TGZ --strip-components=1 -C $UNPACKED_SPARK_TGZ; + +if [[ $IMAGE_TAG == "N/A" ]]; +then + IMAGE_TAG=$(uuidgen); + cd $UNPACKED_SPARK_TGZ + if [[ $DEPLOY_MODE == cloud ]] ; + then + $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG build + if [[ $IMAGE_REPO == gcr.io* ]] ; + then + gcloud docker -- push $IMAGE_REPO/spark:$IMAGE_TAG + else + $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG push + fi + else + # -m option for minikube. + $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t $IMAGE_TAG build + fi + cd - +fi + +rm -f $IMAGE_TAG_OUTPUT_FILE +echo -n $IMAGE_TAG > $IMAGE_TAG_OUTPUT_FILE diff --git a/integration-test/src/test/resources/log4j.properties b/src/test/resources/log4j.properties similarity index 100% rename from integration-test/src/test/resources/log4j.properties rename to src/test/resources/log4j.properties diff --git a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala similarity index 87% rename from integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala rename to src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 6e4f10d..7802b97 100644 --- a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -17,12 +17,12 @@ package org.apache.spark.deploy.k8s.integrationtest import java.io.File -import java.nio.file.Paths +import java.net.URI +import java.nio.file.{Path, Paths} import java.util.UUID import java.util.regex.Pattern import scala.collection.JavaConverters._ - import com.google.common.io.PatternFilenameFilter import io.fabric8.kubernetes.api.model.{Container, Pod} import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, FunSuite} @@ -31,7 +31,6 @@ import org.scalatest.time.{Minutes, Seconds, Span} import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackendFactory import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.MinikubeTestBackend -import org.apache.spark.deploy.k8s.integrationtest.constants._ import org.apache.spark.deploy.k8s.integrationtest.config._ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll with BeforeAndAfter { @@ -39,21 +38,27 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit import KubernetesSuite._ private val testBackend = IntegrationTestBackendFactory.getTestBackend() private val APP_LOCATOR_LABEL = UUID.randomUUID().toString.replaceAll("-", "") + private var sparkHomeDir: Path = _ private var kubernetesTestComponents: KubernetesTestComponents = _ private var sparkAppConf: SparkAppConf = _ - - private val driverImage = System.getProperty( - "spark.docker.test.driverImage", - "spark-driver:latest") - private val executorImage = System.getProperty( - "spark.docker.test.executorImage", - "spark-executor:latest") - private val initContainerImage = System.getProperty( - "spark.docker.test.initContainerImage", - "spark-init:latest") - + private var image: String = _ + private var containerLocalSparkDistroExamplesJar: String = _ override def beforeAll(): Unit = { + val sparkDirProp = System.getProperty("spark.kubernetes.test.unpackSparkDir") + require(sparkDirProp != null, "Spark home directory must be provided in system properties.") + sparkHomeDir = Paths.get(sparkDirProp) + require(sparkHomeDir.toFile.isDirectory, + s"No directory found for spark home specified at $sparkHomeDir.") + val imageTag = getTestImageTag + val imageRepo = getTestImageRepo + image = s"$imageRepo/spark:$imageTag" + + val sparkDistroExamplesJarFile: File = sparkHomeDir.resolve(Paths.get("examples", "jars")) + .toFile + .listFiles(new PatternFilenameFilter(Pattern.compile("^spark-examples_.*\\.jar$")))(0) + containerLocalSparkDistroExamplesJar = s"local:///opt/spark/examples/jars/" + + s"${sparkDistroExamplesJarFile.getName}" testBackend.initialize() kubernetesTestComponents = new KubernetesTestComponents(testBackend.getKubernetesClient) } @@ -64,12 +69,8 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit before { sparkAppConf = kubernetesTestComponents.newSparkAppConf() - .set("spark.kubernetes.driver.container.image", driverImage) - .set("spark.kubernetes.executor.container.image", executorImage) + .set("spark.kubernetes.container.image", image) .set("spark.kubernetes.driver.label.spark-app-locator", APP_LOCATOR_LABEL) - .set(DRIVER_DOCKER_IMAGE, tagImage("spark-driver")) - .set(EXECUTOR_DOCKER_IMAGE, tagImage("spark-executor")) - .set(INIT_CONTAINER_DOCKER_IMAGE, tagImage("spark-init")) .set("spark.kubernetes.executor.label.spark-app-locator", APP_LOCATOR_LABEL) kubernetesTestComponents.createNamespace() } @@ -151,7 +152,6 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit .set("spark.kubernetes.mountDependencies.filesDownloadDir", CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) - .set("spark.kubernetes.initContainer.image", initContainerImage) runSparkPageRankAndVerifyCompletion( appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE)) } @@ -164,7 +164,6 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) .set(s"spark.kubernetes.driver.secrets.$TEST_SECRET_NAME", TEST_SECRET_MOUNT_PATH) .set(s"spark.kubernetes.executor.secrets.$TEST_SECRET_NAME", TEST_SECRET_MOUNT_PATH) - .set("spark.kubernetes.initContainer.image", initContainerImage) createTestSecret() @@ -181,7 +180,7 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit } private def runSparkPiAndVerifyCompletion( - appResource: String = CONTAINER_LOCAL_SPARK_DISTRO_EXAMPLES_JAR, + appResource: String = containerLocalSparkDistroExamplesJar, driverPodChecker: Pod => Unit = doBasicDriverPodCheck, executorPodChecker: Pod => Unit = doBasicExecutorPodCheck, appArgs: Array[String] = Array.empty[String]): Unit = { @@ -193,9 +192,8 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit driverPodChecker, executorPodChecker) } - private def runSparkPageRankAndVerifyCompletion( - appResource: String = CONTAINER_LOCAL_SPARK_DISTRO_EXAMPLES_JAR, + appResource: String = containerLocalSparkDistroExamplesJar, driverPodChecker: Pod => Unit = doBasicDriverPodCheck, executorPodChecker: Pod => Unit = doBasicExecutorPodCheck, appArgs: Array[String]): Unit = { @@ -219,7 +217,7 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit mainAppResource = appResource, mainClass = mainClass, appArgs = appArgs) - SparkAppLauncher.launch(appArguments, sparkAppConf, TIMEOUT.value.toSeconds.toInt) + SparkAppLauncher.launch(appArguments, sparkAppConf, TIMEOUT.value.toSeconds.toInt, sparkHomeDir) val driverPod = kubernetesTestComponents.kubernetesClient .pods() @@ -250,15 +248,14 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit } } } - private def tagImage(image: String): String = s"$image:${testBackend.dockerImageTag()}" private def doBasicDriverPodCheck(driverPod: Pod): Unit = { - assert(driverPod.getSpec.getContainers.get(0).getImage === driverImage) + assert(driverPod.getSpec.getContainers.get(0).getImage === image) assert(driverPod.getSpec.getContainers.get(0).getName === "spark-kubernetes-driver") } private def doBasicExecutorPodCheck(executorPod: Pod): Unit = { - assert(executorPod.getSpec.getContainers.get(0).getImage === executorImage) + assert(executorPod.getSpec.getContainers.get(0).getImage === image) assert(executorPod.getSpec.getContainers.get(0).getName === "executor") } @@ -318,12 +315,6 @@ private[spark] object KubernetesSuite { val TIMEOUT = PatienceConfiguration.Timeout(Span(2, Minutes)) val INTERVAL = PatienceConfiguration.Interval(Span(2, Seconds)) - val SPARK_DISTRO_EXAMPLES_JAR_FILE: File = Paths.get(SPARK_DISTRO_PATH.toFile.getAbsolutePath, - "examples", "jars") - .toFile - .listFiles(new PatternFilenameFilter(Pattern.compile("^spark-examples_.*\\.jar$")))(0) - val CONTAINER_LOCAL_SPARK_DISTRO_EXAMPLES_JAR: String = s"local:///opt/spark/examples/jars/" + - s"${SPARK_DISTRO_EXAMPLES_JAR_FILE.getName}" val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi" val SPARK_PAGE_RANK_MAIN_CLASS: String = "org.apache.spark.examples.SparkPageRank" diff --git a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala similarity index 89% rename from integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala rename to src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala index 00ef1c5..fb285e7 100644 --- a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala +++ b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala @@ -16,17 +16,14 @@ */ package org.apache.spark.deploy.k8s.integrationtest -import java.nio.file.Paths +import java.nio.file.{Path, Paths} import java.util.UUID import scala.collection.mutable import scala.collection.JavaConverters._ - import io.fabric8.kubernetes.client.DefaultKubernetesClient import org.scalatest.concurrent.Eventually -import org.apache.spark.deploy.k8s.integrationtest.constants.SPARK_DISTRO_PATH - private[spark] class KubernetesTestComponents(defaultClient: DefaultKubernetesClient) { val namespace = UUID.randomUUID().toString.replaceAll("-", "") @@ -92,12 +89,14 @@ private[spark] case class SparkAppArguments( private[spark] object SparkAppLauncher extends Logging { - private val SPARK_SUBMIT_EXECUTABLE_DEST = Paths.get(SPARK_DISTRO_PATH.toFile.getAbsolutePath, - "bin", "spark-submit").toFile - - def launch(appArguments: SparkAppArguments, appConf: SparkAppConf, timeoutSecs: Int): Unit = { + def launch( + appArguments: SparkAppArguments, + appConf: SparkAppConf, + timeoutSecs: Int, + sparkHomeDir: Path): Unit = { + val sparkSubmitExecutable = sparkHomeDir.resolve(Paths.get("bin", "spark-submit")) logInfo(s"Launching a spark app with arguments $appArguments and conf $appConf") - val commandLine = Array(SPARK_SUBMIT_EXECUTABLE_DEST.getAbsolutePath, + val commandLine = Array(sparkSubmitExecutable.toFile.getAbsolutePath, "--deploy-mode", "cluster", "--class", appArguments.mainClass, "--master", appConf.get("spark.master") diff --git a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala similarity index 100% rename from integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala rename to src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala diff --git a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala similarity index 98% rename from integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala rename to src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala index e9f143c..aa6425d 100644 --- a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala +++ b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala @@ -39,6 +39,6 @@ object ProcessUtils extends Logging { assert(proc.waitFor(timeout, TimeUnit.SECONDS), s"Timed out while executing ${fullCommand.mkString(" ")}") assert(proc.exitValue == 0, s"Failed to execute ${fullCommand.mkString(" ")}") - outputLines.toSeq + outputLines } } diff --git a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala similarity index 100% rename from integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala rename to src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala diff --git a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala similarity index 100% rename from integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala rename to src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala diff --git a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/GCE/GCETestBackend.scala b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/GCE/GCETestBackend.scala similarity index 92% rename from integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/GCE/GCETestBackend.scala rename to src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/GCE/GCETestBackend.scala index 345ccc8..5639f97 100644 --- a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/GCE/GCETestBackend.scala +++ b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/GCE/GCETestBackend.scala @@ -36,8 +36,4 @@ private[spark] class GCETestBackend(val master: String) extends IntegrationTestB override def getKubernetesClient(): DefaultKubernetesClient = { defaultClient } - - override def dockerImageTag(): String = { - return System.getProperty(KUBERNETES_TEST_DOCKER_TAG_SYSTEM_PROPERTY, "latest") - } } diff --git a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala similarity index 97% rename from integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala rename to src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala index 9c64c64..67f8540 100644 --- a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala +++ b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala @@ -25,7 +25,6 @@ import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.MinikubeTest private[spark] trait IntegrationTestBackend { def initialize(): Unit def getKubernetesClient: DefaultKubernetesClient - def dockerImageTag(): String def cleanUp(): Unit = {} } diff --git a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala similarity index 87% rename from integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala rename to src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala index 8204852..7145d85 100644 --- a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala +++ b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala @@ -16,6 +16,7 @@ */ package org.apache.spark.deploy.k8s.integrationtest.backend.minikube +import java.io.File import java.nio.file.Paths import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient} @@ -24,6 +25,7 @@ import org.apache.spark.deploy.k8s.integrationtest.{Logging, ProcessUtils} // TODO support windows private[spark] object Minikube extends Logging { + private val MINIKUBE_STARTUP_TIMEOUT_SECONDS = 60 def getMinikubeIp: String = { @@ -43,14 +45,6 @@ private[spark] object Minikube extends Logging { .getOrElse(throw new IllegalStateException(s"Unknown status $statusString")) } - def getDockerEnv: Map[String, String] = { - executeMinikube("docker-env", "--shell", "bash") - .filter(_.startsWith("export")) - .map(_.replaceFirst("export ", "").split('=')) - .map(arr => (arr(0), arr(1).replaceAllLiterally("\"", ""))) - .toMap - } - def getKubernetesClient: DefaultKubernetesClient = { val kubernetesMaster = s"https://${getMinikubeIp}:8443" val userHome = System.getProperty("user.home") @@ -64,13 +58,9 @@ private[spark] object Minikube extends Logging { new DefaultKubernetesClient(kubernetesConf) } - def executeMinikubeSsh(command: String): Unit = { - executeMinikube("ssh", command) - } - private def executeMinikube(action: String, args: String*): Seq[String] = { ProcessUtils.executeProcess( - Array("minikube", action) ++ args, MINIKUBE_STARTUP_TIMEOUT_SECONDS) + Array("bash", "-c", s"minikube $action") ++ args, MINIKUBE_STARTUP_TIMEOUT_SECONDS) } } diff --git a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala similarity index 66% rename from integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala rename to src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala index 89db42f..1bb888f 100644 --- a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala +++ b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala @@ -19,33 +19,23 @@ package org.apache.spark.deploy.k8s.integrationtest.backend.minikube import io.fabric8.kubernetes.client.DefaultKubernetesClient import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend -import org.apache.spark.deploy.k8s.integrationtest.config._ -import org.apache.spark.deploy.k8s.integrationtest.docker.KubernetesSuiteDockerManager private[spark] object MinikubeTestBackend extends IntegrationTestBackend { private var defaultClient: DefaultKubernetesClient = _ - private val userProvidedDockerImageTag = Option( - System.getProperty(KUBERNETES_TEST_DOCKER_TAG_SYSTEM_PROPERTY)) - private val dockerManager = new KubernetesSuiteDockerManager( - Minikube.getDockerEnv, userProvidedDockerImageTag) override def initialize(): Unit = { val minikubeStatus = Minikube.getMinikubeStatus require(minikubeStatus == MinikubeStatus.RUNNING, - s"Minikube must be running before integration tests can execute. Current status" + - s" is: $minikubeStatus") - dockerManager.buildSparkDockerImages() + s"Minikube must be running to use the Minikube backend for integration tests." + + s" Current status is: $minikubeStatus.") defaultClient = Minikube.getKubernetesClient } override def cleanUp(): Unit = { super.cleanUp() - dockerManager.deleteImages() } - override def getKubernetesClient(): DefaultKubernetesClient = { + override def getKubernetesClient: DefaultKubernetesClient = { defaultClient } - - override def dockerImageTag(): String = dockerManager.dockerImageTag() } diff --git a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala similarity index 54% rename from integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala rename to src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala index d82a1de..a81ef45 100644 --- a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala +++ b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala @@ -16,9 +16,23 @@ */ package org.apache.spark.deploy.k8s.integrationtest +import java.io.File + +import com.google.common.base.Charsets +import com.google.common.io.Files + package object config { - val KUBERNETES_TEST_DOCKER_TAG_SYSTEM_PROPERTY = "spark.kubernetes.test.imageDockerTag" - val DRIVER_DOCKER_IMAGE = "spark.kubernetes.driver.container.image" - val EXECUTOR_DOCKER_IMAGE = "spark.kubernetes.executor.container.image" - val INIT_CONTAINER_DOCKER_IMAGE = "spark.kubernetes.initcontainer.container.image" + def getTestImageTag: String = { + val imageTagFileProp = System.getProperty("spark.kubernetes.test.imageTagFile") + require(imageTagFileProp != null, "Image tag file must be provided in system properties.") + val imageTagFile = new File(imageTagFileProp) + require(imageTagFile.isFile, s"No file found for image tag at ${imageTagFile.getAbsolutePath}.") + Files.toString(imageTagFile, Charsets.UTF_8).trim + } + + def getTestImageRepo: String = { + val imageRepo = System.getProperty("spark.kubernetes.test.imageRepo") + require(imageRepo != null, "Image repo must be provided in system properties.") + imageRepo + } } diff --git a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala similarity index 91% rename from integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala rename to src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala index 9137199..0807a68 100644 --- a/integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala +++ b/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala @@ -16,10 +16,7 @@ */ package org.apache.spark.deploy.k8s.integrationtest -import java.nio.file.Paths - package object constants { val MINIKUBE_TEST_BACKEND = "minikube" val GCE_TEST_BACKEND = "gce" - val SPARK_DISTRO_PATH = Paths.get("target", "spark-distro") }