Merge remote-tracking branch 'origin/master' into 'ifilonenko/master'

Author: mccheah

README.md

@@ -6,9 +6,45 @@ title: Spark on Kubernetes Integration Tests
 # Running the Kubernetes Integration Tests
 
 Note that the integration test framework is currently being heavily revised and
-is subject to change.
+is subject to change. Note that currently the integration tests only run with Java 8.
 
-Note that currently the integration tests only run with Java 8.
+As shorthand to run the tests against any given cluster, you can use the `e2e/runner.sh` script.
+The script assumes that you have a functioning Kubernetes cluster (1.6+) with kubectl
+configured to access it. The master URL of the currently configured cluster on your
+machine can be discovered as follows:
+
+```
+$ kubectl cluster-info
+
+Kubernetes master is running at https://xyz
+```
+
+If you want to use a local [minikube](https://github.com/kubernetes/minikube) cluster,
+the minimum tested version is 0.23.0, with the kube-dns addon enabled
+and the recommended configuration is 3 CPUs and 4G of memory. There is also a wrapper
+script for running on minikube, `e2e/e2e-minikube.sh` for testing the apache/spark repo
+in specific.
+
+```
+$ minikube start --memory 4000 --cpus 3
+```
+
+If you're using a non-local cluster, you must provide an image repository
+which you have write access to, using the `-i` option, in order to store docker images
+generated during the test.
+
+Example usages of the script:
+
+```
+$ ./e2e/runner.sh -m https://xyz -i docker.io/foxish -d cloud
+$ ./e2e/runner.sh -m https://xyz -i test -d minikube
+$ ./e2e/runner.sh -m https://xyz -i test -r https://github.com/my-spark/spark -d minikube
+
+```
+
+# Detailed Documentation
+
+## Running the tests using maven
 
 Integration tests firstly require installing [Minikube](https://kubernetes.io/docs/getting-started-guides/minikube/) on
 your machine, and for the `Minikube` binary to be on your `PATH`.. Refer to the Minikube documentation for instructions
@@ -44,16 +80,15 @@ $ mvn clean integration-test  \
     -Dspark-distro-tgz=spark/spark-2.3.0-SNAPSHOT-bin.tgz
 ```
 
-# Running against an arbitrary cluster
+## Running against an arbitrary cluster
 
 In order to run against any cluster, use the following:
 ```sh
 $ mvn clean integration-test  \
     -Dspark-distro-tgz=spark/spark-2.3.0-SNAPSHOT-bin.tgz  \
     -DextraScalaTestArgs="-Dspark.kubernetes.test.master=k8s://https://<master>
-```
 
-# Specify existing docker images via image:tag
+## Reuse the previous Docker images
 
 The integration tests build a number of Docker images, which takes some time.
 By default, the images are built every time the tests run.  You may want to skip

e2e/e2e-minikube.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+### This script can be used to run integration tests locally on minikube.
+### Requirements: minikube v0.23+ with the DNS addon enabled, and kubectl configured to point to it.
+
+set -ex
+
+### Basic Validation ###
+if [ ! -d "integration-test" ]; then
+  echo "This script must be invoked from the top-level directory of the integration-tests repository"
+  usage
+  exit 1
+fi
+
+# Set up config.
+master=$(kubectl cluster-info | head -n 1 | grep -oE "https?://[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?")
+repo="https://github.com/apache/spark"
+image_repo=test
+
+# Run tests in minikube mode.
+./e2e/runner.sh -m $master -r $repo -i $image_repo -d minikube

e2e/e2e-prow.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+### This script is used by Kubernetes Test Infrastructure to run integration tests.
+### See documenation at https://github.com/kubernetes/test-infra/tree/master/prow
+### To run the integration tests yourself, use e2e/runner.sh.
+
+set -ex
+
+# Install basic dependencies
+echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list
+apt-get update && apt-get install -y curl wget git tar
+apt-get install -t jessie-backports -y openjdk-8-jdk
+
+# Set up config.
+master=$(kubectl cluster-info | head -n 1 | grep -oE "https?://[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?")
+repo="https://github.com/apache/spark"
+
+# Special GCP project for publishing docker images built by test.
+image_repo="gcr.io/spark-testing-191023"
+cd "$(dirname "$0")"/../
+./e2e/runner.sh -m $master -r $repo -i $image_repo -d cloud
+
+# Copy out the junit xml files for consumption by k8s test-infra.
+ls -1 ./integration-test/target/surefire-reports/*.xml | cat -n | while read n f; do cp "$f" "/workspace/_artifacts/junit_0$n.xml"; done

e2e/runner.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+usage () {
+  echo "Usage:"
+  echo "    ./e2e/runner.sh -h Display this help message."
+  echo "    ./e2e/runner.sh -m <master-url> -r <spark-repo> -i <image-repo> -d [minikube|cloud]"
+  echo "        note that you must have kubectl configured to access the specified"
+  echo "        <master-url>. Also you must have access to the <image-repo>. "
+  echo "        The deployment mode can be specified using the 'd' flag."
+}
+
+### Basic Validation ###
+if [ ! -d "integration-test" ]; then
+  echo "This script must be invoked from the top-level directory of the integration-tests repository"
+  usage
+  exit 1
+fi
+
+### Set sensible defaults ###
+REPO="https://github.com/apache/spark"
+IMAGE_REPO="docker.io/kubespark"
+DEPLOY_MODE="minikube"
+
+### Parse options ###
+while getopts h:m:r:i:d: option
+do
+ case "${option}"
+ in
+ h)
+  usage
+  exit 0
+  ;;
+ m) MASTER=${OPTARG};;
+ r) REPO=${OPTARG};;
+ i) IMAGE_REPO=${OPTARG};;
+ d) DEPLOY_MODE=${OPTARG};;
+ \? )
+ echo "Invalid Option: -$OPTARG" 1>&2
+ exit 1
+ ;;
+ esac
+done
+
+### Ensure cluster is set.
+if [ -z "$MASTER" ]
+then
+   echo "Missing master-url (-m) argument."
+   echo ""
+   usage
+   exit
+fi
+
+### Ensure deployment mode is minikube/cloud.
+if [[ $DEPLOY_MODE != minikube && $DEPLOY_MODE != cloud ]];
+then
+  echo "Invalid deployment mode $DEPLOY_MODE"
+  usage
+  exit 1
+fi
+
+echo "Running tests on cluster $MASTER against $REPO."
+echo "Spark images will be created in $IMAGE_REPO"
+
+set -ex
+root=$(pwd)
+
+# clone spark distribution if needed.
+if [ -d "spark" ];
+then
+  (cd spark && git pull);
+else
+  git clone $REPO;
+fi
+
+cd spark && ./dev/make-distribution.sh --tgz -Phadoop-2.7 -Pkubernetes -DskipTests
+tag=$(git rev-parse HEAD | cut -c -6)
+echo "Spark distribution built at SHA $tag"
+
+if  [[ $DEPLOY_MODE == cloud ]] ;
+then
+  cd dist && ./sbin/build-push-docker-images.sh -r $IMAGE_REPO -t $tag build
+  if  [[ $IMAGE_REPO == gcr.io* ]] ;
+  then
+    gcloud docker -- push $IMAGE_REPO/spark-driver:$tag && \
+    gcloud docker -- push $IMAGE_REPO/spark-executor:$tag && \
+    gcloud docker -- push $IMAGE_REPO/spark-init:$tag
+  else
+    ./sbin/build-push-docker-images.sh -r $IMAGE_REPO -t $tag push
+  fi
+else
+  # -m option for minikube.
+  cd dist && ./sbin/build-push-docker-images.sh -m -r $IMAGE_REPO -t $tag build
+fi
+
+cd $root/integration-test
+$root/spark/build/mvn clean -Ddownload.plugin.skip=true integration-test \
+          -Dspark-distro-tgz=$root/spark/*.tgz \
+          -DextraScalaTestArgs="-Dspark.kubernetes.test.master=k8s://$MASTER \
+            -Dspark.docker.test.driverImage=$IMAGE_REPO/spark-driver:$tag \
+            -Dspark.docker.test.executorImage=$IMAGE_REPO/spark-executor:$tag \
+            -Dspark.docker.test.initContainerImage=$IMAGE_REPO/spark-init:$tag" || :
+
+echo "TEST SUITE FINISHED"