tests: add GCE based conformance test scripts

Author: aaronlevy

hack/quickstart/init-master.sh

@@ -5,6 +5,7 @@ REMOTE_HOST=$1
 REMOTE_PORT=${REMOTE_PORT:-22}
 CLUSTER_DIR=${CLUSTER_DIR:-cluster}
 IDENT=${IDENT:-${HOME}/.ssh/id_rsa}
+SSH_OPTS=${SSH_OPTS:-}
 
 BOOTKUBE_REPO=quay.io/coreos/bootkube
 BOOTKUBE_VERSION=v0.2.2
@@ -105,19 +106,19 @@ function init_master_node() {
 # After assets are available on the remote host, the script will execute itself in "local" mode.
 if [ "${REMOTE_HOST}" != "local" ]; then
     # Set up the kubelet.service on remote host
-    scp -i ${IDENT} -P ${REMOTE_PORT} kubelet.master core@${REMOTE_HOST}:/home/core/kubelet.master
-    ssh -i ${IDENT} -p ${REMOTE_PORT} core@${REMOTE_HOST} "sudo mv /home/core/kubelet.master /etc/systemd/system/kubelet.service"
+    scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} kubelet.master core@${REMOTE_HOST}:/home/core/kubelet.master
+    ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "sudo mv /home/core/kubelet.master /etc/systemd/system/kubelet.service"
 
     # Copy self to remote host so script can be executed in "local" mode
-    scp -i ${IDENT} -P ${REMOTE_PORT} ${BASH_SOURCE[0]} core@${REMOTE_HOST}:/home/core/init-master.sh
-    ssh -i ${IDENT} -p ${REMOTE_PORT} core@${REMOTE_HOST} "sudo /home/core/init-master.sh local"
+    scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${BASH_SOURCE[0]} core@${REMOTE_HOST}:/home/core/init-master.sh
+    ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "sudo /home/core/init-master.sh local"
 
     # Copy assets from remote host to a local directory. These can be used to launch additional nodes & contain TLS assets
     mkdir ${CLUSTER_DIR}
-    scp -q -i ${IDENT} -P ${REMOTE_PORT} -r core@${REMOTE_HOST}:/home/core/assets/* ${CLUSTER_DIR}
+    scp -q -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} -r core@${REMOTE_HOST}:/home/core/assets/* ${CLUSTER_DIR}
 
     # Cleanup
-    ssh -i ${IDENT} -p ${REMOTE_PORT} core@${REMOTE_HOST} "rm -rf /home/core/assets && rm -rf /home/core/init-master.sh"
+    ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "rm -rf /home/core/assets && rm -rf /home/core/init-master.sh"
 
     echo "Cluster assets copied to ${CLUSTER_DIR}"
     echo

hack/quickstart/init-worker.sh

@@ -5,6 +5,7 @@ REMOTE_HOST=$1
 KUBECONFIG=$2
 REMOTE_PORT=${REMOTE_PORT:-22}
 IDENT=${IDENT:-${HOME}/.ssh/id_rsa}
+SSH_OPTS=${SSH_OPTS:-}
 
 function usage() {
     echo "USAGE:"
@@ -34,7 +35,9 @@ function extract_master_endpoint (){
     grep 'client-key-data' ${KUBECONFIG} | awk '{print $2}'| base64 -d > /home/core/client.key
 
     MASTER_PUB="$(awk '/server:/ {print $2}' ${KUBECONFIG} | awk -F/ '{print $3}' | awk -F: '{print $1}')"
-    MASTER_PRIV=$(curl https://${MASTER_PUB}:443/api/v1/namespaces/default/endpoints/kubernetes \
+    # TODO (aaron): The -k was added with the gce conformance tests - figure out why it's needed here.
+    #     The certs are seemingly signed correctly, but says no SAN for MASTER_PUB
+    MASTER_PRIV=$(curl -k https://${MASTER_PUB}:443/api/v1/namespaces/default/endpoints/kubernetes \
         --cacert /home/core/ca.crt --cert /home/core/client.crt --key /home/core/client.key \
         | jq -r '.subsets[0].addresses[0].ip')
     rm -f /home/core/ca.crt /home/core/client.crt /home/core/client.key
@@ -66,15 +69,15 @@ function init_worker_node() {
 if [ "${REMOTE_HOST}" != "local" ]; then
 
     # Copy kubelet service file and kubeconfig to remote host
-    scp -i ${IDENT} -P ${REMOTE_PORT} kubelet.worker core@${REMOTE_HOST}:/home/core/kubelet.worker
-    scp -i ${IDENT} -P ${REMOTE_PORT} ${KUBECONFIG} core@${REMOTE_HOST}:/home/core/kubeconfig
+    scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} kubelet.worker core@${REMOTE_HOST}:/home/core/kubelet.worker
+    scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${KUBECONFIG} core@${REMOTE_HOST}:/home/core/kubeconfig
 
     # Copy self to remote host so script can be executed in "local" mode
-    scp -i ${IDENT} -P ${REMOTE_PORT} ${BASH_SOURCE[0]} core@${REMOTE_HOST}:/home/core/init-worker.sh
-    ssh -i ${IDENT} -p ${REMOTE_PORT} core@${REMOTE_HOST} "sudo /home/core/init-worker.sh local /home/core/kubeconfig"
+    scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${BASH_SOURCE[0]} core@${REMOTE_HOST}:/home/core/init-worker.sh
+    ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "sudo /home/core/init-worker.sh local /home/core/kubeconfig"
 
     # Cleanup
-    ssh -i ${IDENT} -p ${REMOTE_PORT} core@${REMOTE_HOST} "rm /home/core/init-worker.sh"
+    ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "rm /home/core/init-worker.sh"
 
     echo
     echo "Node bootstrap complete. It may take a few minutes for the node to become ready. Access your kubernetes cluster using:"

hack/single-node/conformance-test.sh

@@ -4,5 +4,4 @@ set -euo pipefail
 ssh_key="$(vagrant ssh-config | awk '/IdentityFile/ {print $2}' | tr -d '"')"
 ssh_port="$(vagrant ssh-config | awk '/Port [0-9]+/ {print $2}')"
 
-export CHECK_NODE_COUNT=false
 ../tests/conformance-test.sh "127.0.0.1" "${ssh_port}" "${ssh_key}"

hack/tests/conformance-gce.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+set -euo pipefail
+
+# DESCRIPTION:
+#
+# This script is meant to launch GCE nodes, run bootkube to bootstrap a self-hosted k8s cluster, then run conformance tests.
+#
+# REQUIREMENTS:
+#  - gcloud cli is installed
+#  - rkt is available on the host
+#  - $BUILD_ROOT environment variable is set and contains a checkout of bootkube at $BUILD_ROOT/bootkube
+#  - $KEY_FILE environment variable is set as path to GCE service account keyfile
+#
+# PROCESS:
+#
+# Inside a rkt container:
+#   - Use gcloud to launch master node
+#     - Use the quickstart init-master.sh script to run bootkube on that node
+#   - Use gcloud to launch worker node(s)
+#     - Use the quickstart init-worker.sh script to join node to kubernetes cluster
+#   - Run conformance tests against the launched cluster
+#
+WORKER_COUNT=4
+COREOS_IMAGE=${COREOS_IMAGE:-'https://www.googleapis.com/compute/v1/projects/coreos-cloud/global/images/coreos-stable-1122-2-0-v20160906'}
+
+function cleanup {
+    gcloud compute instances delete --quiet --zone us-central1-a bootkube-ci-m1 || true
+    gcloud compute firewall-rules delete --quiet bootkube-ci-api-443 || true
+    for i in $(seq 1 ${WORKER_COUNT}); do
+        gcloud compute instances delete --quiet --zone us-central1-a bootkube-ci-w${i} || true
+    done
+    rm -rf /build/cluster
+}
+
+function init {
+    curl https://sdk.cloud.google.com | bash
+    source ~/.bashrc
+    gcloud config set project coreos-gce-testing
+    gcloud auth activate-service-account [email protected] --key-file=/build/keyfile
+    apt-get update && apt-get install -y jq
+
+    ssh-keygen -t rsa -f /root/.ssh/id_rsa -N ""
+    awk '{print "core:" $1 " " $2 " core@bootkube-ci"}' /root/.ssh/id_rsa.pub > /root/.ssh/gce-format.pub
+}
+
+function add_master {
+    gcloud compute instances create bootkube-ci-m1 \
+        --image ${COREOS_IMAGE} --zone us-central1-a --machine-type n1-standard-4 --boot-disk-size=10GB
+
+    gcloud compute instances add-tags --zone us-central1-a bootkube-ci-m1 --tags bootkube-ci-apiserver
+    gcloud compute firewall-rules create bootkube-ci-api-443 --target-tags=bootkube-ci-apiserver --allow tcp:443
+
+    gcloud compute instances add-metadata bootkube-ci-m1 --zone us-central1-a --metadata-from-file ssh-keys=/root/.ssh/gce-format.pub
+
+    MASTER_IP=$(gcloud compute instances list bootkube-ci-m1 --format=json | jq --raw-output '.[].networkInterfaces[].accessConfigs[].natIP')
+    cd /build/bootkube/hack/quickstart && SSH_OPTS="-o StrictHostKeyChecking=no" CLUSTER_DIR=/build/cluster ./init-master.sh ${MASTER_IP}
+}
+
+function add_workers {
+    #TODO (aaron): parallelize launching workers
+    for i in $(seq 1 ${WORKER_COUNT}); do
+        gcloud compute instances create bootkube-ci-w${i} \
+            --image ${COREOS_IMAGE} --zone us-central1-a --machine-type n1-standard-1
+
+        gcloud compute instances add-metadata bootkube-ci-w${i} --zone us-central1-a --metadata-from-file ssh-keys=/root/.ssh/gce-format.pub
+
+        local WORKER_IP=$(gcloud compute instances list bootkube-ci-w${i} --format=json | jq --raw-output '.[].networkInterfaces[].accessConfigs[].natIP')
+        cd /build/bootkube/hack/quickstart && SSH_OPTS="-o StrictHostKeyChecking=no" ./init-worker.sh ${WORKER_IP} /build/cluster/auth/kubeconfig
+    done
+}
+
+IN_CONTAINER=${IN_CONTAINER:-false}
+if [ "${IN_CONTAINER}" == true ]; then
+    #TODO(aaron): should probably run cleanup as part of init (not just on exit). Or add some random identifier to objects created during this run.
+    trap cleanup EXIT
+    init
+    add_master
+    add_workers
+    KUBECONFIG=/etc/kubernetes/kubeconfig WORKER_COUNT=${WORKER_COUNT} /build/bootkube/hack/tests/conformance-test.sh ${MASTER_IP} 22 /root/.ssh/id_rsa
+else
+    BUILD_ROOT=${BUILD_ROOT:-}
+    if [ -z "$BUILD_ROOT" ]; then
+        echo "BUILD_ROOT must be set"
+        exit 1
+    fi
+    if [ -z "$KEY_FILE" ]; then
+        echo "KEY_FILE must be set"
+        exit 1
+    fi
+
+    RKT_OPTS=$(echo \
+        "--volume buildroot,kind=host,source=${BUILD_ROOT} " \
+        "--mount volume=buildroot,target=/build " \
+        "--volume keyfile,kind=host,source=${KEY_FILE} " \
+        "--mount volume=keyfile,target=/build/keyfile " \
+    )
+
+    sudo rkt run --insecure-options=image ${RKT_OPTS} docker://golang:1.6.3 --exec /bin/bash -- -c "IN_CONTAINER=true /build/bootkube/hack/tests/$(basename $0)"
+fi

hack/tests/conformance-test.sh

@@ -1,10 +1,8 @@
 #!/bin/bash
 set -euo pipefail
 
-CHECK_NODE_COUNT=${CHECK_NODE_COUNT:-true}
 CONFORMANCE_REPO=${CONFORMANCE_REPO:-github.com/coreos/kubernetes}
 CONFORMANCE_VERSION=${CONFORMANCE_VERSION:-v1.4.5+coreos.0}
-TEST_ARGS=${TEST_ARGS:-"--ginkgo.focus='\[Conformance\]' --ginkgo.skip='\[Flaky\]|\[Feature:.+\]'"}
 
 usage() {
     echo "USAGE:"
@@ -22,29 +20,29 @@ ssh_host=$1
 ssh_port=$2
 ssh_key=$3
 
+KUBECONFIG=${KUBECONFIG:-/home/core/cluster/auth/kubeconfig}
 K8S_SRC=/home/core/go/src/k8s.io/kubernetes
 ssh -q -o stricthostkeychecking=no -i ${ssh_key} -p ${ssh_port} core@${ssh_host} \
     "mkdir -p ${K8S_SRC} && [[ -d ${K8S_SRC}/.git ]] || git clone https://${CONFORMANCE_REPO} ${K8S_SRC}"
 
-RKT_OPTS=$(echo \
-    "--volume=kc,kind=host,source=/home/core/cluster/auth/kubeconfig "\
-    "--volume=k8s,kind=host,source=${K8S_SRC} " \
-    "--mount volume=kc,target=/kubeconfig " \
-    "--mount volume=k8s,target=/go/src/k8s.io/kubernetes")
+RKT_OPTS="\
+ --volume=kc,kind=host,source=${KUBECONFIG} \
+ --volume=k8s,kind=host,source=${K8S_SRC} \
+ --mount volume=kc,target=/kubeconfig \
+ --mount volume=k8s,target=/go/src/k8s.io/kubernetes"
 
 # Init steps necessary to run conformance in golang container
 INIT="apt-get update && apt-get install -y rsync && go get -u github.com/jteeuwen/go-bindata/go-bindata"
 
-TEST_FLAGS="-v --test -check_version_skew=false -check_node_count=${CHECK_NODE_COUNT} --test_args=\"${TEST_ARGS}\""
+BUILD="cd /go/src/k8s.io/kubernetes && \
+ git checkout ${CONFORMANCE_VERSION} && \
+ make all WHAT=cmd/kubectl && \
+ make all WHAT=vendor/github.com/onsi/ginkgo/ginkgo && \
+ make all WHAT=test/e2e/e2e.test"
 
-CONFORMANCE=$(echo \
-    "cd /go/src/k8s.io/kubernetes && " \
-    "git checkout ${CONFORMANCE_VERSION} && " \
-    "make all WHAT=cmd/kubectl && " \
-    "make all WHAT=vendor/github.com/onsi/ginkgo/ginkgo && " \
-    "make all WHAT=test/e2e/e2e.test && " \
-    "KUBECONFIG=/kubeconfig KUBERNETES_PROVIDER=skeleton KUBERNETES_CONFORMANCE_TEST=Y go run hack/e2e.go ${TEST_FLAGS}")
-
-CMD="sudo rkt run --insecure-options=image ${RKT_OPTS} docker://golang:1.6.3 --exec /bin/bash -- -c \"${INIT} && ${CONFORMANCE}\""
+CONFORMANCE="\
+ KUBECONFIG=/kubeconfig KUBERNETES_PROVIDER=skeleton KUBERNETES_CONFORMANCE_TEST=Y go run hack/e2e.go \
+ -v --test -check_version_skew=false -check_node_count=false --test_args='--ginkgo.focus=\[Conformance\]'"
 
+CMD="sudo rkt run --insecure-options=image ${RKT_OPTS} docker://golang:1.6.3 --exec /bin/bash -- -c \"${INIT} && ${BUILD} && ${CONFORMANCE}\""
 ssh -q -o stricthostkeychecking=no -i ${ssh_key} -p ${ssh_port} core@${ssh_host} "${CMD}"