ci: build e2e prerequisites in parallel

With this change we build the prerequisites of the e2e tests in parallel
with the devstack build. We also upload the e2e image to devstack while
devstack is still building, as soon as SSH is available.

Author: mdbooth

Makefile

@@ -184,7 +184,7 @@ $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/cluster-template.yaml: $(E2E_KUSTOMIZE_DIR)/wit
 $(E2E_NO_ARTIFACT_TEMPLATES_DIR)/cluster-template-%.yaml: $(E2E_KUSTOMIZE_DIR)/% $(KUSTOMIZE) FORCE
 	$(KUSTOMIZE) build "$<" > "$@"
 
-e2e-prerequisites: e2e-templates e2e-image test-e2e-image-prerequisites ## Build all artifacts required by e2e tests
+e2e-prerequisites: $(GINKGO) e2e-templates e2e-image test-e2e-image-prerequisites ## Build all artifacts required by e2e tests
 
 # Can be run manually, e.g. via:
 # export OPENSTACK_CLOUD_YAML_FILE="$(pwd)/clouds.yaml"
@@ -198,6 +198,12 @@ test-e2e: $(GINKGO) e2e-prerequisites ## Run e2e tests
 			-config-path="$(E2E_CONF_PATH)" -artifacts-folder="$(ARTIFACTS)" \
 			-data-folder="$(E2E_DATA_DIR)" $(E2E_ARGS)
 
+# Pre-compile tests
+# This is not required, but it will make the tests start faster
+.PHONY: build-e2e-tests
+build-e2e-tests: $(GINKGO)
+	$(GINKGO) build -tags=e2e ./test/e2e/suites/e2e/...
+
 .PHONY: e2e-image
 e2e-image: CONTROLLER_IMG_TAG = "gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e"
 e2e-image: docker-build

hack/ci/common.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# retry $1 times with $2 sleep in between
+function retry {
+    attempt=0
+    max_attempts=${1}
+    interval=${2}
+    shift; shift
+    until [[ "$attempt" -ge "$max_attempts" ]] ; do
+        attempt=$((attempt+1))
+        set +e
+        eval "$*" && return || echo "failed $attempt times: $*"
+        set -e
+        sleep "$interval"
+    done
+    echo "error: reached max attempts at retry($*)"
+    return 1
+}
+
+function wait_for_ssh {
+    local ip=$1 && shift
+
+    retry 10 30 "$(get_ssh_cmd) ${ip} -- true"
+}
+
+function get_ssh_cmd {
+    echo "ssh -l cloud $(get_ssh_common_args)"
+}
+
+function get_ssh_common_args {
+    local private_key_file=$(get_ssh_private_key_file)
+    if [ -z "$private_key_file" ]; then
+        # If there's no private key file use the public key instead
+        # This allows us to specify a private key which is held only on a
+        # hardware device and therefore has no key file
+        private_key_file=$(get_ssh_public_key_file)
+    fi
+
+    echo "-i ${private_key_file} " \
+         "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o PasswordAuthentication=no "
+}

hack/ci/create_devstack.sh

@@ -63,22 +63,7 @@ ARTIFACTS=${ARTIFACTS:-/tmp/${CLUSTER_NAME}-artifacts}
 devstackdir="${ARTIFACTS}/devstack"
 mkdir -p "$devstackdir"
 
-# retry $1 times with $2 sleep in between
-function retry {
-    attempt=0
-    max_attempts=${1}
-    interval=${2}
-    shift; shift
-    until [[ "$attempt" -ge "$max_attempts" ]] ; do
-        attempt=$((attempt+1))
-        set +e
-        eval "$*" && return || echo "failed $attempt times: $*"
-        set -e
-        sleep "$interval"
-    done
-    echo "error: reached max attempts at retry($*)"
-    return 1
-}
+source "${REPO_ROOT}/hack/ci/common.sh"
 
 function ensure_openstack_client {
     if ! command -v openstack;
@@ -102,12 +87,6 @@ function ensure_openstack_client {
     fi
 }
 
-function wait_for_ssh {
-    local ip=$1 && shift
-
-    retry 10 30 "$(get_ssh_cmd) ${ip} -- true"
-}
-
 function start_sshuttle {
     if ! command -v sshuttle;
     then
@@ -175,19 +154,6 @@ function wait_for_devstack {
     done
 }
 
-function get_ssh_cmd {
-    local private_key_file=$(get_ssh_private_key_file)
-    if [ -z "$private_key_file" ]; then
-        # If there's no private key file use the public key instead
-        # This allows us to specify a private key which is held only on a
-        # hardware device and therefore has no key file
-        private_key_file=$(get_ssh_public_key_file)
-    fi
-
-    echo "ssh -i ${private_key_file} -l cloud " \
-         "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o PasswordAuthentication=no "
-}
-
 function create_devstack {
     local name=$1 && shift
     local ip=$1 && shift

scripts/ci-e2e.sh

@@ -80,25 +80,41 @@ if [ -n "${BOSKOS_HOST:-}" ]; then
   HEART_BEAT_PID=$!
 fi
 
-"hack/ci/create_devstack.sh"
-
-# Upload image for e2e clusterctl upgrade tests
-source "${REPO_ROOT}/hack/ci/${RESOURCE_TYPE}.sh"
-CONTAINER_ARCHIVE="/tmp/capo-e2e-image.tar"
-SSH_KEY="$(get_ssh_private_key_file)"
-SSH_ARGS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o PasswordAuthentication=no"
-CONTROLLER_IP=${CONTROLLER_IP:-"10.0.3.15"}
-
-make e2e-image
-docker save -o "${CONTAINER_ARCHIVE}" gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e
-scp -i "${SSH_KEY}" ${SSH_ARGS} "${CONTAINER_ARCHIVE}" "cloud@${CONTROLLER_IP}:capo-e2e-image.tar"
-ssh -i "${SSH_KEY}" ${SSH_ARGS} "cloud@${CONTROLLER_IP}" -- sudo chown root:root capo-e2e-image.tar
-ssh -i "${SSH_KEY}" ${SSH_ARGS} "cloud@${CONTROLLER_IP}" -- sudo chmod u=rw,g=r,o=r capo-e2e-image.tar
-ssh -i "${SSH_KEY}" ${SSH_ARGS} "cloud@${CONTROLLER_IP}" -- sudo mv capo-e2e-image.tar /var/www/html/capo-e2e-image.tar
-
-export OPENSTACK_CLOUD_YAML_FILE
-OPENSTACK_CLOUD_YAML_FILE="$(pwd)/clouds.yaml"
-make test-e2e
+# Run e2e prerequisites concurrently with devstack build to save time
+build_out=$(mktemp)
+(
+    # Run prerequisites at low priority to avoid slowing down devstack tasks,
+    # which generally take much longer
+    ionice nice make e2e-image e2e-prerequisites build-e2e-tests
+
+    container_archive=$(mktemp --suffix=.tar)
+    ionice nice docker save -o "${container_archive}" gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e
+
+    # Wait for SSH to become available in the provisioning devstack
+    # infrastructure before uploading image archive
+    CONTROLLER_IP=${CONTROLLER_IP:-"10.0.3.15"}
+    source "${REPO_ROOT}/hack/ci/${RESOURCE_TYPE}.sh"
+    source "${REPO_ROOT}/hack/ci/common.sh"
+    wait_for_ssh "${CONTROLLER_IP}"
+
+    retry 10 10 scp $(get_ssh_common_args) "${container_archive}" "cloud@${CONTROLLER_IP}:capo-e2e-image.tar"
+    retry 10 10 $(get_ssh_cmd) ${CONTROLLER_IP} -- sudo chown root:root capo-e2e-image.tar
+    retry 10 10 $(get_ssh_cmd) ${CONTROLLER_IP} -- sudo chmod u=rw,g=r,o=r capo-e2e-image.tar
+    retry 10 10 $(get_ssh_cmd) ${CONTROLLER_IP} -- sudo mv capo-e2e-image.tar /var/www/html/capo-e2e-image.tar
+) >"$build_out" 2>&1 &
+build_pid=$!
+
+# Build the devstack environment
+hack/ci/create_devstack.sh
+
+# Wait for and capture exit of build process
+# Log build output and exit if the build failed
+wait $build_pid
+ret=$?
+cat "$build_out"
+[ "$ret" != 0 ] && exit "$ret"
+
+make test-e2e OPENSTACK_CLOUD_YAML_FILE="$(pwd)/clouds.yaml"
 test_status="${?}"
 
 # If Boskos is being used then release the resource back to Boskos.