Merge pull request kubernetes-sigs#638 from chewong/clusterctl-test-framework

✨ switch to clusterctl-based e2e framework

Author: k8s-ci-robot

.gitignore

@@ -54,6 +54,8 @@ tilt-settings.json
 
 # e2e output
 test/e2e/junit.e2e_suite.*.xml
+test/e2e/capz-e2e-*.yaml
+test/e2e/config/azure-dev-envsubst.yaml
 test/e2e/logs/*
 _artifacts
 

Makefile

@@ -73,6 +73,13 @@ RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
 # Allow overriding the imagePullPolicy
 PULL_POLICY ?= Always
 
+# Allow overriding the e2e configurations
+ARTIFACTS ?= $(ROOT_DIR)/_artifacts
+E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml
+E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
+SKIP_CLEANUP ?= false
+SKIP_CREATE_MGMT_CLUSTER ?= false
+
 CLUSTER_TEMPLATE ?= cluster-template.yaml
 MANAGED_CLUSTER_TEMPLATE ?= cluster-template-aks.yaml
 
@@ -101,10 +108,15 @@ test-integration: ## Run integration tests
 	go test -v -tags=integration ./test/integration/...
 
 .PHONY: test-e2e
-test-e2e: ## Run e2e tests
+test-e2e: $(ENVSUBST) ## Run e2e tests
 	PULL_POLICY=IfNotPresent $(MAKE) docker-build
 	MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \
-	go test ./test/e2e -v -tags=e2e -ginkgo.v -ginkgo.trace -count=1 -timeout=90m
+	$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
+	go test ./test/e2e -v -tags=e2e -ginkgo.v -ginkgo.trace -count=1 -timeout=90m \
+		-e2e.artifacts-folder="$(ARTIFACTS)" \
+		-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
+		-e2e.skip-resource-cleanup="$(SKIP_CLEANUP)" \
+		-e2e.use-existing-cluster="$(SKIP_CREATE_MGMT_CLUSTER)"
 
 $(KUBECTL) $(KUBE_APISERVER) $(ETCD): ## install test asset kubectl, kube-apiserver, etcd
 	source ./scripts/fetch_ext_bins.sh && fetch_tools

docs/development.md

@@ -335,6 +335,8 @@ To generate the mocks you can run
 make generate-go
 ```
 
+
+
 #### E2E Testing
 
 To run E2E locally, set `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_SUBSCRIPTION_ID`, `AZURE_TENANT_ID` and run:
@@ -343,7 +345,15 @@ To run E2E locally, set `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_SUBSCRI
 ./scripts/ci-e2e.sh
 ```
 
-You can optionally set `AZURE_SSH_PUBLIC_KEY_FILE` to use your own SSH key.
+You can optionally set the following variables:
+
+| Variable                   | Description                                                                                                               | Default                                                                               |
+|----------------------------|---------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------|
+| `E2E_CONF_FILE`            | The path of the [E2E configuration file](https://cluster-api.sigs.k8s.io/developer/e2e.html#defining-an-e2e-config-file).      | `${GOPATH}/src/sigs.k8s.io/cluster-api-provider-azure/test/e2e/config/azure-dev.yaml`  |
+| `SKIP_CLEANUP`             | Set to `true` if you do not want the bootstrap and workload clusters to be cleaned up after running E2E tests.            | `false`                                                                               |
+| `SKIP_CREATE_MGMT_CLUSTER` | Skip management cluster creation.                                                                                         | `false`                                                                               |
+
+You can also customize the configuration of the CAPZ cluster created by the E2E tests (except for `CLUSTER_NAME`, `AZURE_RESOURCE_GROUP`, `AZURE_VNET_NAME`, `CONTROL_PLANE_MACHINE_COUNT`, and `WORKER_MACHINE_COUNT`, since they are generated by individual test cases). See [Customizing the cluster deployment](#customizing-the-cluster-deployment) for more details.
 
 #### Conformance Testing
 
@@ -355,20 +365,20 @@ To run Conformance locally, set `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE
 
 You can optionally set the following variables:
 
-| Variable                    | Description                                                                                                   |
-|-----------------------------|---------------------------------------------------------------------------------------------------------------|
-| `AZURE_SSH_PUBLIC_KEY_FILE` | Use your own SSH key.                                                                                         |
-| `SKIP_CREATE_CLUSTER`       | Skip cluster creation.                                                                                        |
-| `SKIP_UPSTREAM_E2E_TESTS`   | Skip running upstream Kubernetes E2E tests.                                                                   |
-| `SKIP_CLEANUP`              | Skip deleting the cluster after the tests finish running.                                                      |
-| `KUBECONFIG`                | Provide your existing cluster kubeconfig filepath. If no kubeconfig is provided, `./kubeconfig` will be used.     |
-| `SKIP`                      | Regexp for test cases to skip.                                                                                |
-| `FOCUS`                     | Regexp for which test cases to run.                                                                           |
-| `PARALLEL`                  | Skip serial tests and set --ginkgo-parallel.                                                                  |
-| `USE_CI_ARTIFACTS`          | Use a CI version of Kubernetes, ie. not a released version (eg. `v1.19.0-alpha.1.426+0926c9c47677e9`)         |
-| `CI_VERSION`                | Provide a custom CI version of Kubernetes. By default, the latest master commit will be used.                 |
-
-You can also customize the configuration of the CAPZ cluster (assuming that `SKIP_CREATE_CLUSTER` is not set). See [Customizing the cluster deployment](#customizing-the-cluster-deployment) for more details.
+| Variable                       | Description                                                                                                   |
+|--------------------------------|---------------------------------------------------------------------------------------------------------------|
+| `AZURE_SSH_PUBLIC_KEY_FILE`    | Use your own SSH key.                                                                                         |
+| `SKIP_CREATE_WORKLOAD_CLUSTER` | Skip workload cluster creation.                                                                               |
+| `SKIP_UPSTREAM_E2E_TESTS`      | Skip running upstream Kubernetes E2E tests.                                                                   |
+| `SKIP_CLEANUP`                 | Skip deleting the cluster after the tests finish running.                                                      |
+| `KUBECONFIG`                   | Provide your existing cluster kubeconfig filepath. If no kubeconfig is provided, `./kubeconfig` will be used.     |
+| `SKIP`                         | Regexp for test cases to skip.                                                                                |
+| `FOCUS`                        | Regexp for which test cases to run.                                                                           |
+| `PARALLEL`                     | Skip serial tests and set --ginkgo-parallel.                                                                  |
+| `USE_CI_ARTIFACTS`             | Use a CI version of Kubernetes, ie. not a released version (eg. `v1.19.0-alpha.1.426+0926c9c47677e9`)         |
+| `CI_VERSION`                   | Provide a custom CI version of Kubernetes. By default, the latest master commit will be used.                 |
+
+You can also customize the configuration of the CAPZ cluster (assuming that `SKIP_CREATE_WORKLOAD_CLUSTER` is not set). See [Customizing the cluster deployment](#customizing-the-cluster-deployment) for more details.
 
 In addition to upstream E2E, you can append custom commands to `./scripts/ci-entrypoint.sh` to run E2E from other projects against a CAPZ cluster:
 

go.mod

@@ -15,10 +15,8 @@ require (
 	github.com/google/gofuzz v1.1.0
 	github.com/onsi/ginkgo v1.12.0
 	github.com/onsi/gomega v1.9.0
-	github.com/pelletier/go-toml v1.7.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.6.3
 	golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
 	golang.org/x/net v0.0.0-20200301022130-244492dfa37a
 	k8s.io/api v0.17.2

go.sum

@@ -92,7 +92,6 @@ github.com/coredns/corefile-migration v1.0.7 h1:T2eOj/NKN1Q1W1bD9MFeZiBYryS0JlWT
 github.com/coredns/corefile-migration v1.0.7/go.mod h1:OFwBp/Wc9dJt5cAZzHWMNhK1r5L0p0jDwIBc6j8NC8E=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -373,8 +372,6 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
 github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
-github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI=
-github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -450,8 +447,6 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E=
 github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
-github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs=
-github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=

scripts/ci-e2e.sh

@@ -33,8 +33,42 @@ source "${REPO_ROOT}/hack/ensure-kind.sh"
 source "${REPO_ROOT}/hack/ensure-kubectl.sh"
 # shellcheck source=../hack/ensure-kustomize.sh
 source "${REPO_ROOT}/hack/ensure-kustomize.sh"
+# shellcheck source=../hack/parse-prow-creds.sh
+source "${REPO_ROOT}/hack/parse-prow-creds.sh"
 
-export REGISTRY=e2e
+# Verify the required Environment Variables are present.
+: "${AZURE_SUBSCRIPTION_ID:?Environment variable empty or not defined.}"
+: "${AZURE_TENANT_ID:?Environment variable empty or not defined.}"
+: "${AZURE_CLIENT_ID:?Environment variable empty or not defined.}"
+: "${AZURE_CLIENT_SECRET:?Environment variable empty or not defined.}"
+
+get_random_region() {
+    local REGIONS=("eastus" "eastus2" "southcentralus" "westus2" "westeurope")
+    echo "${REGIONS[${RANDOM} % ${#REGIONS[@]}]}"
+}
+
+export REGISTRY="e2e"
+export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZURE_SUBSCRIPTION_ID" | base64 | tr -d '\n')"
+export AZURE_TENANT_ID_B64="$(echo -n "$AZURE_TENANT_ID" | base64 | tr -d '\n')"
+export AZURE_CLIENT_ID_B64="$(echo -n "$AZURE_CLIENT_ID" | base64 | tr -d '\n')"
+export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZURE_CLIENT_SECRET" | base64 | tr -d '\n')"
+export AZURE_LOCATION="${AZURE_LOCATION:-$(get_random_region)}"
+export AZURE_CONTROL_PLANE_MACHINE_TYPE="${AZURE_CONTROL_PLANE_MACHINE_TYPE:-"Standard_D2s_v3"}"
+export AZURE_NODE_MACHINE_TYPE="${AZURE_NODE_MACHINE_TYPE:-"Standard_D2s_v3"}"
+
+# Generate SSH key.
+AZURE_SSH_PUBLIC_KEY_FILE=${AZURE_SSH_PUBLIC_KEY_FILE:-""}
+if ! [ -n "${AZURE_SSH_PUBLIC_KEY_FILE}" ]; then
+    SSH_KEY_FILE=.sshkey
+    rm -f "${SSH_KEY_FILE}" 2>/dev/null
+    ssh-keygen -t rsa -b 2048 -f "${SSH_KEY_FILE}" -N '' 1>/dev/null
+    AZURE_SSH_PUBLIC_KEY_FILE="${SSH_KEY_FILE}.pub"
+fi
+export AZURE_SSH_PUBLIC_KEY=$(cat "${AZURE_SSH_PUBLIC_KEY_FILE}" | base64 | tr -d '\r\n')
+
+# timestamp is in RFC-3339 format to match kubetest
+export TIMESTAMP="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+export JOB_NAME="${JOB_NAME:-"cluster-api-provider-azure-e2e"}"
 
 make test-e2e
 test_status="${?}"

scripts/ci-entrypoint.sh

@@ -156,7 +156,7 @@ ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
 mkdir -p "${ARTIFACTS}/logs"
 
 # create cluster
-if [[ -z "${SKIP_CREATE_CLUSTER:-}" ]]; then
+if [[ -z "${SKIP_CREATE_WORKLOAD_CLUSTER:-}" ]]; then
     create_cluster
 fi