adopt e2e labels usage

Author: cahillsf

Makefile

@@ -79,6 +79,7 @@ export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH)
 #
 GINKGO_FOCUS ?=
 GINKGO_SKIP ?=
+GINKGO_LABEL_FILTER ?=
 GINKGO_NODES ?= 1
 GINKGO_TIMEOUT ?= 2h
 GINKGO_POLL_PROGRESS_AFTER ?= 60m
@@ -978,7 +979,7 @@ test-test-extension-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integra
 .PHONY: test-e2e
 test-e2e: $(GINKGO) generate-e2e-templates ## Run the end-to-end tests
 	$(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \
-		-poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" \
+		-poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" --label-filter="$(GINKGO_LABEL_FILTER)" \
 		$(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \
 		--output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(ROOT_DIR)/$(TEST_DIR)/e2e -- \
 	    -e2e.artifacts-folder="$(ARTIFACTS)" \

docs/book/src/developer/core/testing.md

@@ -170,7 +170,7 @@ For example to run [pull-cluster-api-e2e-main](https://github.com/kubernetes/tes
 just execute:
 
 ```bash
-GINKGO_FOCUS="\[PR-Blocking\]" ./scripts/ci-e2e.sh
+GINKGO_LABEL_FILTER="PR-Blocking" ./scripts/ci-e2e.sh
 ```
 
 ### Test execution via make test-e2e
@@ -190,7 +190,7 @@ kind images). This can be done by executing the `./scripts/ci-e2e.sh` script.
 # Notes:
 # * You can cancel the script as soon as it starts the actual test execution via `make test-e2e`.
 # * If you want to run other tests (e.g. upgrade tests), make sure all required env variables are set (see the Prow Job config).
-GINKGO_FOCUS="\[PR-Blocking\]" ./scripts/ci-e2e.sh
+GINKGO_LABEL_FILTER="PR-Blocking" ./scripts/ci-e2e.sh
 ```
 
 Now, the tests can be run in an IDE. The following describes how this can be done in IntelliJ IDEA and VS Code. It should work
@@ -272,24 +272,24 @@ kustomize_substitutions:
 
 ### Running specific tests
 
-To run a subset of tests, a combination of either one or both of `GINKGO_FOCUS` and `GINKGO_SKIP` env variables can be set.
+To run a subset of tests the `GINKGO_LABEL_FILTER` env variable can be set.  See [Ginkgo Spec Labels v2](https://onsi.github.io/ginkgo/MIGRATING_TO_V2#spec-labels) for complete syntax documentation.
+
 Each of these can be used to match tests, for example:
-- `[PR-Blocking]` => Sanity tests run before each PR merge
-- `[K8s-Upgrade]` => Tests which verify k8s component version upgrades on workload clusters
-- `[Conformance]` => Tests which run the k8s conformance suite on workload clusters
-- `[ClusterClass]` => Tests which use a ClusterClass to create a workload cluster
-- `When testing KCP.*` => Tests which start with `When testing KCP`
+- `PR-Blocking` => Sanity tests run before each PR merge
+- `K8s-Upgrade` => Tests which verify k8s component version upgrades on workload clusters
+- `Conformance` => Tests which run the k8s conformance suite on workload clusters
+- `ClusterClass` => Tests which use a ClusterClass to create a workload cluster
+- `/When testing KCP.*/` => Tests which start with `When testing KCP`
 
 For example:
-` GINKGO_FOCUS="\\[PR-Blocking\\]" make test-e2e ` can be used to run the sanity E2E tests
-` GINKGO_SKIP="\\[K8s-Upgrade\\]" make test-e2e ` can be used to skip the upgrade E2E tests
+` GINKGO_LABEL_FILTER="PR-Blocking" make test-e2e ` can be used to run the sanity E2E tests
+` GINKGO_LABEL_FILTER="!K8s-Upgrade" make test-e2e ` can be used to skip the upgrade E2E tests
 
 ### Further customization
 
 The following env variables can be set to customize the test execution:
 
-- `GINKGO_FOCUS` to set ginkgo focus (default empty - all tests)
-- `GINKGO_SKIP` to set ginkgo skip (default empty - to allow running all tests)
+- `GINKGO_LABEL_FILTER` to set ginkgo label filter (default empty - all tests)
 - `GINKGO_NODES` to set the number of ginkgo parallel nodes (default to 1)
 - `E2E_CONF_FILE` to set the e2e test config file (default to ${REPO_ROOT}/test/e2e/config/docker.yaml)
 - `ARTIFACTS` to set the folder where test artifact will be stored (default to ${REPO_ROOT}/_artifacts)

scripts/ci-e2e-lib.sh

@@ -34,6 +34,60 @@ capi:buildDockerImages () {
   fi
 }
 
+# k8s::prepareKindestImagesVariables defaults the environment variables KUBERNETES_VERSION_MANAGEMENT, KUBERNETES_VERSION,
+# KUBERNETES_VERSION_UPGRADE_TO, KUBERNETES_VERSION_UPGRADE_FROM and KUBERNETES_VERSION_LATEST_CI
+# depending on what is set in GINKGO_LABEL_FILTER.
+# Note: We do this to ensure that the kindest/node image gets built if it does
+# not already exist, e.g. for pre-releases, but only if necessary.
+k8s::prepareKindestImagesVariablesGingkoFilters() {
+  # Always default KUBERNETES_VERSION_MANAGEMENT because we always create a management cluster out of it.
+  if [[ -z "${KUBERNETES_VERSION_MANAGEMENT:-}" ]]; then
+    KUBERNETES_VERSION_MANAGEMENT=$(grep KUBERNETES_VERSION_MANAGEMENT: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
+    echo "Defaulting KUBERNETES_VERSION_MANAGEMENT to ${KUBERNETES_VERSION_MANAGEMENT} to trigger image build (because env var is not set)"
+  fi
+
+  if [[ ${GINKGO_LABEL_FILTER:-} == *"K8s-Install-ci-latest"* ]]; then
+    # If the test focuses on [K8s-Install-ci-latest], only default KUBERNETES_VERSION_LATEST_CI
+    # to the value in the e2e config and only if it is not set.
+    # Note: We do this because we want to specify KUBERNETES_VERSION_LATEST_CI *only* in the e2e config.
+    if [[ -z "${KUBERNETES_VERSION_LATEST_CI:-}" ]]; then
+      KUBERNETES_VERSION_LATEST_CI=$(grep KUBERNETES_VERSION_LATEST_CI: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
+      echo "Defaulting KUBERNETES_VERSION_LATEST_CI to ${KUBERNETES_VERSION_LATEST_CI} to trigger image build (because env var is not set)"
+    fi
+  elif [[ ${GINKGO_LABEL_FILTER:-} != *"K8s-Upgrade"* ]]; then
+    # In any other case which is not [K8s-Upgrade], default KUBERNETES_VERSION if it is not set to make sure
+    # the corresponding kindest/node image exists.
+    if [[ -z "${KUBERNETES_VERSION:-}" ]]; then
+      KUBERNETES_VERSION=$(grep KUBERNETES_VERSION: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
+      echo "Defaulting KUBERNETES_VERSION to ${KUBERNETES_VERSION} to trigger image build (because env var is not set)"
+    fi
+  fi
+
+  # Tests not focusing on anything and skipping [Conformance] run a clusterctl upgrade test
+  # on the latest kubernetes version as management cluster.
+  if  [[ ${GINKGO_LABEL_FILTER} =~ ^!Conformance$ ]]; then
+    # Note: We do this because we want to specify KUBERNETES_VERSION_LATEST_CI *only* in the e2e config.
+    if [[ -z "${KUBERNETES_VERSION_LATEST_CI:-}" ]]; then
+      KUBERNETES_VERSION_LATEST_CI=$(grep KUBERNETES_VERSION_LATEST_CI: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
+      echo "Defaulting KUBERNETES_VERSION_LATEST_CI to ${KUBERNETES_VERSION_LATEST_CI} to trigger image build (because env var is not set)"
+    fi
+  fi
+
+  # Tests not focusing on [PR-Blocking], [K8s-Install] or [K8s-Install-ci-latest],
+  # also run upgrade tests so default KUBERNETES_VERSION_UPGRADE_TO and KUBERNETES_VERSION_UPGRADE_FROM
+  # to the value in the e2e config if they are not set.
+  if [[ ${GINKGO_LABEL_FILTER:-} != *"PR-Blocking"* ]] && [[ ${GINKGO_LABEL_FILTER:-} != *"K8s-Install"* ]]; then
+    if [[ -z "${KUBERNETES_VERSION_UPGRADE_TO:-}" ]]; then
+      KUBERNETES_VERSION_UPGRADE_TO=$(grep KUBERNETES_VERSION_UPGRADE_TO: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
+      echo "Defaulting KUBERNETES_VERSION_UPGRADE_TO to ${KUBERNETES_VERSION_UPGRADE_TO} to trigger image build (because env var is not set)"
+    fi
+    if [[ -z "${KUBERNETES_VERSION_UPGRADE_FROM:-}" ]]; then
+      KUBERNETES_VERSION_UPGRADE_FROM=$(grep KUBERNETES_VERSION_UPGRADE_FROM: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
+      echo "Defaulting KUBERNETES_VERSION_UPGRADE_FROM to ${KUBERNETES_VERSION_UPGRADE_FROM} to trigger image build (because env var is not set)"
+    fi
+  fi
+}
+
 # k8s::prepareKindestImagesVariables defaults the environment variables KUBERNETES_VERSION_MANAGEMENT, KUBERNETES_VERSION,
 # KUBERNETES_VERSION_UPGRADE_TO, KUBERNETES_VERSION_UPGRADE_FROM and KUBERNETES_VERSION_LATEST_CI
 # depending on what is set in GINKGO_FOCUS.

scripts/ci-e2e.sh

@@ -53,7 +53,12 @@ export USE_EXISTING_CLUSTER=false
 # - KUBERNETES_VERSION_UPGRADE_FROM
 # - KUBERNETES_VERSION_LATEST_CI
 # - KUBERNETES_VERSION_MANAGEMENT
-k8s::prepareKindestImagesVariables
+if [[ -n "${GINKGO_LABEL_FILTER}" ]]; then
+  echo "Preparing kindest/node images for Ginkgo label filter: ${GINKGO_LABEL_FILTER}"
+  k8s::prepareKindestImagesVariablesGingkoFilters
+else
+  k8s::prepareKindestImagesVariables
+fi
 k8s::prepareKindestImages
 
 # pre-pull all the images that will be used in the e2e, thus making the actual test run

test/infrastructure/inmemory/README.md

@@ -126,7 +126,7 @@ kubectl --kubeconfig=/tmp/kubeconfig --server=https://127.0.0.1:$CONTROL_PLANE_E
 ### E2E tests
 
 CAPIM could be used to run a subset of CAPI E2E tests, but as of today we maintain only a smoke E2E scale test 
-(10 clusters, 1 CP and 3 workers each) that can be executed by setting `GINKGO_FOCUS="in-memory"`.
+(10 clusters, 1 CP and 3 workers each) that can be executed by setting `GINKGO_LABEL_FILTER="in-memory"`.
 
 See [Running the end-to-end tests locally](https://cluster-api.sigs.k8s.io/developer/testing#running-the-end-to-end-tests-locally) for more details.