iterate

Author: cahillsf

Makefile

@@ -78,7 +78,6 @@ export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH)
 # Ginkgo configuration.
 #
 GINKGO_LABEL_FILTER ?=
-GINKGO_SKIP ?=
 GINKGO_NODES ?= 1
 GINKGO_TIMEOUT ?= 2h
 GINKGO_POLL_PROGRESS_AFTER ?= 60m
@@ -88,11 +87,6 @@ SKIP_RESOURCE_CLEANUP ?= false
 USE_EXISTING_CLUSTER ?= false
 GINKGO_NOCOLOR ?= false
 
-# to set multiple ginkgo skip flags, if any
-ifneq ($(strip $(GINKGO_SKIP)),)
-_SKIP_ARGS := $(foreach arg,$(strip $(GINKGO_SKIP)),-skip="$(arg)")
-endif
-
 # Helper function to get dependency version from go.mod
 get_go_version = $(shell go list -m $1 | awk '{print $$2}')
 
@@ -979,7 +973,7 @@ test-test-extension-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integra
 test-e2e: $(GINKGO) generate-e2e-templates ## Run the end-to-end tests
 	$(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \
 		-poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --label-filter="$(GINKGO_LABEL_FILTER)" \
-		$(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \
+	  --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \
 		--output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(ROOT_DIR)/$(TEST_DIR)/e2e -- \
 	    -e2e.artifacts-folder="$(ARTIFACTS)" \
 	    -e2e.config="$(E2E_CONF_FILE)" \

docs/book/src/developer/core/testing.md

@@ -272,24 +272,24 @@ kustomize_substitutions:
 
 ### Running specific tests
 
-To run a subset of tests, a combination of either one or both of `GINKGO_FOCUS` and `GINKGO_SKIP` env variables can be set.
+To run a subset of tests the `GINKGO_LABEL_FILTER` env variable can be set.  See [Ginkgo Spec Labels v2](https://onsi.github.io/ginkgo/MIGRATING_TO_V2#spec-labels) for complete syntax documentation.
+
 Each of these can be used to match tests, for example:
-- `[PR-Blocking]` => Sanity tests run before each PR merge
-- `[K8s-Upgrade]` => Tests which verify k8s component version upgrades on workload clusters
-- `[Conformance]` => Tests which run the k8s conformance suite on workload clusters
-- `[ClusterClass]` => Tests which use a ClusterClass to create a workload cluster
-- `When testing KCP.*` => Tests which start with `When testing KCP`
+- `PR-Blocking` => Sanity tests run before each PR merge
+- `K8s-Upgrade` => Tests which verify k8s component version upgrades on workload clusters
+- `Conformance` => Tests which run the k8s conformance suite on workload clusters
+- `ClusterClass` => Tests which use a ClusterClass to create a workload cluster
+- `/When testing KCP.*/` => Tests which start with `When testing KCP`
 
 For example:
 ` GINKGO_LABEL_FILTER="PR-Blocking" make test-e2e ` can be used to run the sanity E2E tests
-` GINKGO_LABEL_FILTER="K8s-Upgrade" make test-e2e ` can be used to skip the upgrade E2E tests
+` GINKGO_LABEL_FILTER="!K8s-Upgrade" make test-e2e ` can be used to skip the upgrade E2E tests
 
 ### Further customization
 
 The following env variables can be set to customize the test execution:
 
 - `GINKGO_LABEL_FILTER` to set ginkgo label filter (default empty - all tests)
-- `GINKGO_SKIP` to set ginkgo skip (default empty - to allow running all tests)
 - `GINKGO_NODES` to set the number of ginkgo parallel nodes (default to 1)
 - `E2E_CONF_FILE` to set the e2e test config file (default to ${REPO_ROOT}/test/e2e/config/docker.yaml)
 - `ARTIFACTS` to set the folder where test artifact will be stored (default to ${REPO_ROOT}/_artifacts)

scripts/ci-e2e-lib.sh

@@ -36,7 +36,7 @@ capi:buildDockerImages () {
 
 # k8s::prepareKindestImagesVariables defaults the environment variables KUBERNETES_VERSION_MANAGEMENT, KUBERNETES_VERSION,
 # KUBERNETES_VERSION_UPGRADE_TO, KUBERNETES_VERSION_UPGRADE_FROM and KUBERNETES_VERSION_LATEST_CI
-# depending on what is set in GINKGO_FOCUS.
+# depending on what is set in GINKGO_LABEL_FILTER.
 # Note: We do this to ensure that the kindest/node image gets built if it does
 # not already exist, e.g. for pre-releases, but only if necessary.
 k8s::prepareKindestImagesVariables() {
@@ -46,15 +46,15 @@ k8s::prepareKindestImagesVariables() {
     echo "Defaulting KUBERNETES_VERSION_MANAGEMENT to ${KUBERNETES_VERSION_MANAGEMENT} to trigger image build (because env var is not set)"
   fi
 
-  if [[ ${GINKGO_FOCUS:-} == *"K8s-Install-ci-latest"* ]]; then
+  if [[ ${GINKGO_LABEL_FILTER:-} == *"K8s-Install-ci-latest"* ]]; then
     # If the test focuses on [K8s-Install-ci-latest], only default KUBERNETES_VERSION_LATEST_CI
     # to the value in the e2e config and only if it is not set.
     # Note: We do this because we want to specify KUBERNETES_VERSION_LATEST_CI *only* in the e2e config.
     if [[ -z "${KUBERNETES_VERSION_LATEST_CI:-}" ]]; then
       KUBERNETES_VERSION_LATEST_CI=$(grep KUBERNETES_VERSION_LATEST_CI: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
       echo "Defaulting KUBERNETES_VERSION_LATEST_CI to ${KUBERNETES_VERSION_LATEST_CI} to trigger image build (because env var is not set)"
     fi
-  elif [[ ${GINKGO_FOCUS:-} != *"K8s-Upgrade"* ]]; then
+  elif [[ ${GINKGO_LABEL_FILTER:-} != *"K8s-Upgrade"* ]]; then
     # In any other case which is not [K8s-Upgrade], default KUBERNETES_VERSION if it is not set to make sure
     # the corresponding kindest/node image exists.
     if [[ -z "${KUBERNETES_VERSION:-}" ]]; then
@@ -65,7 +65,7 @@ k8s::prepareKindestImagesVariables() {
 
   # Tests not focusing on anything and skipping [Conformance] run a clusterctl upgrade test
   # on the latest kubernetes version as management cluster.
-  if  [[ ${GINKGO_FOCUS:-} == "" ]] && [[ ${GINKGO_SKIP} == *"Conformance"* ]]; then
+  if  [[ ${GINKGO_LABEL_FILTER:-} == "" ]] && [[ ${GINKGO_LABEL_FILTER} == *"!Conformance"* ]]; then
     # Note: We do this because we want to specify KUBERNETES_VERSION_LATEST_CI *only* in the e2e config.
     if [[ -z "${KUBERNETES_VERSION_LATEST_CI:-}" ]]; then
       KUBERNETES_VERSION_LATEST_CI=$(grep KUBERNETES_VERSION_LATEST_CI: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
@@ -76,7 +76,7 @@ k8s::prepareKindestImagesVariables() {
   # Tests not focusing on [PR-Blocking], [K8s-Install] or [K8s-Install-ci-latest],
   # also run upgrade tests so default KUBERNETES_VERSION_UPGRADE_TO and KUBERNETES_VERSION_UPGRADE_FROM
   # to the value in the e2e config if they are not set.
-  if [[ ${GINKGO_FOCUS:-} != *"PR-Blocking"* ]] && [[ ${GINKGO_FOCUS:-} != *"K8s-Install"* ]]; then
+  if [[ ${GINKGO_LABEL_FILTER:-} != *"PR-Blocking"* ]] && [[ ${GINKGO_LABEL_FILTER:-} != *"K8s-Install"* ]]; then
     if [[ -z "${KUBERNETES_VERSION_UPGRADE_TO:-}" ]]; then
       KUBERNETES_VERSION_UPGRADE_TO=$(grep KUBERNETES_VERSION_UPGRADE_TO: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}')
       echo "Defaulting KUBERNETES_VERSION_UPGRADE_TO to ${KUBERNETES_VERSION_UPGRADE_TO} to trigger image build (because env var is not set)"

test/infrastructure/inmemory/README.md

@@ -126,7 +126,7 @@ kubectl --kubeconfig=/tmp/kubeconfig --server=https://127.0.0.1:$CONTROL_PLANE_E
 ### E2E tests
 
 CAPIM could be used to run a subset of CAPI E2E tests, but as of today we maintain only a smoke E2E scale test 
-(10 clusters, 1 CP and 3 workers each) that can be executed by setting `GINKGO_FOCUS="in-memory"`.
+(10 clusters, 1 CP and 3 workers each) that can be executed by setting `GINKGO_LABEL_FILTER="in-memory"`.
 
 See [Running the end-to-end tests locally](https://cluster-api.sigs.k8s.io/developer/testing#running-the-end-to-end-tests-locally) for more details.