diff --git a/Makefile b/Makefile index 71bc7dae4837..c2e2f29a715e 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,7 @@ export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) # GINKGO_FOCUS ?= GINKGO_SKIP ?= +GINKGO_LABEL_FILTER ?= GINKGO_NODES ?= 1 GINKGO_TIMEOUT ?= 2h GINKGO_POLL_PROGRESS_AFTER ?= 60m @@ -973,7 +974,7 @@ test-test-extension-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integra .PHONY: test-e2e test-e2e: $(GINKGO) generate-e2e-templates ## Run the end-to-end tests $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \ - -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" \ + -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" --label-filter="$(GINKGO_LABEL_FILTER)" \ $(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \ --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(ROOT_DIR)/$(TEST_DIR)/e2e -- \ -e2e.artifacts-folder="$(ARTIFACTS)" \ diff --git a/docs/book/src/developer/core/testing.md b/docs/book/src/developer/core/testing.md index 87d38667bd47..4d4d7a2c44a9 100644 --- a/docs/book/src/developer/core/testing.md +++ b/docs/book/src/developer/core/testing.md @@ -170,7 +170,7 @@ For example to run [pull-cluster-api-e2e-main](https://github.com/kubernetes/tes just execute: ```bash -GINKGO_FOCUS="\[PR-Blocking\]" ./scripts/ci-e2e.sh +GINKGO_LABEL_FILTER="PR-Blocking" ./scripts/ci-e2e.sh ``` ### Test execution via make test-e2e @@ -190,7 +190,7 @@ kind images). This can be done by executing the `./scripts/ci-e2e.sh` script. # Notes: # * You can cancel the script as soon as it starts the actual test execution via `make test-e2e`. # * If you want to run other tests (e.g. upgrade tests), make sure all required env variables are set (see the Prow Job config). -GINKGO_FOCUS="\[PR-Blocking\]" ./scripts/ci-e2e.sh +GINKGO_LABEL_FILTER="PR-Blocking" ./scripts/ci-e2e.sh ``` Now, the tests can be run in an IDE. The following describes how this can be done in IntelliJ IDEA and VS Code. It should work @@ -272,24 +272,24 @@ kustomize_substitutions: ### Running specific tests -To run a subset of tests, a combination of either one or both of `GINKGO_FOCUS` and `GINKGO_SKIP` env variables can be set. +To run a subset of tests the `GINKGO_LABEL_FILTER` env variable can be set. See [Ginkgo Spec Labels v2](https://onsi.github.io/ginkgo/MIGRATING_TO_V2#spec-labels) for complete syntax documentation. + Each of these can be used to match tests, for example: -- `[PR-Blocking]` => Sanity tests run before each PR merge -- `[K8s-Upgrade]` => Tests which verify k8s component version upgrades on workload clusters -- `[Conformance]` => Tests which run the k8s conformance suite on workload clusters -- `[ClusterClass]` => Tests which use a ClusterClass to create a workload cluster -- `When testing KCP.*` => Tests which start with `When testing KCP` +- `PR-Blocking` => Sanity tests run before each PR merge +- `K8s-Upgrade` => Tests which verify k8s component version upgrades on workload clusters +- `Conformance` => Tests which run the k8s conformance suite on workload clusters +- `ClusterClass` => Tests which use a ClusterClass to create a workload cluster +- `/When testing KCP.*/` => Tests which start with `When testing KCP` For example: -` GINKGO_FOCUS="\\[PR-Blocking\\]" make test-e2e ` can be used to run the sanity E2E tests -` GINKGO_SKIP="\\[K8s-Upgrade\\]" make test-e2e ` can be used to skip the upgrade E2E tests +` GINKGO_LABEL_FILTER="PR-Blocking" make test-e2e ` can be used to run the sanity E2E tests +` GINKGO_LABEL_FILTER="!K8s-Upgrade" make test-e2e ` can be used to skip the upgrade E2E tests ### Further customization The following env variables can be set to customize the test execution: -- `GINKGO_FOCUS` to set ginkgo focus (default empty - all tests) -- `GINKGO_SKIP` to set ginkgo skip (default empty - to allow running all tests) +- `GINKGO_LABEL_FILTER` to set ginkgo label filter (default empty - all tests) - `GINKGO_NODES` to set the number of ginkgo parallel nodes (default to 1) - `E2E_CONF_FILE` to set the e2e test config file (default to ${REPO_ROOT}/test/e2e/config/docker.yaml) - `ARTIFACTS` to set the folder where test artifact will be stored (default to ${REPO_ROOT}/_artifacts) diff --git a/test/infrastructure/inmemory/README.md b/test/infrastructure/inmemory/README.md index 5ecc0d6d207f..fcb6173340c6 100644 --- a/test/infrastructure/inmemory/README.md +++ b/test/infrastructure/inmemory/README.md @@ -126,7 +126,7 @@ kubectl --kubeconfig=/tmp/kubeconfig --server=https://127.0.0.1:$CONTROL_PLANE_E ### E2E tests CAPIM could be used to run a subset of CAPI E2E tests, but as of today we maintain only a smoke E2E scale test -(10 clusters, 1 CP and 3 workers each) that can be executed by setting `GINKGO_FOCUS="in-memory"`. +(10 clusters, 1 CP and 3 workers each) that can be executed by setting `GINKGO_LABEL_FILTER="in-memory"`. See [Running the end-to-end tests locally](https://cluster-api.sigs.k8s.io/developer/testing#running-the-end-to-end-tests-locally) for more details.