From 72ffef07ce89c047856eb9dc2a87d0c04b669170 Mon Sep 17 00:00:00 2001 From: Stephen Cahill Date: Mon, 20 Jan 2025 20:51:50 -0500 Subject: [PATCH 1/3] adopt e2e labels usage --- Makefile | 3 +- docs/book/src/developer/core/testing.md | 24 +++++------ scripts/ci-e2e-lib.sh | 54 +++++++++++++++++++++++++ scripts/ci-e2e.sh | 7 +++- test/infrastructure/inmemory/README.md | 2 +- 5 files changed, 75 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 71bc7dae4837..c2e2f29a715e 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,7 @@ export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) # GINKGO_FOCUS ?= GINKGO_SKIP ?= +GINKGO_LABEL_FILTER ?= GINKGO_NODES ?= 1 GINKGO_TIMEOUT ?= 2h GINKGO_POLL_PROGRESS_AFTER ?= 60m @@ -973,7 +974,7 @@ test-test-extension-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integra .PHONY: test-e2e test-e2e: $(GINKGO) generate-e2e-templates ## Run the end-to-end tests $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \ - -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" \ + -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" --label-filter="$(GINKGO_LABEL_FILTER)" \ $(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \ --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(ROOT_DIR)/$(TEST_DIR)/e2e -- \ -e2e.artifacts-folder="$(ARTIFACTS)" \ diff --git a/docs/book/src/developer/core/testing.md b/docs/book/src/developer/core/testing.md index 87d38667bd47..4d4d7a2c44a9 100644 --- a/docs/book/src/developer/core/testing.md +++ b/docs/book/src/developer/core/testing.md @@ -170,7 +170,7 @@ For example to run [pull-cluster-api-e2e-main](https://github.com/kubernetes/tes just execute: ```bash -GINKGO_FOCUS="\[PR-Blocking\]" ./scripts/ci-e2e.sh +GINKGO_LABEL_FILTER="PR-Blocking" ./scripts/ci-e2e.sh ``` ### Test execution via make test-e2e @@ -190,7 +190,7 @@ kind images). This can be done by executing the `./scripts/ci-e2e.sh` script. # Notes: # * You can cancel the script as soon as it starts the actual test execution via `make test-e2e`. # * If you want to run other tests (e.g. upgrade tests), make sure all required env variables are set (see the Prow Job config). -GINKGO_FOCUS="\[PR-Blocking\]" ./scripts/ci-e2e.sh +GINKGO_LABEL_FILTER="PR-Blocking" ./scripts/ci-e2e.sh ``` Now, the tests can be run in an IDE. The following describes how this can be done in IntelliJ IDEA and VS Code. It should work @@ -272,24 +272,24 @@ kustomize_substitutions: ### Running specific tests -To run a subset of tests, a combination of either one or both of `GINKGO_FOCUS` and `GINKGO_SKIP` env variables can be set. +To run a subset of tests the `GINKGO_LABEL_FILTER` env variable can be set. See [Ginkgo Spec Labels v2](https://onsi.github.io/ginkgo/MIGRATING_TO_V2#spec-labels) for complete syntax documentation. + Each of these can be used to match tests, for example: -- `[PR-Blocking]` => Sanity tests run before each PR merge -- `[K8s-Upgrade]` => Tests which verify k8s component version upgrades on workload clusters -- `[Conformance]` => Tests which run the k8s conformance suite on workload clusters -- `[ClusterClass]` => Tests which use a ClusterClass to create a workload cluster -- `When testing KCP.*` => Tests which start with `When testing KCP` +- `PR-Blocking` => Sanity tests run before each PR merge +- `K8s-Upgrade` => Tests which verify k8s component version upgrades on workload clusters +- `Conformance` => Tests which run the k8s conformance suite on workload clusters +- `ClusterClass` => Tests which use a ClusterClass to create a workload cluster +- `/When testing KCP.*/` => Tests which start with `When testing KCP` For example: -` GINKGO_FOCUS="\\[PR-Blocking\\]" make test-e2e ` can be used to run the sanity E2E tests -` GINKGO_SKIP="\\[K8s-Upgrade\\]" make test-e2e ` can be used to skip the upgrade E2E tests +` GINKGO_LABEL_FILTER="PR-Blocking" make test-e2e ` can be used to run the sanity E2E tests +` GINKGO_LABEL_FILTER="!K8s-Upgrade" make test-e2e ` can be used to skip the upgrade E2E tests ### Further customization The following env variables can be set to customize the test execution: -- `GINKGO_FOCUS` to set ginkgo focus (default empty - all tests) -- `GINKGO_SKIP` to set ginkgo skip (default empty - to allow running all tests) +- `GINKGO_LABEL_FILTER` to set ginkgo label filter (default empty - all tests) - `GINKGO_NODES` to set the number of ginkgo parallel nodes (default to 1) - `E2E_CONF_FILE` to set the e2e test config file (default to ${REPO_ROOT}/test/e2e/config/docker.yaml) - `ARTIFACTS` to set the folder where test artifact will be stored (default to ${REPO_ROOT}/_artifacts) diff --git a/scripts/ci-e2e-lib.sh b/scripts/ci-e2e-lib.sh index bdd94c1ed935..fcdd0aa9dd82 100644 --- a/scripts/ci-e2e-lib.sh +++ b/scripts/ci-e2e-lib.sh @@ -34,6 +34,60 @@ capi:buildDockerImages () { fi } +# k8s::prepareKindestImagesVariables defaults the environment variables KUBERNETES_VERSION_MANAGEMENT, KUBERNETES_VERSION, +# KUBERNETES_VERSION_UPGRADE_TO, KUBERNETES_VERSION_UPGRADE_FROM and KUBERNETES_VERSION_LATEST_CI +# depending on what is set in GINKGO_LABEL_FILTER. +# Note: We do this to ensure that the kindest/node image gets built if it does +# not already exist, e.g. for pre-releases, but only if necessary. +k8s::prepareKindestImagesVariablesGingkoFilters() { + # Always default KUBERNETES_VERSION_MANAGEMENT because we always create a management cluster out of it. + if [[ -z "${KUBERNETES_VERSION_MANAGEMENT:-}" ]]; then + KUBERNETES_VERSION_MANAGEMENT=$(grep KUBERNETES_VERSION_MANAGEMENT: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}') + echo "Defaulting KUBERNETES_VERSION_MANAGEMENT to ${KUBERNETES_VERSION_MANAGEMENT} to trigger image build (because env var is not set)" + fi + + if [[ ${GINKGO_LABEL_FILTER:-} == *"K8s-Install-ci-latest"* ]]; then + # If the test focuses on [K8s-Install-ci-latest], only default KUBERNETES_VERSION_LATEST_CI + # to the value in the e2e config and only if it is not set. + # Note: We do this because we want to specify KUBERNETES_VERSION_LATEST_CI *only* in the e2e config. + if [[ -z "${KUBERNETES_VERSION_LATEST_CI:-}" ]]; then + KUBERNETES_VERSION_LATEST_CI=$(grep KUBERNETES_VERSION_LATEST_CI: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}') + echo "Defaulting KUBERNETES_VERSION_LATEST_CI to ${KUBERNETES_VERSION_LATEST_CI} to trigger image build (because env var is not set)" + fi + elif [[ ${GINKGO_LABEL_FILTER:-} != *"K8s-Upgrade"* ]]; then + # In any other case which is not [K8s-Upgrade], default KUBERNETES_VERSION if it is not set to make sure + # the corresponding kindest/node image exists. + if [[ -z "${KUBERNETES_VERSION:-}" ]]; then + KUBERNETES_VERSION=$(grep KUBERNETES_VERSION: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}') + echo "Defaulting KUBERNETES_VERSION to ${KUBERNETES_VERSION} to trigger image build (because env var is not set)" + fi + fi + + # Tests not focusing on anything and skipping [Conformance] run a clusterctl upgrade test + # on the latest kubernetes version as management cluster. + if [[ ${GINKGO_LABEL_FILTER} =~ ^!Conformance$ ]]; then + # Note: We do this because we want to specify KUBERNETES_VERSION_LATEST_CI *only* in the e2e config. + if [[ -z "${KUBERNETES_VERSION_LATEST_CI:-}" ]]; then + KUBERNETES_VERSION_LATEST_CI=$(grep KUBERNETES_VERSION_LATEST_CI: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}') + echo "Defaulting KUBERNETES_VERSION_LATEST_CI to ${KUBERNETES_VERSION_LATEST_CI} to trigger image build (because env var is not set)" + fi + fi + + # Tests not focusing on [PR-Blocking], [K8s-Install] or [K8s-Install-ci-latest], + # also run upgrade tests so default KUBERNETES_VERSION_UPGRADE_TO and KUBERNETES_VERSION_UPGRADE_FROM + # to the value in the e2e config if they are not set. + if [[ ${GINKGO_LABEL_FILTER:-} != *"PR-Blocking"* ]] && [[ ${GINKGO_LABEL_FILTER:-} != *"K8s-Install"* ]]; then + if [[ -z "${KUBERNETES_VERSION_UPGRADE_TO:-}" ]]; then + KUBERNETES_VERSION_UPGRADE_TO=$(grep KUBERNETES_VERSION_UPGRADE_TO: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}') + echo "Defaulting KUBERNETES_VERSION_UPGRADE_TO to ${KUBERNETES_VERSION_UPGRADE_TO} to trigger image build (because env var is not set)" + fi + if [[ -z "${KUBERNETES_VERSION_UPGRADE_FROM:-}" ]]; then + KUBERNETES_VERSION_UPGRADE_FROM=$(grep KUBERNETES_VERSION_UPGRADE_FROM: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}') + echo "Defaulting KUBERNETES_VERSION_UPGRADE_FROM to ${KUBERNETES_VERSION_UPGRADE_FROM} to trigger image build (because env var is not set)" + fi + fi +} + # k8s::prepareKindestImagesVariables defaults the environment variables KUBERNETES_VERSION_MANAGEMENT, KUBERNETES_VERSION, # KUBERNETES_VERSION_UPGRADE_TO, KUBERNETES_VERSION_UPGRADE_FROM and KUBERNETES_VERSION_LATEST_CI # depending on what is set in GINKGO_FOCUS. diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index ca40d90b90c1..1be9aced7e6e 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -53,7 +53,12 @@ export USE_EXISTING_CLUSTER=false # - KUBERNETES_VERSION_UPGRADE_FROM # - KUBERNETES_VERSION_LATEST_CI # - KUBERNETES_VERSION_MANAGEMENT -k8s::prepareKindestImagesVariables +if [[ -n "${GINKGO_LABEL_FILTER}" ]]; then + echo "Preparing kindest/node images for Ginkgo label filter: ${GINKGO_LABEL_FILTER}" + k8s::prepareKindestImagesVariablesGingkoFilters +else + k8s::prepareKindestImagesVariables +fi k8s::prepareKindestImages # pre-pull all the images that will be used in the e2e, thus making the actual test run diff --git a/test/infrastructure/inmemory/README.md b/test/infrastructure/inmemory/README.md index 5ecc0d6d207f..fcb6173340c6 100644 --- a/test/infrastructure/inmemory/README.md +++ b/test/infrastructure/inmemory/README.md @@ -126,7 +126,7 @@ kubectl --kubeconfig=/tmp/kubeconfig --server=https://127.0.0.1:$CONTROL_PLANE_E ### E2E tests CAPIM could be used to run a subset of CAPI E2E tests, but as of today we maintain only a smoke E2E scale test -(10 clusters, 1 CP and 3 workers each) that can be executed by setting `GINKGO_FOCUS="in-memory"`. +(10 clusters, 1 CP and 3 workers each) that can be executed by setting `GINKGO_LABEL_FILTER="in-memory"`. See [Running the end-to-end tests locally](https://cluster-api.sigs.k8s.io/developer/testing#running-the-end-to-end-tests-locally) for more details. From c09e6f18c4f417bd585b73196ef120f39c4f0874 Mon Sep 17 00:00:00 2001 From: Stephen Cahill Date: Mon, 27 Jan 2025 20:30:55 -0500 Subject: [PATCH 2/3] fix unbound variable --- scripts/ci-e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index 1be9aced7e6e..c521b3176e27 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -53,7 +53,7 @@ export USE_EXISTING_CLUSTER=false # - KUBERNETES_VERSION_UPGRADE_FROM # - KUBERNETES_VERSION_LATEST_CI # - KUBERNETES_VERSION_MANAGEMENT -if [[ -n "${GINKGO_LABEL_FILTER}" ]]; then +if [[ -n "${GINKGO_LABEL_FILTER+x}" && -n "${GINKGO_LABEL_FILTER}" ]]; then echo "Preparing kindest/node images for Ginkgo label filter: ${GINKGO_LABEL_FILTER}" k8s::prepareKindestImagesVariablesGingkoFilters else From 7d528c94ef5c302da0a3c374f2ea06ab0c5c7e75 Mon Sep 17 00:00:00 2001 From: Stephen Cahill Date: Mon, 3 Feb 2025 18:40:52 -0500 Subject: [PATCH 3/3] revert script changes --- scripts/ci-e2e-lib.sh | 54 ------------------------------------------- scripts/ci-e2e.sh | 7 +----- 2 files changed, 1 insertion(+), 60 deletions(-) diff --git a/scripts/ci-e2e-lib.sh b/scripts/ci-e2e-lib.sh index fcdd0aa9dd82..bdd94c1ed935 100644 --- a/scripts/ci-e2e-lib.sh +++ b/scripts/ci-e2e-lib.sh @@ -34,60 +34,6 @@ capi:buildDockerImages () { fi } -# k8s::prepareKindestImagesVariables defaults the environment variables KUBERNETES_VERSION_MANAGEMENT, KUBERNETES_VERSION, -# KUBERNETES_VERSION_UPGRADE_TO, KUBERNETES_VERSION_UPGRADE_FROM and KUBERNETES_VERSION_LATEST_CI -# depending on what is set in GINKGO_LABEL_FILTER. -# Note: We do this to ensure that the kindest/node image gets built if it does -# not already exist, e.g. for pre-releases, but only if necessary. -k8s::prepareKindestImagesVariablesGingkoFilters() { - # Always default KUBERNETES_VERSION_MANAGEMENT because we always create a management cluster out of it. - if [[ -z "${KUBERNETES_VERSION_MANAGEMENT:-}" ]]; then - KUBERNETES_VERSION_MANAGEMENT=$(grep KUBERNETES_VERSION_MANAGEMENT: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}') - echo "Defaulting KUBERNETES_VERSION_MANAGEMENT to ${KUBERNETES_VERSION_MANAGEMENT} to trigger image build (because env var is not set)" - fi - - if [[ ${GINKGO_LABEL_FILTER:-} == *"K8s-Install-ci-latest"* ]]; then - # If the test focuses on [K8s-Install-ci-latest], only default KUBERNETES_VERSION_LATEST_CI - # to the value in the e2e config and only if it is not set. - # Note: We do this because we want to specify KUBERNETES_VERSION_LATEST_CI *only* in the e2e config. - if [[ -z "${KUBERNETES_VERSION_LATEST_CI:-}" ]]; then - KUBERNETES_VERSION_LATEST_CI=$(grep KUBERNETES_VERSION_LATEST_CI: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}') - echo "Defaulting KUBERNETES_VERSION_LATEST_CI to ${KUBERNETES_VERSION_LATEST_CI} to trigger image build (because env var is not set)" - fi - elif [[ ${GINKGO_LABEL_FILTER:-} != *"K8s-Upgrade"* ]]; then - # In any other case which is not [K8s-Upgrade], default KUBERNETES_VERSION if it is not set to make sure - # the corresponding kindest/node image exists. - if [[ -z "${KUBERNETES_VERSION:-}" ]]; then - KUBERNETES_VERSION=$(grep KUBERNETES_VERSION: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}') - echo "Defaulting KUBERNETES_VERSION to ${KUBERNETES_VERSION} to trigger image build (because env var is not set)" - fi - fi - - # Tests not focusing on anything and skipping [Conformance] run a clusterctl upgrade test - # on the latest kubernetes version as management cluster. - if [[ ${GINKGO_LABEL_FILTER} =~ ^!Conformance$ ]]; then - # Note: We do this because we want to specify KUBERNETES_VERSION_LATEST_CI *only* in the e2e config. - if [[ -z "${KUBERNETES_VERSION_LATEST_CI:-}" ]]; then - KUBERNETES_VERSION_LATEST_CI=$(grep KUBERNETES_VERSION_LATEST_CI: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}') - echo "Defaulting KUBERNETES_VERSION_LATEST_CI to ${KUBERNETES_VERSION_LATEST_CI} to trigger image build (because env var is not set)" - fi - fi - - # Tests not focusing on [PR-Blocking], [K8s-Install] or [K8s-Install-ci-latest], - # also run upgrade tests so default KUBERNETES_VERSION_UPGRADE_TO and KUBERNETES_VERSION_UPGRADE_FROM - # to the value in the e2e config if they are not set. - if [[ ${GINKGO_LABEL_FILTER:-} != *"PR-Blocking"* ]] && [[ ${GINKGO_LABEL_FILTER:-} != *"K8s-Install"* ]]; then - if [[ -z "${KUBERNETES_VERSION_UPGRADE_TO:-}" ]]; then - KUBERNETES_VERSION_UPGRADE_TO=$(grep KUBERNETES_VERSION_UPGRADE_TO: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}') - echo "Defaulting KUBERNETES_VERSION_UPGRADE_TO to ${KUBERNETES_VERSION_UPGRADE_TO} to trigger image build (because env var is not set)" - fi - if [[ -z "${KUBERNETES_VERSION_UPGRADE_FROM:-}" ]]; then - KUBERNETES_VERSION_UPGRADE_FROM=$(grep KUBERNETES_VERSION_UPGRADE_FROM: < "$E2E_CONF_FILE" | awk -F'"' '{ print $2}') - echo "Defaulting KUBERNETES_VERSION_UPGRADE_FROM to ${KUBERNETES_VERSION_UPGRADE_FROM} to trigger image build (because env var is not set)" - fi - fi -} - # k8s::prepareKindestImagesVariables defaults the environment variables KUBERNETES_VERSION_MANAGEMENT, KUBERNETES_VERSION, # KUBERNETES_VERSION_UPGRADE_TO, KUBERNETES_VERSION_UPGRADE_FROM and KUBERNETES_VERSION_LATEST_CI # depending on what is set in GINKGO_FOCUS. diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index c521b3176e27..ca40d90b90c1 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -53,12 +53,7 @@ export USE_EXISTING_CLUSTER=false # - KUBERNETES_VERSION_UPGRADE_FROM # - KUBERNETES_VERSION_LATEST_CI # - KUBERNETES_VERSION_MANAGEMENT -if [[ -n "${GINKGO_LABEL_FILTER+x}" && -n "${GINKGO_LABEL_FILTER}" ]]; then - echo "Preparing kindest/node images for Ginkgo label filter: ${GINKGO_LABEL_FILTER}" - k8s::prepareKindestImagesVariablesGingkoFilters -else - k8s::prepareKindestImagesVariables -fi +k8s::prepareKindestImagesVariables k8s::prepareKindestImages # pre-pull all the images that will be used in the e2e, thus making the actual test run