remove annotations and replace with environment selector based test skipping and disablement

remove the render_test_report command as it is no longer used, and update vendor files

add suite to the name

put annotation framework back for k8s tests only

don't add suite info to excluded tests

log new test name for storage test rename purposes

exclude suite names from External Storage tests

remove util/annotate

separate out suite appending logic to helper function

deps

Author: smg247

README.md

@@ -37,34 +37,29 @@ will switch to vendoring origin-specific branches (e.g
 carries that need to be considered in the context of
 `openshift/kubernetes` rebases.
 
-### Test annotation rules
+### Test exclusion rules
 
-Test annotation rules are used to label e2e tests so that they can be
-filtered or skipped. For example, rules can be defined that match kube
-e2e tests that are known to be incompatible with openshift and label
-those tests to be skipped.
+Test exclusion is now handled through environmental selector based filtering
+rather than test annotations. Environmental selectors allow tests to be
+filtered or skipped based on the cluster environment and configuration.
+For example, selectors can be defined that match kube e2e tests that are
+known to be incompatible with specific OpenShift configurations and exclude
+those tests from running.
 
-Maintenance of test annotation rules is split between the
+Maintenance of test exclusion rules is split between the
 `openshift/kubernetes` and `origin` repos to ensure that PRs proposed
 to `openshift/kubernetes` can be validated against the set of kube e2e
-tests known to be compatible with openshift.
+tests known to be compatible with OpenShift.
 
-Test annotation rules for kubernetes e2e tests are maintained in:
+Test exclusion rules for kubernetes e2e tests are maintained in: https://github.com/openshift/kubernetes/blob/master/openshift-hack/cmd/k8s-tests-ext:
+* [environment_selectors.go](https://github.com/openshift/kubernetes/blob/master/openshift-hack/cmd/k8s-tests-ext/environment_selectors.go)
+* [disabled_tests.go](https://github.com/openshift/kubernetes/blob/master/openshift-hack/cmd/k8s-tests-ext/disabled_tests.go)
 
-https://github.com/openshift/kubernetes/blob/master/openshift-hack/e2e/annotate/rules.go
+Test exclusion rules for openshift e2e tests are maintained in: https://github.com/openshift/origin/blob/main/pkg/test/extensions:
+* [environment_selectors.go](https://github.com/openshift/origin/blob/main/pkg/test/extensions/environment_selectors.go)
+* [disabled_tests.go](https://github.com/openshift/origin/blob/main/pkg/test/extensions/disabled_tests.go)
 
-Test annotation rules for openshift e2e tests are maintained in:
-
-https://github.com/openshift/origin/blob/main/test/extended/util/annotate/rules.go
-
-Origin vendors the kube rules and applies both the kube and openshift
-rules to the set of tests included in the `openshift-tests` binary.
-
-In order to update test annotation rules for kube e2e tests, it will
-be necessary to:
-
- - Update `rules.go` in `openshift/kubernetes`
- - Bump the version of `openshift/kubernetes` vendored in origin
+To update test exclusion rules for kube e2e tests, update the environmental selectors in `openshift/kubernetes`. For OpenShift e2e tests, update the selectors in `origin`.
 
 ### Vendoring from `openshift/kubernetes`
 

cmd/openshift-tests/openshift-tests.go

@@ -28,7 +28,6 @@ import (
 	"github.com/openshift/origin/pkg/cmd/openshift-tests/monitor"
 	run_monitor "github.com/openshift/origin/pkg/cmd/openshift-tests/monitor/run"
 	"github.com/openshift/origin/pkg/cmd/openshift-tests/monitor/timeline"
-	"github.com/openshift/origin/pkg/cmd/openshift-tests/render"
 	risk_analysis "github.com/openshift/origin/pkg/cmd/openshift-tests/risk-analysis"
 	"github.com/openshift/origin/pkg/cmd/openshift-tests/run"
 	run_disruption "github.com/openshift/origin/pkg/cmd/openshift-tests/run-disruption"
@@ -106,7 +105,6 @@ func main() {
 		timeline.NewTimelineCommand(ioStreams),
 		run_disruption.NewRunInClusterDisruptionMonitorCommand(ioStreams),
 		collectdiskcertificates.NewRunCollectDiskCertificatesCommand(ioStreams),
-		render.NewRenderCommand(ioStreams),
 		versioncmd.NewVersionCommand(ioStreams),
 	)
 

hack/update-generated.sh

@@ -7,11 +7,6 @@ source "$(dirname "${BASH_SOURCE}")/lib/init.sh"
 # https://github.com/kubernetes/kubernetes/pull/109541
 ENABLE_STORAGE_GCE_PD_DRIVER=yes go generate -mod vendor ./test/extended
 
-# the test names are now contained in test/extended/util/annotate/generated/zz_generated.annotations.go
-# A txt file would be more extensible, but Maciej stacked the debt here and he's the same person who will have pay it down
-# as we add more images with more tests, so this seems acceptable.
-go run -mod vendor ./cmd/openshift-tests render test-report --output-dir=./zz_generated.manifests
-
 # Update mirror mapping from upstream to quay
 # By default, "openshift-tests images" lists images from external binaries. However, we force
 # this script to list images from built-in tests in order to avoid requiring an OCP release image.

hack/verify-generated.sh

@@ -11,8 +11,6 @@ trap "cleanup" EXIT
 
 os::test::junit::declare_suite_start "verify/generated"
 os::cmd::expect_success "${OS_ROOT}/hack/update-generated.sh"
-os::cmd::expect_success "git diff --exit-code ${OS_ROOT}/test/extended/util/annotate/generated/"
 os::cmd::expect_success "git diff --exit-code ${OS_ROOT}/test/extended/util/image/zz_generated.txt"
-os::cmd::expect_success "git diff --exit-code ${OS_ROOT}/zz_generated.manifests"
 
 os::test::junit::declare_suite_end

images/tests/Dockerfile.rhel

@@ -7,8 +7,6 @@ RUN make; \
 
 FROM registry.ci.openshift.org/ocp/4.20:tools
 COPY --from=builder /tmp/build/openshift-tests /usr/bin/
-RUN mkdir -p /manifests
-COPY --from=builder /go/src/github.com/openshift/origin/zz_generated.manifests/* /manifests
 
 RUN PACKAGES="git gzip util-linux" && \
     if [ $HOSTTYPE = x86_64 ]; then PACKAGES="$PACKAGES python3-cinderclient"; fi && \

pkg/cmd/openshift-tests/render/render_command.go

@@ -16,11 +16,10 @@ import (
 	"syscall"
 	"time"
 
-	originVersion "github.com/openshift/origin/pkg/version"
-
 	"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
 	"github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
 	g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
+	originVersion "github.com/openshift/origin/pkg/version"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/mod/semver"
@@ -34,7 +33,6 @@ import (
 	"github.com/openshift/origin/pkg/clioptions/imagesetup"
 	"github.com/openshift/origin/pkg/clioptions/upgradeoptions"
 	exutil "github.com/openshift/origin/test/extended/util"
-	origingenerated "github.com/openshift/origin/test/extended/util/annotate/generated"
 	"github.com/openshift/origin/test/extended/util/image"
 )
 
@@ -78,24 +76,26 @@ func InitializeOpenShiftTestsExtensionFramework() (*extension.Registry, *extensi
 		return nil, nil, fmt.Errorf("failed to build extension test specs: %w", err)
 	}
 
-	// Apply annotations to test names
+	// Apply annotations to test names only for upstream tests
 	specs.Walk(func(spec *extensiontests.ExtensionTestSpec) {
-		// we need to ensure the default path always annotates both
-		// origin and k8s tests accordingly, since each of these
-		// currently have their own annotations which are not
-		// merged anywhere else but applied here
-		if append, ok := origingenerated.Annotations[spec.Name]; ok {
-			spec.Name += append
-		}
 		if append, ok := k8sgenerated.Annotations[spec.Name]; ok {
 			spec.Name += append
 		}
 	})
 
+	klog.Infof("Found %d test specs", len(specs))
 	// Filter out kube tests, vendor filtering isn't working within origin
 	specs = specs.Select(func(spec *extensiontests.ExtensionTestSpec) bool {
 		return !strings.Contains(spec.Name, "[Suite:k8s")
 	})
+	klog.Infof("%d test specs remain, after filtering out k8s", len(specs))
+
+	// Filter out tests that are always disabled based on name matching
+	specs = filterOutDisabledSpecs(specs)
+	// Add environment selectors that include or exclude tests in specific environments
+	addEnvironmentSelectors(specs)
+	addLabelsToSpecs(specs)
+	appendSuiteNames(specs)
 
 	specs.AddBeforeAll(func() {
 		config, err := clusterdiscovery.DecodeProvider(os.Getenv("TEST_PROVIDER"), false, false, nil)