Fix e2e test grid by using ginkgo labels (#3451)

perdasilva · Per Goncalves da Silva · web-flow · commit 77ce6ff415bb · 2024-11-26T10:44:53.000+01:00
* Fix e2e gha num chunks

Signed-off-by: Per Goncalves da Silva &lt;pegoncal@redhat.com&gt;

* Refactor e2e split tool for labels

Signed-off-by: Per Goncalves da Silva &lt;pegoncal@redhat.com&gt;

* Add ginkgo labels to e2e tests

Signed-off-by: Per Goncalves da Silva &lt;pegoncal@redhat.com&gt;

---------

Signed-off-by: Per Goncalves da Silva &lt;pegoncal@redhat.com&gt;
Co-authored-by: Per Goncalves da Silva &lt;pegoncal@redhat.com&gt;
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
@@ -92,12 +92,12 @@ jobs:
 
       # run non-flakes if matrix-id is not 'flakes'
       - name: Run e2e tests
-        # calculate the number of chunks as the number of parallel jobs minus 1 (flakes job)
+        # calculate the number of chunks as the number of parallel jobs
         # use the split tool to split the test suite into chunks and run the chunk corresponding to the matrix-id
         # focus on those tests and skip tests marked as FLAKE
         run: |
-          E2E_TEST_NUM_CHUNKS=$(( ${{ strategy.job-total }} - 1 )) \
-          GINKGO_OPTS="${GINKGO_OPTS} -focus '$(go run ./test/e2e/split/... -chunks $E2E_TEST_NUM_CHUNKS -print-chunk $E2E_TEST_CHUNK ./test/e2e)' -skip '\[FLAKE\]'" \
+          E2E_TEST_NUM_CHUNKS=$(( ${{ strategy.job-total }} )) \
+          GINKGO_OPTS="${GINKGO_OPTS} -label-filter '$(go run ./test/e2e/split/... -chunks $E2E_TEST_NUM_CHUNKS -print-chunk $E2E_TEST_CHUNK ./test/e2e)' -skip '\[FLAKE\]'" \
           make e2e;
 
       # archive test results
diff --git a/test/e2e/bundle_e2e_test.go b/test/e2e/bundle_e2e_test.go
@@ -26,7 +26,7 @@ import (
 //go:embed testdata/vpa/crd.yaml
 var vpaCRDRaw []byte
 
-var _ = Describe("Installing bundles with new object types", func() {
+var _ = Describe("Installing bundles with new object types", Label("ObjectTypes"), func() {
 	var (
 		kubeClient         operatorclient.ClientInterface
 		operatorClient     versioned.Interface
diff --git a/test/e2e/catalog_e2e_test.go b/test/e2e/catalog_e2e_test.go
@@ -43,7 +43,7 @@ const (
 	badCSVDir             = "bad-csv"
 )
 
-var _ = Describe("Starting CatalogSource e2e tests", func() {
+var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), func() {
 	var (
 		generatedNamespace  corev1.Namespace
 		c                   operatorclient.ClientInterface
diff --git a/test/e2e/catalog_exclusion_test.go b/test/e2e/catalog_exclusion_test.go
@@ -19,7 +19,7 @@ import (
 
 const magicCatalogDir = "magiccatalog"
 
-var _ = Describe("Global Catalog Exclusion", func() {
+var _ = Describe("Global Catalog Exclusion", Label("CatalogExclusion"), func() {
 	var (
 		generatedNamespace  corev1.Namespace
 		determinedE2eClient *util.DeterminedE2EClient
diff --git a/test/e2e/catsrc_pod_config_e2e_test.go b/test/e2e/catsrc_pod_config_e2e_test.go
@@ -18,7 +18,7 @@ const catalogSourceLabel = "olm.catalogSource"
 
 var _ = By
 
-var _ = Describe("CatalogSource Grpc Pod Config", func() {
+var _ = Describe("CatalogSource Grpc Pod Config", Label("CatalogSourcePodConfig"), func() {
 
 	var (
 		generatedNamespace corev1.Namespace
diff --git a/test/e2e/crd_e2e_test.go b/test/e2e/crd_e2e_test.go
@@ -22,7 +22,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-var _ = Describe("CRD Versions", func() {
+var _ = Describe("CRD Versions", Label("CRDs"), func() {
 	var (
 		generatedNamespace corev1.Namespace
 		c                  operatorclient.ClientInterface
diff --git a/test/e2e/csv_e2e_test.go b/test/e2e/csv_e2e_test.go
@@ -34,7 +34,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("ClusterServiceVersion", func() {
+var _ = Describe("ClusterServiceVersion", Label("ClusterServiceVersion"), func() {
 	var (
 		generatedNamespace corev1.Namespace
 		c                  operatorclient.ClientInterface
diff --git a/test/e2e/deprecated_e2e_test.go b/test/e2e/deprecated_e2e_test.go
@@ -18,7 +18,7 @@ import (
 
 var missingAPI = `{"apiVersion":"verticalpodautoscalers.autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"name":"my.thing","namespace":"foo"}}`
 
-var _ = Describe("Not found APIs", func() {
+var _ = Describe("Not found APIs", Label("APIDeprecation"), func() {
 	var generatedNamespace corev1.Namespace
 
 	BeforeEach(func() {
diff --git a/test/e2e/disabling_copied_csv_e2e_test.go b/test/e2e/disabling_copied_csv_e2e_test.go
@@ -26,7 +26,7 @@ const (
 	protectedCopiedCSVNamespacesRuntimeFlag = "--protectedCopiedCSVNamespaces"
 )
 
-var _ = Describe("Disabling copied CSVs", func() {
+var _ = Describe("Disabling copied CSVs", Label("DisablingCopiedCSVs"), func() {
 	var (
 		generatedNamespace              corev1.Namespace
 		csv                             operatorsv1alpha1.ClusterServiceVersion
diff --git a/test/e2e/dynamic_resource_e2e_test.go b/test/e2e/dynamic_resource_e2e_test.go
@@ -20,7 +20,7 @@ import (
 // This test was disabled because both of its tests are currently being skipped
 // We need to understand why and whether this test is even needed:
 // https://github.com/operator-framework/operator-lifecycle-manager/issues/3402
-var _ = XDescribe("Subscriptions create required objects from Catalogs", func() {
+var _ = XDescribe("Subscriptions create required objects from Catalogs", Label("DynamicResource"), func() {
 	var (
 		crc                versioned.Interface
 		generatedNamespace corev1.Namespace
diff --git a/test/e2e/fail_forward_e2e_test.go b/test/e2e/fail_forward_e2e_test.go
@@ -123,7 +123,7 @@ func updateCatalogSource(namespace, name string, packages ...string) (func(), er
 	return deployCatalogSource(namespace, name, packages...)
 }
 
-var _ = Describe("Fail Forward Upgrades", func() {
+var _ = Describe("Fail Forward Upgrades", Label("FailForward"), func() {
 
 	var (
 		generatedNamespace corev1.Namespace
diff --git a/test/e2e/gc_e2e_test.go b/test/e2e/gc_e2e_test.go
@@ -23,7 +23,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("Garbage collection for dependent resources", func() {
+var _ = Describe("Garbage collection for dependent resources", Label("GarbageCollection"), func() {
 	var (
 		kubeClient         operatorclient.ClientInterface
 		operatorClient     versioned.Interface
diff --git a/test/e2e/installplan_e2e_test.go b/test/e2e/installplan_e2e_test.go
@@ -50,7 +50,7 @@ const (
 	deprecatedCRDDir = "deprecated-crd"
 )
 
-var _ = Describe("Install Plan", func() {
+var _ = Describe("Install Plan", Label("InstallPlan"), func() {
 	var (
 		c                  operatorclient.ClientInterface
 		crc                versioned.Interface
diff --git a/test/e2e/magic_catalog_test.go b/test/e2e/magic_catalog_test.go
@@ -13,7 +13,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-var _ = Describe("MagicCatalog", func() {
+var _ = Describe("MagicCatalog", Label("MagicCatalog"), func() {
 	var (
 		generatedNamespace corev1.Namespace
 		c                  client.Client
diff --git a/test/e2e/metrics_e2e_test.go b/test/e2e/metrics_e2e_test.go
@@ -27,7 +27,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("Metrics are generated for OLM managed resources", func() {
+var _ = Describe("Metrics are generated for OLM managed resources", Label("Metrics"), func() {
 	var (
 		c                  operatorclient.ClientInterface
 		crc                versioned.Interface
diff --git a/test/e2e/operator_condition_e2e_test.go b/test/e2e/operator_condition_e2e_test.go
@@ -19,7 +19,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("Operator Condition", func() {
+var _ = Describe("Operator Condition", Label("OperatorCondition"), func() {
 	var (
 		generatedNamespace corev1.Namespace
 	)
diff --git a/test/e2e/operator_groups_e2e_test.go b/test/e2e/operator_groups_e2e_test.go
@@ -35,7 +35,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("Operator Group", func() {
+var _ = Describe("Operator Group", Label("OperatorGroup"), func() {
 	var (
 		c                  operatorclient.ClientInterface
 		crc                versioned.Interface
diff --git a/test/e2e/operator_test.go b/test/e2e/operator_test.go
@@ -31,7 +31,7 @@ import (
 )
 
 // Describes test specs for the Operator resource.
-var _ = Describe("Operator API", func() {
+var _ = Describe("Operator API", Label("Operator"), func() {
 	var (
 		clientCtx       context.Context
 		scheme          *runtime.Scheme
diff --git a/test/e2e/packagemanifest_e2e_test.go b/test/e2e/packagemanifest_e2e_test.go
@@ -23,7 +23,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("Package Manifest API lists available Operators from Catalog Sources", func() {
+var _ = Describe("Package Manifest API lists available Operators from Catalog Sources", Label("PackageManifest"), func() {
 	var (
 		crc                versioned.Interface
 		pmc                pmversioned.Interface
diff --git a/test/e2e/resource_manager_test.go b/test/e2e/resource_manager_test.go
@@ -14,7 +14,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("ResourceManager", func() {
+var _ = Describe("ResourceManager", Label("ResourceManager"), func() {
 	var generatedNamespace corev1.Namespace
 
 	BeforeEach(func() {
diff --git a/test/e2e/scoped_client_test.go b/test/e2e/scoped_client_test.go
@@ -18,7 +18,7 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"
 )
 
-var _ = Describe("Scoped Client bound to a service account can be used to make API calls", func() {
+var _ = Describe("Scoped Client bound to a service account can be used to make API calls", Label("ScopedClient"), func() {
 	// TestScopedClient ensures that we can create a scoped client bound to a
 	// service account and then we can use the scoped client to make API calls.
 	var (
diff --git a/test/e2e/split/integration_test.sh b/test/e2e/split/integration_test.sh
@@ -5,8 +5,8 @@ function get_total_specs() {
 }
 
 unfocused_specs=$(get_total_specs)
-regexp=$(go run ./test/e2e/split/... -chunks 1 -print-chunk 0 ./test/e2e)
-focused_specs=$(get_total_specs -focus "$regexp")
+label_filter=$(go run ./test/e2e/split/... -chunks 1 -print-chunk 0 ./test/e2e)
+focused_specs=$(get_total_specs -label-filter "$label_filter")
 
 if ! [ $unfocused_specs -eq $focused_specs ]; then
   echo "expected number of unfocused specs $unfocused_specs to equal focus specs $focused_specs"
diff --git a/test/e2e/split/main.go b/test/e2e/split/main.go
diff --git a/test/e2e/split/main_test.go b/test/e2e/split/main_test.go
diff --git a/test/e2e/split/testdata/some_other_test.go b/test/e2e/split/testdata/some_other_test.go
diff --git a/test/e2e/split/testdata/some_test.go b/test/e2e/split/testdata/some_test.go
diff --git a/test/e2e/subscription_e2e_test.go b/test/e2e/subscription_e2e_test.go
diff --git a/test/e2e/user_defined_sa_test.go b/test/e2e/user_defined_sa_test.go
diff --git a/test/e2e/webhook_e2e_test.go b/test/e2e/webhook_e2e_test.go

Original file line number	Diff line number	Diff line change
`@@ -43,7 +43,7 @@ const (`
`43`	`43`	`badCSVDir = "bad-csv"`
`44`	`44`	`)`
`45`	`45`
`46`		`-var _ = Describe("Starting CatalogSource e2e tests", func() {`
	`46`	`+var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), func() {`
`47`	`47`	`var (`
`48`	`48`	`generatedNamespace corev1.Namespace`
`49`	`49`	`c operatorclient.ClientInterface`
Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ import (`
`22`	`22`	`metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"`
`23`	`23`	`)`
`24`	`24`
`25`		`-var _ = Describe("CRD Versions", func() {`
	`25`	`+var _ = Describe("CRD Versions", Label("CRDs"), func() {`
`26`	`26`	`var (`
`27`	`27`	`generatedNamespace corev1.Namespace`
`28`	`28`	`c operatorclient.ClientInterface`
Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@ import (`
`34`	`34`	`"github.com/operator-framework/operator-lifecycle-manager/test/e2e/ctx"`
`35`	`35`	`)`
`36`	`36`
`37`		`-var _ = Describe("ClusterServiceVersion", func() {`
	`37`	`+var _ = Describe("ClusterServiceVersion", Label("ClusterServiceVersion"), func() {`
`38`	`38`	`var (`
`39`	`39`	`generatedNamespace corev1.Namespace`
`40`	`40`	`c operatorclient.ClientInterface`
Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,7 @@ const (`
`26`	`26`	`protectedCopiedCSVNamespacesRuntimeFlag = "--protectedCopiedCSVNamespaces"`
`27`	`27`	`)`
`28`	`28`
`29`		`-var _ = Describe("Disabling copied CSVs", func() {`
	`29`	`+var _ = Describe("Disabling copied CSVs", Label("DisablingCopiedCSVs"), func() {`
`30`	`30`	`var (`
`31`	`31`	`generatedNamespace corev1.Namespace`
`32`	`32`	`csv operatorsv1alpha1.ClusterServiceVersion`