Merge pull request #1233 from Bowenislandsong/e2e_metric

adding e2e test framework for rh-operator metrics

Author: openshift-merge-robot

Makefile

@@ -107,8 +107,9 @@ setup-bare: clean e2e.namespace
 	. ./scripts/package_release.sh 1.0.0 test/e2e/resources test/e2e/e2e-bare-values.yaml
 	. ./scripts/install_bare.sh $(shell cat ./e2e.namespace) test/e2e/resources
 
+# e2e test exculding the rh-operators directory which tests rh-operators and their metric cardinality.
 e2e:
-	go test -v $(MOD_FLAGS) -failfast -timeout 70m ./test/e2e/... -namespace=openshift-operators -kubeconfig=${KUBECONFIG} -olmNamespace=openshift-operator-lifecycle-manager -dummyImage=bitnami/nginx:latest
+	go test -v $(MOD_FLAGS) -failfast -timeout 70m `go list ./test/e2e/... | grep -v /rh-operators` -namespace=openshift-operators -kubeconfig=${KUBECONFIG} -olmNamespace=openshift-operator-lifecycle-manager -dummyImage=bitnami/nginx:latest
 
 e2e-local: build-linux build-wait
 	. ./scripts/build_local.sh
@@ -121,6 +122,9 @@ e2e-local-docker:
 	. ./scripts/build_local.sh
 	. ./scripts/run_e2e_docker.sh $(TEST)
 
+e2e-operator-metrics:
+	go test -v $(MOD_FLAGS) -failfast -timeout 70m ./test/e2e/rh-operators
+
 vendor:
 	go mod tidy
 	go mod vendor

test/e2e/rh-operators/metric_test.go

@@ -0,0 +1,88 @@
+package rh_operators
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
+	pclient "github.com/operator-framework/operator-lifecycle-manager/pkg/package-server/client"
+	psVersioned "github.com/operator-framework/operator-lifecycle-manager/pkg/package-server/client/clientset/versioned"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	kubeconfig = os.Getenv("KUBECONFIG")
+	testlog    = logrus.New()
+)
+
+// TestRHOperators installs all Red Hat Operators available on the cluster and see that they install successfully.
+func TestRHOperators(t *testing.T) {
+	c := operatorclient.NewClientFromConfig(kubeconfig, testlog)
+
+	crc, err := pclient.NewClient(kubeconfig)
+	require.NoError(t, err, "error creating package-server client")
+
+	vc, err := client.NewClient(kubeconfig)
+	require.NoError(t, err, "error creating cr client")
+
+	// Serial Test operators with installModes other than typeOwnNamespace to avoid OperatorGroup intersection.
+	operatorList, err := ListRHOperatorsWithoutInstallModes(crc, v1alpha1.InstallModeTypeOwnNamespace)
+	require.NoError(t, err, "failed to list Red Hat Operators")
+
+	for _, operator := range operatorList {
+		operatorName := operator
+		t.Run(fmt.Sprintf("Testing for operator %s", operator), func(t *testing.T) {
+			testInstallOperators(t, c, crc, vc, operatorName)
+		})
+	}
+
+	CleanupAll(t, c, vc)
+
+	operatorList, err = ListRHOperatorsByInstallModes(crc, v1alpha1.InstallModeTypeOwnNamespace)
+	require.NoError(t, err, "failed to list Red Hat Operators with OwnNamespace InstallMode")
+
+	for _, operator := range operatorList {
+		operatorName := operator
+		t.Run(fmt.Sprintf("Testing for operator %s", operator), func(t *testing.T) {
+			t.Parallel()
+			testInstallOperators(t, c, crc, vc, operatorName)
+		})
+	}
+
+	CleanupAll(t, c, vc)
+}
+
+// testInstallOperators installs operators and wait till its CSV is in succeeded phase then deletes created resources
+// and wait till the created Namespace is terminated.
+func testInstallOperators(t *testing.T, c operatorclient.ClientInterface, crc psVersioned.Interface,
+	vc versioned.Interface, operatorName string) {
+	t.Logf("Installing %s operator", operatorName)
+
+	o, err := NewOperator(c, crc, vc, operatorName)
+	if err != nil {
+		assert.Failf(t, err.Error(), "failed to load operator %s from packagemanifests", operatorName)
+		return
+	}
+
+	err = o.Subscribe()
+	assert.NoError(t, err, "error creating subscription for operator %s", operatorName)
+
+	err = o.Unsubscribe(false)
+	assert.NoError(t, err, "error cleaning up subscription for operator %s", operatorName)
+	if err != nil {
+		CleanupOperatorCSVs(t, o)
+		CleanupOperatorNamespace(t, o)
+	}
+
+	err = o.WaitToDeleteNamespace(o.namespace)
+	if err != nil {
+		t.Logf("error cleaning up Namespace: %s, %v", o.namespace, err)
+	}
+}