Merge pull request #476 from ratnam915/feature/OSD-30036

OSD-30036: To create E2E Test for CAD - Cluster has gone missing - UpgradeConfigSyncFailureOver4Hr

Author: openshift-merge-bot[bot]

test/e2e/configuration_anomaly_detection_test.go

@@ -5,6 +5,7 @@ package osde2etests
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
@@ -20,7 +21,6 @@ import (
 	v1beta1 "github.com/openshift/api/machine/v1beta1"
 	awsinternal "github.com/openshift/configuration-anomaly-detection/pkg/aws"
 	machineutil "github.com/openshift/configuration-anomaly-detection/pkg/investigations/utils/machine"
-	"github.com/openshift/configuration-anomaly-detection/pkg/ocm"
 	"github.com/openshift/configuration-anomaly-detection/test/e2e/utils"
 	ocme2e "github.com/openshift/osde2e-common/pkg/clients/ocm"
 	"github.com/openshift/osde2e-common/pkg/clients/openshift"
@@ -36,7 +36,6 @@ import (
 var _ = Describe("Configuration Anomaly Detection", Ordered, func() {
 	var (
 		ocme2eCli    *ocme2e.Client
-		ocmCli       ocm.Client
 		k8s          *openshift.Client
 		region       string
 		provider     string
@@ -52,18 +51,13 @@ var _ = Describe("Configuration Anomaly Detection", Ordered, func() {
 		clientID := os.Getenv("CLIENT_ID")
 		clientSecret := os.Getenv("CLIENT_SECRET")
 		clusterID = os.Getenv("OCM_CLUSTER_ID")
-		cadOcmFilePath := os.Getenv("CAD_OCM_FILE_PATH")
 
 		Expect(ocmToken).NotTo(BeEmpty(), "OCM_TOKEN must be set")
 		Expect(clusterID).NotTo(BeEmpty(), "CLUSTER_ID must be set")
-		Expect(cadOcmFilePath).NotTo(BeEmpty(), "CAD_OCM_FILE_PATH must be set")
 
 		ocme2eCli, err = ocme2e.New(ctx, ocmToken, clientID, clientSecret, ocmEnv)
 		Expect(err).ShouldNot(HaveOccurred(), "Unable to setup E2E OCM Client")
 
-		ocmCli, err = ocm.New(cadOcmFilePath)
-		Expect(err).ShouldNot(HaveOccurred(), "Unable to setup ocm anomaly detection client")
-
 		k8s, err = openshift.New(ginkgo.GinkgoLogr)
 		Expect(err).ShouldNot(HaveOccurred(), "Unable to setup k8s client")
 
@@ -165,7 +159,7 @@ var _ = Describe("Configuration Anomaly Detection", Ordered, func() {
 			Expect(cluster).ToNot(BeNil(), "received nil cluster from OCM")
 
 			// Get service logs
-			logs, err := utils.GetServiceLogs(ocmCli, cluster)
+			logs, err := utils.GetServiceLogs(ocme2eCli, cluster)
 			Expect(err).ToNot(HaveOccurred(), "Failed to get service logs")
 			logsBefore := logs.Items().Slice()
 
@@ -228,7 +222,7 @@ var _ = Describe("Configuration Anomaly Detection", Ordered, func() {
 
 			time.Sleep(1 * time.Minute)
 
-			logs, err = utils.GetServiceLogs(ocmCli, cluster)
+			logs, err = utils.GetServiceLogs(ocme2eCli, cluster)
 			Expect(err).ToNot(HaveOccurred(), "Failed to get service logs")
 			logsAfter := logs.Items().Slice()
 
@@ -505,8 +499,8 @@ var _ = Describe("Configuration Anomaly Detection", Ordered, func() {
 			ginkgo.GinkgoWriter.Println("Step 7: Test completed: Node NotReady condition simulated and checked.")
 		}
 	})
-  
-  It("AWS CCS: clustermonitoringerrorbudgetburn", func(ctx context.Context) {
+
+	It("AWS CCS: clustermonitoringerrorbudgetburn", func(ctx context.Context) {
 		if provider == "aws" {
 			const (
 				namespace     = "openshift-user-workload-monitoring"
@@ -520,7 +514,7 @@ var _ = Describe("Configuration Anomaly Detection", Ordered, func() {
 			Expect(cluster).ToNot(BeNil(), "Cluster response is nil")
 
 			fmt.Println("Step 1: Getting service logs before misconfiguration")
-			logs, err := utils.GetServiceLogs(ocmCli, cluster)
+			logs, err := utils.GetServiceLogs(ocme2eCli, cluster)
 			Expect(err).ToNot(HaveOccurred(), "Failed to fetch service logs before misconfig")
 			logsBefore := logs.Items().Slice()
 
@@ -574,15 +568,15 @@ var _ = Describe("Configuration Anomaly Detection", Ordered, func() {
 			time.Sleep(2 * time.Minute)
 
 			fmt.Println("Step 5: Fetching service logs after misconfiguration")
-			logs, err = utils.GetServiceLogs(ocmCli, cluster)
+			logs, err = utils.GetServiceLogs(ocme2eCli, cluster)
 			Expect(err).ToNot(HaveOccurred(), "Failed to get service logs")
 			logsAfter := logs.Items().Slice()
 
 			Expect(logsAfter).To(HaveLen(len(logsBefore)), "Service logs count changed after scale down/up")
-    }
-  })
-  
-  It("AWS CCS: InsightsOperatorDown (blocked egress)", Label("aws", "ccs", "insights-operator", "blocking-egress"), func(ctx context.Context) {
+		}
+	})
+
+	It("AWS CCS: InsightsOperatorDown (blocked egress)", Label("aws", "ccs", "insights-operator", "blocking-egress"), func(ctx context.Context) {
 		if provider == "aws" {
 			awsAccessKey := os.Getenv("AWS_ACCESS_KEY_ID")
 			awsSecretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
@@ -616,7 +610,7 @@ var _ = Describe("Configuration Anomaly Detection", Ordered, func() {
 			Expect(err).NotTo(HaveOccurred(), "Failed to get security group ID")
 
 			// Step 1: Get logs before action
-			logsBefore, err := utils.GetServiceLogs(ocmCli, cluster)
+			logsBefore, err := utils.GetServiceLogs(ocme2eCli, cluster)
 			Expect(err).ToNot(HaveOccurred(), "Failed to get service logs before action")
 
 			existingLogIDs := map[string]bool{}
@@ -656,8 +650,10 @@ var _ = Describe("Configuration Anomaly Detection", Ordered, func() {
 			_, err = testPdClient.TriggerIncident("InsightsOperatorDown", clusterID)
 			Expect(err).NotTo(HaveOccurred(), "Failed to trigger silent PagerDuty alert")
 
+			time.Sleep(2 * time.Minute)
+
 			// Step 4: Get logs again and find new entries
-			logsAfter, err := utils.GetServiceLogs(ocmCli, cluster)
+			logsAfter, err := utils.GetServiceLogs(ocme2eCli, cluster)
 			Expect(err).ToNot(HaveOccurred(), "Failed to get service logs after action")
 
 			newLogs := []interface{}{}
@@ -672,4 +668,122 @@ var _ = Describe("Configuration Anomaly Detection", Ordered, func() {
 		}
 	})
 
+	It("UpgradeConfigSyncFailureOver4Hr: corrupted pull secret investigation", Label("pull-secret", "upgrade-config-sync", "user-banned-check"), func(ctx context.Context) {
+		// Get cluster information from OCM
+		response, err := ocme2eCli.ClustersMgmt().V1().Clusters().Cluster(clusterID).Get().Send()
+		Expect(err).ToNot(HaveOccurred(), "failed to get cluster from OCM")
+		cluster := response.Body()
+		Expect(cluster).ToNot(BeNil(), "received nil cluster from OCM")
+
+		lsResponseBefore, err := utils.GetLimitedSupportReasons(ocme2eCli, clusterID)
+		var lsReasonsBefore int
+		if err != nil {
+			ginkgo.GinkgoWriter.Printf("Could not get limited support reasons before test: %v\n", err)
+			lsReasonsBefore = 0
+		} else {
+			lsReasonsBefore = lsResponseBefore.Items().Len()
+			ginkgo.GinkgoWriter.Printf("Limited support reasons before pull secret corruption %d\n", lsReasonsBefore)
+		}
+
+		// Get the original pull secret for backup
+		var originalPullSecret corev1.Secret
+		err = k8s.Get(ctx, "pull-secret", "openshift-config", &originalPullSecret)
+		Expect(err).NotTo(HaveOccurred(), "Failed to get original pull secret")
+		ginkgo.GinkgoWriter.Print("Original pull secret retrieved successfully\n")
+
+		// Setup deferred restoration to ensure pull secret is restored regardless of test outcome
+		defer func() {
+			ginkgo.GinkgoWriter.Print("Restoring original pull secret...\n")
+			err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+				currentSecret := &corev1.Secret{}
+				err := k8s.Get(ctx, "pull-secret", "openshift-config", currentSecret)
+				if err != nil {
+					return err
+				}
+				// Restore original data
+				currentSecret.Data = originalPullSecret.Data
+				return k8s.Update(ctx, currentSecret)
+			})
+			if err != nil {
+				ginkgo.GinkgoWriter.Print("Failed to restore original pull secret: %v\n", err)
+			} else {
+				ginkgo.GinkgoWriter.Print("Original pull secret restored successfully\n")
+			}
+		}()
+
+		// Corrupt the pull secret to simulate the UpgradeConfigSyncFailure scenario
+		ginkgo.GinkgoWriter.Print("Corrupting pull secret to simulate sync failure...\n")
+		err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			pullSecret := &corev1.Secret{}
+			err := k8s.Get(ctx, "pull-secret", "openshift-config", pullSecret)
+			if err != nil {
+				return err
+			}
+
+			// Create a corrupted docker config json
+			corruptedConfig := map[string]interface{}{
+				"auths": map[string]interface{}{
+					"cloud.openshift.com": map[string]interface{}{
+						"auth":  "Y29ycnVwdGVkX3Rva2VuOmNvcnJ1cHRlZF9wYXNzd29yZA==",
+						"email": "[email protected]",
+					},
+					"registry.connect.redhat.com": map[string]interface{}{
+						"auth":  "Y29ycnVwdGVkX3Rva2VuOmNvcnJ1cHRlZF9wYXNzd29yZA==",
+						"email": "[email protected]",
+					},
+				},
+			}
+
+			corruptedConfigBytes, err := json.Marshal(corruptedConfig)
+			if err != nil {
+				return err
+			}
+
+			// Update the pull secret with corrupted data
+			pullSecret.Data[".dockerconfigjson"] = corruptedConfigBytes
+			return k8s.Update(ctx, pullSecret)
+		})
+		Expect(err).NotTo(HaveOccurred(), "Failed to corrupt pull secret")
+		ginkgo.GinkgoWriter.Print("Pull secret corrupted successfully\n")
+
+		// Trigger the UpgradeConfigSyncFailureOver4Hr alert
+		_, err = testPdClient.TriggerIncident("UpgradeConfigSyncFailureOver4HrSRE", clusterID)
+		Expect(err).NotTo(HaveOccurred(), "Failed to trigger UpgradeConfigSyncFailureOver4Hr PagerDuty alert")
+
+		// Wait for the investigation to process
+		ginkgo.GinkgoWriter.Print("Waiting for investigation to process corrupted pull secret...\n")
+		time.Sleep(2 * time.Minute)
+
+		// Get limited support reasons after corruption
+		lsResponseAfter, err := utils.GetLimitedSupportReasons(ocme2eCli, clusterID)
+		if err != nil {
+			ginkgo.GinkgoWriter.Printf("Could not get limited support reasons after test: %v\n", err)
+		} else {
+			// Print the response data
+			fmt.Println("Limited Support Response After Pull Secret Corruption:")
+			fmt.Printf("Total items: %d\n", lsResponseAfter.Items().Len())
+
+			// Iterate through each item and print details
+			items := lsResponseAfter.Items().Slice()
+			for i, item := range items {
+				fmt.Printf("Reason #%d:\n", i+1)
+				fmt.Printf("  - Summary: %s\n", item.Summary())
+				fmt.Printf("  - Details: %s\n", item.Details())
+			}
+
+			// Compare with before if we had baseline data
+			if lsReasonsBefore >= 0 {
+				if lsResponseAfter.Items().Len() > lsReasonsBefore {
+					ginkgo.GinkgoWriter.Printf("Limited support reasons increased from %d to %d\n",
+						lsReasonsBefore, lsResponseAfter.Items().Len())
+				} else {
+					ginkgo.GinkgoWriter.Printf("Limited support reasons remained at %d\n",
+						lsResponseAfter.Items().Len())
+				}
+			}
+		}
+
+		fmt.Println("Test completed: UpgradeConfigSyncFailureOver4Hr investigation simulated successfully")
+	})
+
 }, ginkgo.ContinueOnFailure)

test/e2e/utils/generate_incident.go

@@ -16,6 +16,7 @@ const (
 	AlertInsightsOperatorDown                          = "InsightsOperatorDown"
 	AlertMachineHealthCheckUnterminatedShortCircuitSRE = "MachineHealthCheckUnterminatedShortCircuitSRE"
 	AlertApiErrorBudgetBurn                            = "ApiErrorBudgetBurn"
+	UpgradeConfigSyncFailureOver4HrSRE                 = "UpgradeConfigSyncFailureOver4HrSRE"
 )
 
 func GetAlertTitle(alertName string) (string, error) {
@@ -32,6 +33,8 @@ func GetAlertTitle(alertName string) (string, error) {
 		return "MachineHealthCheckUnterminatedShortCircuitSRE CRITICAL (1)", nil
 	case AlertApiErrorBudgetBurn:
 		return "api-ErrorBudgetBurn k8sgpt test CRITICAL (1)", nil
+	case UpgradeConfigSyncFailureOver4HrSRE:
+		return "UpgradeConfigSyncFailureOver4HrSRE Critical (1)", nil
 	default:
 		return "", fmt.Errorf("unknown alert name: %s", alertName)
 	}

test/e2e/utils/utils.go

@@ -6,8 +6,11 @@ package utils
 import (
 	"fmt"
 
+	sdk "github.com/openshift-online/ocm-sdk-go"
+	amv1 "github.com/openshift-online/ocm-sdk-go/accountsmgmt/v1"
 	cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1"
 	servicelogsv1 "github.com/openshift-online/ocm-sdk-go/servicelogs/v1"
+	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
 	"github.com/openshift/configuration-anomaly-detection/pkg/ocm"
 	ocme2e "github.com/openshift/osde2e-common/pkg/clients/ocm"
 	"k8s.io/client-go/tools/clientcmd"
@@ -16,24 +19,91 @@ import (
 
 func GetLimitedSupportReasons(ocme2eCli *ocme2e.Client, clusterID string) (*cmv1.LimitedSupportReasonsListResponse, error) {
 	lsResponse, err := ocme2eCli.ClustersMgmt().V1().Clusters().Cluster(clusterID).LimitedSupportReasons().List().Send()
-
 	if err != nil {
 		return nil, fmt.Errorf("failed sending service log: %w", err)
 	}
 	return lsResponse, nil
 }
 
-func GetServiceLogs(ocmCli ocm.Client, cluster *cmv1.Cluster) (*servicelogsv1.ClusterLogsUUIDListResponse, error) {
-	filter := "log_type='cluster-state-updates'"
-	clusterLogsUUIDListResponse, err := ocmCli.GetServiceLog(cluster, filter)
+func GetServiceLogs(ocmLike interface{}, cluster *cmv1.Cluster) (*servicelogsv1.ClusterLogsUUIDListResponse, error) {
+	const filter = "log_type='cluster-state-updates'"
+
+	switch v := ocmLike.(type) {
+	case ocm.Client:
+		clusterLogsUUIDListResponse, err := v.GetServiceLog(cluster, filter)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to get service log: %w", err)
+		}
+		return clusterLogsUUIDListResponse, nil
+	case *ocme2e.Client:
+		adapter := &e2eOCMAdapter{conn: v.Connection}
+		clusterLogsUUIDListResponse, err := adapter.GetServiceLog(cluster, filter)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to get service log (via adapter): %w", err)
+		}
+		return clusterLogsUUIDListResponse, nil
+	default:
+		return nil, fmt.Errorf("unsupported type for GetServiceLogs: %T", v)
+	}
+}
+
+type e2eOCMAdapter struct {
+	conn *sdk.Connection
+}
+
+func (a *e2eOCMAdapter) GetServiceLog(cluster *cmv1.Cluster, filter string) (*servicelogsv1.ClusterLogsUUIDListResponse, error) {
+	if filter != "" {
+		return a.conn.ServiceLogs().V1().Clusters().Cluster(cluster.ExternalID()).ClusterLogs().List().Search(filter).Send()
+	}
+	return a.conn.ServiceLogs().V1().Clusters().Cluster(cluster.ExternalID()).ClusterLogs().List().Send()
+}
+
+// === IS USER BANNED (uses ocme2eCli.Connection) ===
+func IsUserBanned(ocme2eCli *ocme2e.Client, cluster *cmv1.Cluster) (bool, string, error) {
+	conn := ocme2eCli.Connection
+	user, err := getCreatorFromCluster(conn, cluster)
+	if err != nil {
+		return false, "encountered an issue when checking if the cluster owner is banned. Please investigate.", err
+	}
+
+	if user.Banned() {
+		noteMessage := fmt.Sprintf("User is banned %s. Ban description %s.\n Please open a proactive case, so that MCS can resolve the ban or organize an ownership transfer.", user.BanCode(), user.BanDescription())
+		logging.Warnf(noteMessage)
+		return true, noteMessage, nil
+	}
+	return false, "User is not banned.", nil
+}
+
+func getCreatorFromCluster(conn *sdk.Connection, cluster *cmv1.Cluster) (*amv1.Account, error) {
+	logging.Debugf("Getting subscription from cluster: %s", cluster.ID())
+	sub, ok := cluster.GetSubscription()
+	if !ok {
+		return nil, fmt.Errorf("failed to get subscription from cluster: %s", cluster.ID())
+	}
+	subResp, err := conn.AccountsMgmt().V1().Subscriptions().Subscription(sub.ID()).Get().Send()
 	if err != nil {
-		return nil, fmt.Errorf("Failed to get service log: %w", err)
+		return nil, err
+	}
+	subscription, ok := subResp.GetBody()
+	if !ok {
+		return nil, fmt.Errorf("failed to get subscription body")
+	}
+	if subscription.Status() != "Active" {
+		return nil, fmt.Errorf("expecting status 'Active' found %v", subscription.Status())
+	}
+	accResp, err := conn.AccountsMgmt().V1().Accounts().Account(subscription.Creator().ID()).Get().Send()
+	if err != nil {
+		return nil, err
+	}
+	account, ok := accResp.GetBody()
+	if !ok {
+		return nil, fmt.Errorf("failed to get account body")
 	}
-	return clusterLogsUUIDListResponse, nil
+	return account, nil
 }
 
+// === CREATE CLIENT FROM KUBECONFIG ===
 func CreateClientFromKubeConfig(kubeConfigPath string) (pclient.Client, error) {
-	// Load kubeconfig file and create a client
 	cfg, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build kubeconfig: %v", err)