fix: must-gather e2e test (#1723)

* fix: must-gather e2e test

Signed-off-by: Mateus Oliveira <[email protected]>

* fixup! fix: must-gather e2e test

Signed-off-by: Mateus Oliveira <[email protected]>

* fixup! fixup! fix: must-gather e2e test

Signed-off-by: Mateus Oliveira <[email protected]>

* fixup! fix: must-gather e2e test

Signed-off-by: Mateus Oliveira <[email protected]>

* fixup! fix: must-gather e2e test

Signed-off-by: Mateus Oliveira <[email protected]>

* fixup! fix: must-gather e2e test

Signed-off-by: Mateus Oliveira <[email protected]>

* fixup! fix: must-gather e2e test

Signed-off-by: Mateus Oliveira <[email protected]>

---------

Signed-off-by: Mateus Oliveira <[email protected]>

Author: Mateus Oliveira

.gitignore

@@ -44,3 +44,4 @@ debug.test*
 must-gather/oadp-must-gather
 must-gather/must-gather/
 must-gather/must-gather.local.*/
+tests/e2e/must-gather/

Makefile

@@ -539,7 +539,7 @@ OADP_BUCKET ?= $(shell cat $(OADP_BUCKET_FILE))
 SETTINGS_TMP=/tmp/test-settings
 
 .PHONY: test-e2e-setup
-test-e2e-setup: login-required
+test-e2e-setup: login-required build-must-gather
 	mkdir -p $(SETTINGS_TMP)
 	TMP_DIR=$(SETTINGS_TMP) \
 	OPENSHIFT_CI="$(OPENSHIFT_CI)" \
@@ -590,7 +590,6 @@ test-e2e: test-e2e-setup install-ginkgo ## Run E2E tests against OADP operator i
 	-velero_namespace=$(OADP_TEST_NAMESPACE) \
 	-velero_instance_name=$(VELERO_INSTANCE_NAME) \
 	-artifact_dir=$(ARTIFACT_DIR) \
-	-oc_cli=$(OC_CLI) \
 	-kvm_emulation=$(KVM_EMULATION) \
 	-hco_upstream=$(HCO_UPSTREAM) \
 	--ginkgo.vv \
@@ -635,3 +634,7 @@ endif
 		grep -q "\- $$file_name" $(shell pwd)/config/samples/kustomization.yaml || \
 		$(SED) -i "s%resources:%resources:\n- $$file_name%" $(shell pwd)/config/samples/kustomization.yaml;done
 	@make bundle
+
+.PHONY: build-must-gather
+build-must-gather: ## Build OADP Must-gather binary must-gather/oadp-must-gather
+	cd must-gather && go build -mod=mod -a -o oadp-must-gather cmd/main.go

must-gather/README.md

@@ -13,7 +13,7 @@ podman push <this-image>
 oc adm must-gather --image=<this-image> -- /usr/bin/gather -h
 oc adm must-gather --image=<this-image>
 ```
-TODO mention e2e tests!
+OADP Must-gather is also tested through OADP E2E tests, being run after test cases and checking if summary does not contain errors and all required objects were collected.
 
 To test omg tool, create `omg.Dockerfile` file
 ```Dockerfile

must-gather/pkg/cli.go

@@ -250,7 +250,7 @@ For more information, check OADP must-gather documentation: https://docs.redhat.
 				// OADP dev, community and prod operators have same spec.displayName
 				if csv.Spec.DisplayName == "OADP Operator" {
 					oadpOperatorsText += fmt.Sprintf("Found **%v** version **%v** installed in **%v** namespace\n\n", csv.Spec.DisplayName, csv.Spec.Version, csv.Namespace)
-					if (csv.Spec.Version.Major < 1 || csv.Spec.Version.Minor < 5) && major >= 4 && minor >= 19 {
+					if (csv.Spec.Version.Major < 1 || (csv.Spec.Version.Major == 1 && csv.Spec.Version.Minor < 5)) && major >= 4 && minor >= 19 {
 						oldOADPError += "❌ OADP 1.4 and lower is not supported in OpenShift 4.19 and higher\n\n"
 					}
 					foundOADP = true

must-gather/pkg/templates/summary.go

@@ -340,11 +340,16 @@ func ReplaceDataProtectionApplicationsSection(outputPath string, dataProtectionA
 
 				unsupportedOverridesText := "false"
 				if dataProtectionApplication.Spec.UnsupportedOverrides != nil {
-					summaryTemplateReplaces["ERRORS"] += fmt.Sprintf(
-						"⚠️ DataProtectionApplication **%v** in **%v** namespace is using **unsupportedOverrides**\n\n",
-						dataProtectionApplication.Name, namespace,
-					)
-					unsupportedOverridesText = "⚠️ true"
+					for _, value := range dataProtectionApplication.Spec.UnsupportedOverrides {
+						if value != "" {
+							summaryTemplateReplaces["ERRORS"] += fmt.Sprintf(
+								"⚠️ DataProtectionApplication **%v** in **%v** namespace is using **unsupportedOverrides**\n\n",
+								dataProtectionApplication.Name, namespace,
+							)
+							unsupportedOverridesText = "⚠️ true"
+							break
+						}
+					}
 				}
 
 				dpaStatus := ""

tests/e2e/backup_restore_suite_test.go

@@ -49,10 +49,8 @@ func todoListReady(preBackupState bool, twoVol bool, database string) Verificati
 	})
 }
 
-func prepareBackupAndRestore(brCase BackupRestoreCase, updateLastInstallTime func()) (string, string) {
-	updateLastInstallTime()
-
-	err := dpaCR.CreateOrUpdate(dpaCR.Build(brCase.BackupRestoreType))
+func waitOADPReadiness(backupRestoreType lib.BackupRestoreType) {
+	err := dpaCR.CreateOrUpdate(dpaCR.Build(backupRestoreType))
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 	log.Print("Checking if DPA is reconciled")
@@ -61,7 +59,7 @@ func prepareBackupAndRestore(brCase BackupRestoreCase, updateLastInstallTime fun
 	log.Printf("Waiting for Velero Pod to be running")
 	gomega.Eventually(lib.VeleroPodIsRunning(kubernetesClientForSuiteRun, namespace), time.Minute*3, time.Second*5).Should(gomega.BeTrue())
 
-	if brCase.BackupRestoreType == lib.RESTIC || brCase.BackupRestoreType == lib.KOPIA || brCase.BackupRestoreType == lib.CSIDataMover {
+	if backupRestoreType == lib.RESTIC || backupRestoreType == lib.KOPIA || backupRestoreType == lib.CSIDataMover {
 		log.Printf("Waiting for Node Agent pods to be running")
 		gomega.Eventually(lib.AreNodeAgentPodsRunning(kubernetesClientForSuiteRun, namespace), time.Minute*3, time.Second*5).Should(gomega.BeTrue())
 	}
@@ -70,12 +68,18 @@ func prepareBackupAndRestore(brCase BackupRestoreCase, updateLastInstallTime fun
 
 	log.Print("Checking if BSL is available")
 	gomega.Eventually(dpaCR.BSLsAreAvailable(), time.Minute*3, time.Second*5).Should(gomega.BeTrue())
+}
+
+func prepareBackupAndRestore(brCase BackupRestoreCase, updateLastInstallTime func()) (string, string) {
+	updateLastInstallTime()
+
+	waitOADPReadiness(brCase.BackupRestoreType)
 
 	if brCase.BackupRestoreType == lib.CSI || brCase.BackupRestoreType == lib.CSIDataMover {
 		if provider == "aws" || provider == "ibmcloud" || provider == "gcp" || provider == "azure" || provider == "openstack" {
 			log.Printf("Creating VolumeSnapshotClass for CSI backuprestore of %s", brCase.Name)
 			snapshotClassPath := fmt.Sprintf("./sample-applications/snapclass-csi/%s.yaml", provider)
-			err = lib.InstallApplication(dpaCR.Client, snapshotClassPath)
+			err := lib.InstallApplication(dpaCR.Client, snapshotClassPath)
 			gomega.Expect(err).ToNot(gomega.HaveOccurred())
 		}
 	}
@@ -272,7 +276,7 @@ func tearDownBackupAndRestore(brCase BackupRestoreCase, installTime time.Time, r
 	gomega.Eventually(lib.IsNamespaceDeleted(kubernetesClientForSuiteRun, brCase.Namespace), time.Minute*5, time.Second*5).Should(gomega.BeTrue())
 }
 
-var _ = ginkgo.Describe("Backup and restore tests", func() {
+var _ = ginkgo.Describe("Backup and restore tests", ginkgo.Ordered, func() {
 	var lastBRCase ApplicationBackupRestoreCase
 	var lastInstallTime time.Time
 	updateLastBRcase := func(brCase ApplicationBackupRestoreCase) {
@@ -286,6 +290,19 @@ var _ = ginkgo.Describe("Backup and restore tests", func() {
 		tearDownBackupAndRestore(lastBRCase.BackupRestoreCase, lastInstallTime, ctx.SpecReport())
 	})
 
+	var _ = ginkgo.AfterAll(func() {
+		// DPA just needs to have BSL so gathering of backups/restores logs/describe work
+		// using kopia to collect more info (DaemonSet)
+		waitOADPReadiness(lib.KOPIA)
+
+		log.Printf("Running OADP must-gather")
+		err := lib.RunMustGather(artifact_dir, dpaCR.Client)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		err = dpaCR.Delete()
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	})
+
 	ginkgo.DescribeTable("Backup and restore applications",
 		func(brCase ApplicationBackupRestoreCase, expectedErr error) {
 			if ginkgo.CurrentSpecReport().NumAttempts > 1 && !knownFlake {

tests/e2e/e2e_suite_test.go

@@ -24,8 +24,8 @@ import (
 
 var (
 	// Common vars obtained from flags passed in ginkgo.
-	bslCredFile, namespace, instanceName, provider, vslCredFile, settings, artifact_dir, oc_cli string
-	flakeAttempts                                                                               int64
+	bslCredFile, namespace, instanceName, provider, vslCredFile, settings, artifact_dir string
+	flakeAttempts                                                                       int64
 
 	kubernetesClientForSuiteRun *kubernetes.Clientset
 	runTimeClientForSuiteRun    client.Client
@@ -54,7 +54,6 @@ func init() {
 	flag.StringVar(&instanceName, "velero_instance_name", "example-velero", "Velero Instance Name")
 	flag.StringVar(&provider, "provider", "aws", "Cloud provider")
 	flag.StringVar(&artifact_dir, "artifact_dir", "/tmp", "Directory for storing must gather")
-	flag.StringVar(&oc_cli, "oc_cli", "oc", "OC CLI Client")
 	flag.Int64Var(&flakeAttempts, "flakeAttempts", 3, "Customize the number of flake retries (3)")
 	flag.BoolVar(&kvmEmulation, "kvm_emulation", true, "Enable or disable KVM emulation for virtualization testing")
 	flag.BoolVar(&useUpstreamHco, "hco_upstream", false, "Force use of upstream virtualization operator")
@@ -84,9 +83,6 @@ func init() {
 		if os.Getenv("ARTIFACT_DIR") != "" {
 			artifact_dir = os.Getenv("ARTIFACT_DIR")
 		}
-		if os.Getenv("OC_CLI") != "" {
-			oc_cli = os.Getenv("OC_CLI")
-		}
 		if envValue := os.Getenv("FLAKE_ATTEMPTS"); envValue != "" {
 			// Parse the environment variable as int64
 			parsedValue, err := strconv.ParseInt(envValue, 10, 64)
@@ -188,12 +184,6 @@ var _ = ginkgo.BeforeSuite(func() {
 })
 
 var _ = ginkgo.AfterSuite(func() {
-	// TODO run OADP must-gather
-	// log.Printf("Running OADP must-gather")
-	// err := lib.RunMustGather(oc_cli, artifact_dir)
-	// gomega.Expect(err).ToNot(gomega.HaveOccurred())
-	// TODO validate that everything was collected?
-
 	log.Printf("Deleting Secrets")
 	err := lib.DeleteSecret(kubernetesClientForSuiteRun, namespace, vslSecretName)
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())

tests/e2e/lib/apps.go

@@ -8,15 +8,18 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"reflect"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
 	volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 	ocpappsv1 "github.com/openshift/api/apps/v1"
+	openshiftconfigv1 "github.com/openshift/api/config/v1"
 	security "github.com/openshift/api/security/v1"
 	templatev1 "github.com/openshift/api/template/v1"
 	"github.com/vmware-tanzu/velero/pkg/label"
@@ -373,18 +376,47 @@ func PrintNamespaceEventsAfterTime(c *kubernetes.Clientset, namespace string, st
 	}
 }
 
-func RunMustGather(oc_cli string, artifact_dir string) error {
-	ocClient := oc_cli
-	ocAdmin := "adm"
-	mustGatherCmd := "must-gather"
-	// TODO update image
-	// need to reopen https://github.com/openshift/release/pull/55587
-	mustGatherImg := "--image=quay.io/konveyor/oadp-must-gather:latest"
-	destDir := "--dest-dir=" + artifact_dir
-
-	cmd := exec.Command(ocClient, ocAdmin, mustGatherCmd, mustGatherImg, destDir)
-	_, err := cmd.Output()
-	return err
+func RunMustGather(artifact_dir string, clusterClient client.Client) error {
+	executablePath, err := os.Executable()
+	if err != nil {
+		return err
+	}
+
+	_, err = exec.Command(filepath.Dir(filepath.Dir(filepath.Dir(executablePath))) + "/must-gather/oadp-must-gather").Output()
+	if err != nil {
+		return err
+	}
+
+	_, err = exec.Command("mv", filepath.Dir(executablePath)+"/must-gather", artifact_dir).Output()
+	if err != nil {
+		return err
+	}
+
+	clusterVersionList := &openshiftconfigv1.ClusterVersionList{}
+	err = clusterClient.List(context.Background(), clusterVersionList)
+	if err != nil {
+		return err
+	}
+	if len(clusterVersionList.Items) == 0 {
+		return errors.New("no ClusterVersion found in cluster")
+	}
+	clusterVersion := &clusterVersionList.Items[0]
+	mustGatherPath := fmt.Sprintf("%s/must-gather/clusters/%s/", artifact_dir, string(clusterVersion.Spec.ClusterID[:8]))
+
+	mustGatherSummaryContent, err := os.ReadFile(mustGatherPath + "oadp-must-gather-summary.md")
+	if err != nil {
+		return err
+	}
+
+	mustGatherSummaryText := string(mustGatherSummaryContent)
+
+	if !strings.Contains(mustGatherSummaryText, "No errors happened or were found while running OADP must-gather") {
+		return errors.New("expected no errors in must-gather Errors section")
+	}
+
+	// TODO validate that everything was collected
+
+	return nil
 }
 
 // VerifyBackupRestoreData verifies if app ready before backup and after restore to compare data.

tests/e2e/lib/scheme.go

@@ -4,6 +4,7 @@ import (
 	volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
 	openshiftappsv1 "github.com/openshift/api/apps/v1"
 	openshiftbuildv1 "github.com/openshift/api/build/v1"
+	openshiftconfigv1 "github.com/openshift/api/config/v1"
 	openshiftroutev1 "github.com/openshift/api/route/v1"
 	openshiftsecurityv1 "github.com/openshift/api/security/v1"
 	openshifttemplatev1 "github.com/openshift/api/template/v1"
@@ -36,4 +37,5 @@ func init() {
 	_ = operatorsv1.AddToScheme(Scheme)
 	_ = hypershiftv1.AddToScheme(Scheme)
 	_ = appsv1.AddToScheme(Scheme)
+	_ = openshiftconfigv1.AddToScheme(Scheme)
 }

tests/e2e/virt_backup_restore_suite_test.go

@@ -214,6 +214,17 @@ var _ = ginkgo.Describe("VM backup and restore tests", ginkgo.Ordered, func() {
 	})
 
 	var _ = ginkgo.AfterAll(func() {
+		// DPA just needs to have BSL so gathering of backups/restores logs/describe work
+		// using kopia to collect more info (DaemonSet)
+		waitOADPReadiness(lib.KOPIA)
+
+		log.Printf("Running OADP must-gather")
+		err := lib.RunMustGather(artifact_dir, dpaCR.Client)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+		err = dpaCR.Delete()
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
 		if v != nil && cirrosDownloadedFromTest {
 			v.RemoveDataSource(bootImageNamespace, "cirros")
 			v.RemoveDataVolume(bootImageNamespace, "cirros", 2*time.Minute)
@@ -223,7 +234,7 @@ var _ = ginkgo.Describe("VM backup and restore tests", ginkgo.Ordered, func() {
 			v.EnsureVirtRemoval()
 		}
 
-		err := v.RemoveStorageClass("test-sc-immediate")
+		err = v.RemoveStorageClass("test-sc-immediate")
 		gomega.Expect(err).To(gomega.BeNil())
 		err = v.RemoveStorageClass("test-sc-wffc")
 		gomega.Expect(err).To(gomega.BeNil())