Refactor logs gather flow to support multiple clusters (openshift-service-mesh#452)

MaxBab · web-flow · commit ec13475e4ca5 · 2024-10-28T07:00:52.000-04:00
- Refactor logs gathering flow to support logs gather from multiple
  clusters.
  This is used in multi cluster scenario

- Add "getSecret" function to fetch certs that are used in multicluster
  test.

- Add name cluster to be specified during log gathering.

Signed-off-by: Maxim Babushkin &lt;mbabushk@redhat.com&gt;
diff --git a/tests/e2e/controlplane/control_plane_suite_test.go b/tests/e2e/controlplane/control_plane_suite_test.go
@@ -63,5 +63,5 @@ func setup() {
 	cl, err = k8sclient.InitK8sClient("")
 	Expect(err).NotTo(HaveOccurred())
 
-	k = kubectl.New()
+	k = kubectl.New("clControlPlane")
 }
diff --git a/tests/e2e/controlplane/control_plane_test.go b/tests/e2e/controlplane/control_plane_test.go
@@ -295,7 +295,7 @@ spec:
 
 		AfterAll(func(ctx SpecContext) {
 			if CurrentSpecReport().Failed() {
-				common.LogDebugInfo()
+				common.LogDebugInfo(k)
 				debugInfoLogged = true
 			}
 
@@ -314,7 +314,7 @@ spec:
 
 	AfterAll(func() {
 		if CurrentSpecReport().Failed() && !debugInfoLogged {
-			common.LogDebugInfo()
+			common.LogDebugInfo(k)
 			debugInfoLogged = true
 		}
 
diff --git a/tests/e2e/dualstack/dualstack_suite_test.go b/tests/e2e/dualstack/dualstack_suite_test.go
@@ -63,5 +63,5 @@ func setup() {
 	cl, err = k8sclient.InitK8sClient("")
 	Expect(err).NotTo(HaveOccurred())
 
-	k = kubectl.New()
+	k = kubectl.New("clDualStack")
 }
diff --git a/tests/e2e/dualstack/dualstack_test.go b/tests/e2e/dualstack/dualstack_test.go
@@ -280,7 +280,7 @@ spec:
 
 		AfterAll(func(ctx SpecContext) {
 			if CurrentSpecReport().Failed() {
-				common.LogDebugInfo()
+				common.LogDebugInfo(k)
 				debugInfoLogged = true
 			}
 
@@ -295,7 +295,7 @@ spec:
 
 	AfterAll(func() {
 		if CurrentSpecReport().Failed() && !debugInfoLogged {
-			common.LogDebugInfo()
+			common.LogDebugInfo(k)
 			debugInfoLogged = true
 		}
 
diff --git a/tests/e2e/multicluster/multicluster_multiprimary_test.go b/tests/e2e/multicluster/multicluster_multiprimary_test.go
@@ -253,7 +253,8 @@ spec:
 
 				AfterAll(func(ctx SpecContext) {
 					if CurrentSpecReport().Failed() {
-						common.LogDebugInfo()
+						common.LogDebugInfo(k1)
+						common.LogDebugInfo(k2)
 						debugInfoLogged = true
 					}
 
@@ -277,7 +278,8 @@ spec:
 
 	AfterAll(func(ctx SpecContext) {
 		if CurrentSpecReport().Failed() && !debugInfoLogged {
-			common.LogDebugInfo()
+			common.LogDebugInfo(k1)
+			common.LogDebugInfo(k2)
 			debugInfoLogged = true
 		}
 
diff --git a/tests/e2e/multicluster/multicluster_primaryremote_test.go b/tests/e2e/multicluster/multicluster_primaryremote_test.go
@@ -295,7 +295,8 @@ spec:
 
 				AfterAll(func(ctx SpecContext) {
 					if CurrentSpecReport().Failed() {
-						common.LogDebugInfo()
+						common.LogDebugInfo(k1)
+						common.LogDebugInfo(k2)
 						debugInfoLogged = true
 					}
 
@@ -319,7 +320,8 @@ spec:
 
 	AfterAll(func(ctx SpecContext) {
 		if CurrentSpecReport().Failed() && !debugInfoLogged {
-			common.LogDebugInfo()
+			common.LogDebugInfo(k1)
+			common.LogDebugInfo(k2)
 			debugInfoLogged = true
 		}
 
diff --git a/tests/e2e/multicluster/multicluster_suite_test.go b/tests/e2e/multicluster/multicluster_suite_test.go
@@ -99,6 +99,6 @@ func setup(t *testing.T) {
 	exposeIstiodYAML = fmt.Sprintf("%s/docs/multicluster/expose-istiod.yaml", baseRepoDir)
 
 	// Initialize kubectl utilities, one for each cluster
-	k1 = kubectl.New().WithKubeconfig(kubeconfig)
-	k2 = kubectl.New().WithKubeconfig(kubeconfig2)
+	k1 = kubectl.New("clPrimary").WithKubeconfig(kubeconfig)
+	k2 = kubectl.New("clRemote").WithKubeconfig(kubeconfig2)
 }
diff --git a/tests/e2e/operator/operator_install_test.go b/tests/e2e/operator/operator_install_test.go
@@ -100,14 +100,14 @@ var _ = Describe("Operator", Ordered, func() {
 
 		AfterAll(func() {
 			if CurrentSpecReport().Failed() {
-				common.LogDebugInfo()
+				common.LogDebugInfo(k)
 			}
 		})
 	})
 
 	AfterAll(func() {
 		if CurrentSpecReport().Failed() {
-			common.LogDebugInfo()
+			common.LogDebugInfo(k)
 		}
 
 		if skipDeploy {
diff --git a/tests/e2e/operator/operator_suite_test.go b/tests/e2e/operator/operator_suite_test.go
@@ -62,5 +62,5 @@ func setup() {
 		GinkgoWriter.Println("Running on Kubernetes")
 	}
 
-	k = kubectl.New()
+	k = kubectl.New("clOperator")
 }
diff --git a/tests/e2e/util/common/e2e_utils.go b/tests/e2e/util/common/e2e_utils.go
@@ -57,8 +57,6 @@ var (
 	// - 1.24-alpha.feabc1234
 	// matching only the version before first '_' which is used in the downstream builds, e.g. "1.23.2_ossm_tp.2"
 	istiodVersionRegex = regexp.MustCompile(`Version:"([^"_]*)[^"]*"`)
-
-	k = kubectl.New()
 )
 
 // GetObject returns the object with the given key
@@ -131,73 +129,80 @@ func CheckNamespaceEmpty(ctx SpecContext, cl client.Client, ns string) {
 	}).Should(BeEmpty(), "No Services should be present in the namespace")
 }
 
-func LogDebugInfo() {
+func LogDebugInfo(k kubectl.Kubectl) {
 	// General debugging information to help diagnose the failure
 	// TODO: Add the creation of file with this information to be attached to the test report
 
 	GinkgoWriter.Println()
-	GinkgoWriter.Println("The test run has failures and the debug information is as follows:")
+	GinkgoWriter.Println("The test run has failures and the debug information is as follows from cluster: %q:", k.GetClusterName())
+	GinkgoWriter.Println("=========================================================")
+	logOperatorDebugInfo(k)
 	GinkgoWriter.Println("=========================================================")
-	logOperatorDebugInfo()
+	logIstioDebugInfo(k)
 	GinkgoWriter.Println("=========================================================")
-	logIstioDebugInfo()
+	logCNIDebugInfo(k)
 	GinkgoWriter.Println("=========================================================")
-	logCNIDebugInfo()
+	logCertsDebugInfo(k)
 	GinkgoWriter.Println("=========================================================")
 }
 
-func logOperatorDebugInfo() {
-	k := k.WithNamespace(OperatorNamespace)
+func logOperatorDebugInfo(k kubectl.Kubectl) {
+	k = k.WithNamespace(OperatorNamespace)
 	operator, err := k.GetYAML("deployment", deploymentName)
-	logDebugElement("Operator Deployment YAML", operator, err)
+	logDebugElement("=====Operator Deployment YAML=====", operator, err)
 
 	logs, err := k.Logs("deploy/"+deploymentName, ptr.Of(120*time.Second))
-	logDebugElement("Operator logs", logs, err)
+	logDebugElement("=====Operator logs=====", logs, err)
 
 	events, err := k.GetEvents()
-	logDebugElement("Events in "+OperatorNamespace, events, err)
+	logDebugElement("=====Events in "+OperatorNamespace+"=====", events, err)
 
 	// Temporary information to gather more details about failure
 	pods, err := k.GetPods("", "-o wide")
-	logDebugElement("Pods in "+OperatorNamespace, pods, err)
+	logDebugElement("=====Pods in "+OperatorNamespace+"=====", pods, err)
 
 	describe, err := k.Describe("deployment", deploymentName)
-	logDebugElement("Operator Deployment describe", describe, err)
+	logDebugElement("=====Operator Deployment describe=====", describe, err)
 }
 
-func logIstioDebugInfo() {
+func logIstioDebugInfo(k kubectl.Kubectl) {
 	resource, err := k.GetYAML("istio", istioName)
-	logDebugElement("Istio YAML", resource, err)
+	logDebugElement("=====Istio YAML=====", resource, err)
 
 	output, err := k.WithNamespace(controlPlaneNamespace).GetPods("", "-o wide")
-	logDebugElement("Pods in "+controlPlaneNamespace, output, err)
+	logDebugElement("=====Pods in "+controlPlaneNamespace+"=====", output, err)
 
 	logs, err := k.WithNamespace(controlPlaneNamespace).Logs("deploy/istiod", ptr.Of(120*time.Second))
-	logDebugElement("Istiod logs", logs, err)
+	logDebugElement("=====Istiod logs=====", logs, err)
 
 	events, err := k.WithNamespace(controlPlaneNamespace).GetEvents()
-	logDebugElement("Events in "+controlPlaneNamespace, events, err)
+	logDebugElement("=====Events in "+controlPlaneNamespace+"=====", events, err)
 }
 
-func logCNIDebugInfo() {
+func logCNIDebugInfo(k kubectl.Kubectl) {
 	resource, err := k.GetYAML("istiocni", istioCniName)
-	logDebugElement("IstioCNI YAML", resource, err)
+	logDebugElement("=====IstioCNI YAML=====", resource, err)
 
 	ds, err := k.WithNamespace(istioCniNamespace).GetYAML("daemonset", "istio-cni-node")
-	logDebugElement("Istio CNI DaemonSet YAML", ds, err)
+	logDebugElement("=====Istio CNI DaemonSet YAML=====", ds, err)
 
 	events, err := k.WithNamespace(istioCniNamespace).GetEvents()
-	logDebugElement("Events in "+istioCniNamespace, events, err)
+	logDebugElement("=====Events in "+istioCniNamespace+"=====", events, err)
 
 	// Temporary information to gather more details about failure
 	pods, err := k.WithNamespace(istioCniNamespace).GetPods("", "-o wide")
-	logDebugElement("Pods in "+istioCniNamespace, pods, err)
+	logDebugElement("=====Pods in "+istioCniNamespace+"=====", pods, err)
 
 	describe, err := k.WithNamespace(istioCniNamespace).Describe("daemonset", "istio-cni-node")
-	logDebugElement("Istio CNI DaemonSet describe", describe, err)
+	logDebugElement("=====Istio CNI DaemonSet describe=====", describe, err)
 
 	logs, err := k.WithNamespace(istioCniNamespace).Logs("daemonset/istio-cni-node", ptr.Of(120*time.Second))
-	logDebugElement("Istio CNI logs", logs, err)
+	logDebugElement("=====Istio CNI logs=====", logs, err)
+}
+
+func logCertsDebugInfo(k kubectl.Kubectl) {
+	certs, err := k.WithNamespace(controlPlaneNamespace).GetSecret("cacerts")
+	logDebugElement("=====CA certs=====", certs, err)
 }
 
 func logDebugElement(caption string, info string, err error) {
@@ -211,7 +216,7 @@ func logDebugElement(caption string, info string, err error) {
 }
 
 func GetVersionFromIstiod() (*semver.Version, error) {
-	k := kubectl.New()
+	k := kubectl.New("testCluster")
 	output, err := k.WithNamespace(controlPlaneNamespace).Exec("deploy/istiod", "", "pilot-discovery version")
 	if err != nil {
 		return nil, fmt.Errorf("error getting version from istiod: %w", err)
diff --git a/tests/e2e/util/kubectl/kubectl.go b/tests/e2e/util/kubectl/kubectl.go
@@ -25,14 +25,15 @@ import (
 )
 
 type Kubectl struct {
+	name       string
 	binary     string
 	namespace  string
 	kubeconfig string
 }
 
 // New creates a new kubectl.Kubectl
-func New() Kubectl {
-	return Kubectl{}.WithBinary(os.Getenv("COMMAND"))
+func New(name string) Kubectl {
+	return Kubectl{name: name}.WithBinary(os.Getenv("COMMAND"))
 }
 
 func (k Kubectl) build(cmd string) string {
@@ -217,6 +218,11 @@ func (k Kubectl) ForceDelete(kind, name string) error {
 	return k.Delete(kind, name)
 }
 
+// Gets cluster name defined during initialization
+func (k Kubectl) GetClusterName() string {
+	return k.name
+}
+
 // GetYAML returns the yaml of a resource
 func (k Kubectl) GetYAML(kind, name string) (string, error) {
 	cmd := k.build(fmt.Sprintf(" get %s %s -o yaml", kind, name))
@@ -250,6 +256,17 @@ func (k Kubectl) GetInternalIP(label string) (string, error) {
 	return output, nil
 }
 
+// GetSecret returns the secret of a namespace
+func (k Kubectl) GetSecret(secret string) (string, error) {
+	cmd := k.build(fmt.Sprintf(" get secret %s -o yaml", secret))
+	output, err := k.executeCommand(cmd)
+	if err != nil {
+		return "", fmt.Errorf("error getting secret: %w, output %s", err, output)
+	}
+
+	return output, nil
+}
+
 // Exec executes a command in the pod or specific container
 func (k Kubectl) Exec(pod, container, command string) (string, error) {
 	cmd := k.build(fmt.Sprintf(" exec %s %s -- %s", pod, containerFlag(container), command))

Original file line number	Diff line number	Diff line change
`@@ -63,5 +63,5 @@ func setup() {`
`63`	`63`	`cl, err = k8sclient.InitK8sClient("")`
`64`	`64`	`Expect(err).NotTo(HaveOccurred())`
`65`	`65`
`66`		`- k = kubectl.New()`
	`66`	`+ k = kubectl.New("clControlPlane")`
`67`	`67`	`}`
Original file line number	Diff line number	Diff line change
`@@ -295,7 +295,7 @@ spec:`
`295`	`295`
`296`	`296`	`AfterAll(func(ctx SpecContext) {`
`297`	`297`	`if CurrentSpecReport().Failed() {`
`298`		`- common.LogDebugInfo()`
	`298`	`+ common.LogDebugInfo(k)`
`299`	`299`	`debugInfoLogged = true`
`300`	`300`	`}`
`301`	`301`
`@@ -314,7 +314,7 @@ spec:`
`314`	`314`
`315`	`315`	`AfterAll(func() {`
`316`	`316`	`if CurrentSpecReport().Failed() && !debugInfoLogged {`
`317`		`- common.LogDebugInfo()`
	`317`	`+ common.LogDebugInfo(k)`
`318`	`318`	`debugInfoLogged = true`
`319`	`319`	`}`
`320`	`320`
Original file line number	Diff line number	Diff line change
`@@ -280,7 +280,7 @@ spec:`
`280`	`280`
`281`	`281`	`AfterAll(func(ctx SpecContext) {`
`282`	`282`	`if CurrentSpecReport().Failed() {`
`283`		`- common.LogDebugInfo()`
	`283`	`+ common.LogDebugInfo(k)`
`284`	`284`	`debugInfoLogged = true`
`285`	`285`	`}`
`286`	`286`
`@@ -295,7 +295,7 @@ spec:`
`295`	`295`
`296`	`296`	`AfterAll(func() {`
`297`	`297`	`if CurrentSpecReport().Failed() && !debugInfoLogged {`
`298`		`- common.LogDebugInfo()`
	`298`	`+ common.LogDebugInfo(k)`
`299`	`299`	`debugInfoLogged = true`
`300`	`300`	`}`
`301`	`301`
Original file line number	Diff line number	Diff line change
`@@ -99,6 +99,6 @@ func setup(t *testing.T) {`
`99`	`99`	`exposeIstiodYAML = fmt.Sprintf("%s/docs/multicluster/expose-istiod.yaml", baseRepoDir)`
`100`	`100`
`101`	`101`	`// Initialize kubectl utilities, one for each cluster`
`102`		`- k1 = kubectl.New().WithKubeconfig(kubeconfig)`
`103`		`- k2 = kubectl.New().WithKubeconfig(kubeconfig2)`
	`102`	`+ k1 = kubectl.New("clPrimary").WithKubeconfig(kubeconfig)`
	`103`	`+ k2 = kubectl.New("clRemote").WithKubeconfig(kubeconfig2)`
`104`	`104`	`}`
Original file line number	Diff line number	Diff line change
`@@ -100,14 +100,14 @@ var _ = Describe("Operator", Ordered, func() {`
`100`	`100`
`101`	`101`	`AfterAll(func() {`
`102`	`102`	`if CurrentSpecReport().Failed() {`
`103`		`- common.LogDebugInfo()`
	`103`	`+ common.LogDebugInfo(k)`
`104`	`104`	`}`
`105`	`105`	`})`
`106`	`106`	`})`
`107`	`107`
`108`	`108`	`AfterAll(func() {`
`109`	`109`	`if CurrentSpecReport().Failed() {`
`110`		`- common.LogDebugInfo()`
	`110`	`+ common.LogDebugInfo(k)`
`111`	`111`	`}`
`112`	`112`
`113`	`113`	`if skipDeploy {`
Original file line number	Diff line number	Diff line change
`@@ -62,5 +62,5 @@ func setup() {`
`62`	`62`	`GinkgoWriter.Println("Running on Kubernetes")`
`63`	`63`	`}`
`64`	`64`
`65`		`- k = kubectl.New()`
	`65`	`+ k = kubectl.New("clOperator")`
`66`	`66`	`}`