TAF: Check errors in process files (#1460)

* Added new tests for checking errors in files

* Revert to ci_prod branch

Author: suyadav1

test/ginkgo-e2e/containerstatus/containerstatus_suite_test.go

@@ -1,6 +1,7 @@
 package containerstatus_test
 
 import (
+	"os"
 	"testing"
 
 	"docker-provider/test/utils"
@@ -17,6 +18,7 @@ import (
 var K8sClient *kubernetes.Clientset
 var Cfg *rest.Config
 var ResourceOptimizationEnabled string
+var GenevaIntegrationEnabled string
 
 func TestContainerStatus(t *testing.T) {
 	RegisterFailHandler(Fail)
@@ -30,6 +32,7 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 	ResourceOptimizationEnabled, err = utils.IsResourceOptimizationEnabled(K8sClient, "kube-system", "component", "ama-logs-agent", "ama-logs")
 	Expect(err).NotTo(HaveOccurred())
+	GenevaIntegrationEnabled = os.Getenv("GENEVA_INTEGRATION")
 })
 
 var _ = AfterSuite(func() {

test/ginkgo-e2e/containerstatus/containerstatus_test.go

@@ -117,3 +117,28 @@ var _ = DescribeTable("The container logs should not contain errors",
 	Entry("when checking the ama-logs daemonset pods", "kube-system", "component", "ama-logs-agent"),
 	Entry("when checking the ama-logs-windows daemonset pods", "kube-system", "component", "ama-logs-agent-windows", Label(utils.WindowsLabel)),
 )
+
+/*
+- The containers should not contain any errors for the running processes.
+*/
+var _ = DescribeTable("The ama-logs container should not contain errors in the running processes",
+	func(namespace, labelName, labelValue, containerName, filePath string) {
+		if GenevaIntegrationEnabled == "true" && filePath == "/var/opt/microsoft/docker-cimprov/log/fluent-bit.log" {
+			Skip("Skipping fluent-bit log check for ama-logs container when Geneva integration is enabled")
+		} else if GenevaIntegrationEnabled != "true" && filePath == "/var/opt/microsoft/docker-cimprov/log/fluent-bit-geneva.log" {
+			Skip("Skipping fluent-bit-geneva log check for ama-logs container when Geneva integration is disabled")
+		}
+		err := utils.CheckFileForErrors(K8sClient, Cfg, namespace, labelName, labelValue, containerName, filePath)
+		Expect(err).NotTo(HaveOccurred())
+	},
+	// fluentd logs
+	Entry("when checking the ama-logs container for fluentd", "kube-system", "component", "ama-logs-agent", "ama-logs", "/var/opt/microsoft/docker-cimprov/log/fluentd.log"),
+	// fluent-bit logs
+	Entry("when checking the ama-logs container for fluentbit", "kube-system", "component", "ama-logs-agent", "ama-logs", "/var/opt/microsoft/docker-cimprov/log/fluent-bit.log"),
+	Entry("when checking the ama-logs container for fluentbit", "kube-system", "component", "ama-logs-agent", "ama-logs", "/var/opt/microsoft/docker-cimprov/log/fluent-bit-geneva.log"),
+	Entry("when checking the ama-logs container for fluent_forward_failed", "kube-system", "component", "ama-logs-agent", "ama-logs", "/var/opt/microsoft/docker-cimprov/log/fluent_forward_failed.log"),
+	// telegraf logs
+	Entry("when checking the ama-logs container for telegraf", "kube-system", "component", "ama-logs-agent", "ama-logs", "/var/opt/microsoft/docker-cimprov/log/telegraf_error.log"),
+	// mdsd logs
+	Entry("when checking the ama-logs container for mdsd", "kube-system", "component", "ama-logs-agent", "ama-logs", "/var/opt/microsoft/linuxmonagent/log/mdsd.err"),
+)

test/ginkgo-e2e/utils/kubernetes_api_utils.go

@@ -492,3 +492,30 @@ func GetAllNodes(clientset *kubernetes.Clientset) ([]corev1.Node, error) {
 
 	return nodes.Items, nil
 }
+
+// CheckFileForErrors checks if a specific file in a linux container contains errors.
+func CheckFileForErrors(clientset *kubernetes.Clientset, Cfg *rest.Config, namespace, labelName, labelValue, containerName, filePath string) error {
+	pods, err := GetPodsWithLabel(clientset, namespace, labelName, labelValue)
+	if err != nil {
+		return fmt.Errorf("failed to get pods with label %s=%s: %v", labelName, labelValue, err)
+	}
+
+	for _, pod := range pods {
+		command := []string{"bash", "-c", fmt.Sprintf("grep -i error %s", filePath)}
+		stdout, stderr, err := ExecCmd(clientset, Cfg, pod.Name, containerName, namespace, command)
+		if err != nil {
+			// If grep returns exit code 1, it means no matches were found, which is not an error for our use case
+			if strings.Contains(err.Error(), "exit code 1") {
+				// No errors found in the file, continue
+				continue
+			}
+			return fmt.Errorf("error executing command in pod %s, container %s: %v, stderr: %s", pod.Name, containerName, err, stderr)
+		}
+
+		if stdout != "" {
+			return fmt.Errorf("errors found in file %s in pod %s, container %s: %s", filePath, pod.Name, containerName, stdout)
+		}
+	}
+
+	return nil
+}

test/testkube/testkube-test-crs.yaml

@@ -16,6 +16,11 @@ spec:
       branch: ci_prod
       path: test/ginkgo-e2e
   executionRequest:
+    variables:
+      GENEVA_INTEGRATION:
+        name: GENEVA_INTEGRATION
+        value: "$GENEVA_INTEGRATION"
+        type: basic
     args:
       - "--label-filter"
       - "!(windows)"
@@ -40,6 +45,11 @@ spec:
       branch: ci_prod
       path: test/ginkgo-e2e
   executionRequest:
+    variables:
+      GENEVA_INTEGRATION:
+        name: GENEVA_INTEGRATION
+        value: "$GENEVA_INTEGRATION"
+        type: basic
     args:
       - "--label-filter"
       - "windows"