check logs for CRIT|PANIC|FATAL logs and fail the test

Author: skudasov

framework/config.go

@@ -132,6 +132,8 @@ func Load[X any](t *testing.T) (*X, error) {
 		require.NoError(t, err)
 		err = WriteAllContainersLogs()
 		require.NoError(t, err)
+		err = checkNodeLogErrors(DefaultCTFLogsDir)
+		require.NoError(t, err)
 	})
 	// TODO: not all the people have AWS access, sadly enough, uncomment when granted
 	//if os.Getenv(EnvVarAWSSecretsManager) == "true" {

framework/logs.go

@@ -0,0 +1,43 @@
+package framework
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+)
+
+// checkNodeLogsErrors check Chainlink nodes logs for error levels
+func checkNodeLogErrors(dir string) error {
+	fileRegex := regexp.MustCompile(`^node.*\.log$`)
+	logLevelRegex := regexp.MustCompile(`(CRIT|PANIC|FATAL)`)
+
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && fileRegex.MatchString(info.Name()) {
+			file, err := os.Open(path)
+			if err != nil {
+				return fmt.Errorf("failed to open file %s: %w", path, err)
+			}
+			defer file.Close()
+
+			scanner := bufio.NewScanner(file)
+			lineNumber := 1
+			for scanner.Scan() {
+				line := scanner.Text()
+				if logLevelRegex.MatchString(line) {
+					return fmt.Errorf("file %s contains a matching log level at line %d: %s", path, lineNumber, line)
+				}
+				lineNumber++
+			}
+			if err := scanner.Err(); err != nil {
+				return fmt.Errorf("error reading file %s: %w", path, err)
+			}
+		}
+		return nil
+	})
+	return err
+}

framework/logs_test.go

@@ -0,0 +1,48 @@
+package framework
+
+import (
+	"path/filepath"
+	"testing"
+)
+
+// Table test for checkLogFilesForLevels
+func TestCheckLogFilesForLevels(t *testing.T) {
+	tests := []struct {
+		name        string
+		dir         string
+		content     string
+		expectError bool
+	}{
+		{
+			name:        "Clean",
+			dir:         "clean",
+			expectError: false,
+		},
+		{
+			name:        "Contains CRIT",
+			dir:         "crit",
+			expectError: true,
+		},
+		{
+			name:        "Contains PANIC",
+			dir:         "panic",
+			expectError: true,
+		},
+		{
+			name:        "Contains FATAL",
+			dir:         "fatal",
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := checkNodeLogErrors(filepath.Join("testdata", tt.dir))
+			if tt.expectError && err == nil {
+				t.Errorf("expected error but got none")
+			} else if !tt.expectError && err != nil {
+				t.Errorf("did not expect error but got: %v", err)
+			}
+		})
+	}
+}

framework/testdata/clean/node_clean.log

@@ -0,0 +1,3 @@
+2024-11-29T09:09:31.901Z [INFO]  StartUpHealthReport shutdown complete              services/health.go:47            version=2.17.0@5ebb632
+2024-11-29T09:09:31.901Z [DEBUG]  StartUpHealthReport shutdown complete              services/health.go:47            version=2.17.0@5ebb632
+2024-11-29T09:09:31.901Z [WARN]  StartUpHealthReport shutdown complete              services/health.go:47            version=2.17.0@5ebb632

framework/testdata/crit/node_crit.log

@@ -0,0 +1 @@
+2024-11-29T09:09:31.901Z [CRIT]  StartUpHealthReport shutdown complete              services/health.go:47            version=2.17.0@5ebb632

framework/testdata/fatal/node_fatal.log

@@ -0,0 +1 @@
+2024-11-29T09:09:31.901Z [FATAL]  StartUpHealthReport shutdown complete              services/health.go:47            version=2.17.0@5ebb632

framework/testdata/panic/node_panic.log

@@ -0,0 +1 @@
+2024-11-29T09:09:31.901Z [PANIC]  StartUpHealthReport shutdown complete              services/health.go:47            version=2.17.0@5ebb632