Merge branch 'main' into TT-1858-Add-test-code-owner-in-test-results-metadata

Author: lukaszcl

.github/workflows/framework-golden-tests.yml

@@ -10,6 +10,9 @@ jobs:
     env:
       LOKI_TENANT_ID: promtail
       LOKI_URL: http://localhost:3030/loki/api/v1/push
+      # this is not the best practice, and it must be fixed, run your tests WITHOUT IT!
+      # however, on current latest image we must use this flag
+      CTF_IGNORE_CRITICAL_LOGS: true
     runs-on: ubuntu-latest
     permissions:
       id-token: write

book/src/framework/configuration.md

@@ -11,3 +11,4 @@
 |        LOKI_TENANT_ID        |                                                Streams all components logs to `Loki`, see params below                                                 |                 `string` |                     `promtail`                      |    🚫     |
 |       LOKI_BASIC_AUTH        |                                                          Basic auth in format $user:$password                                                          |        `$user:$password` |                          -                          |    🚫     |
 |         RESTY_DEBUG          |                                                            Log all Resty client HTTP calls                                                             |          `true`, `false` |                       `false`                       |    🚫     |
+|   CTF_IGNORE_CRITICAL_LOGS   |                                      Ignore all logs that has CRIT,FATAL or PANIC levels (Chainlink nodes only!)                                       |          `true`, `false` |                       `false`                       |    🚫     |

framework/.changeset/v0.3.0.md

@@ -1,4 +1,5 @@
 - Add Geth node variant
 - Add Besu node variant
 - Remove unnecessary blockchain params from docs, set defaults
+- Fail any test if there are "CRIT|FATAL|PANIC" log levels (Chainlink nodes only!)
 - Add "ctf c c" command to clean up all cache files

framework/config.go

@@ -132,6 +132,8 @@ func Load[X any](t *testing.T) (*X, error) {
 		require.NoError(t, err)
 		err = WriteAllContainersLogs()
 		require.NoError(t, err)
+		err = checkNodeLogErrors(DefaultCTFLogsDir)
+		require.NoError(t, err)
 	})
 	// TODO: not all the people have AWS access, sadly enough, uncomment when granted
 	//if os.Getenv(EnvVarAWSSecretsManager) == "true" {

framework/logs.go

@@ -0,0 +1,51 @@
+package framework
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+)
+
+const (
+	EnvVarIgnoreCriticalLogs = "CTF_IGNORE_CRITICAL_LOGS"
+)
+
+// checkNodeLogsErrors check Chainlink nodes logs for error levels
+func checkNodeLogErrors(dir string) error {
+	if os.Getenv(EnvVarIgnoreCriticalLogs) == "true" {
+		L.Warn().Msg(`CTF_IGNORE_CRITICAL_LOGS is set to true, we ignore all CRIT|FATAL|PANIC errors in node logs!`)
+		return nil
+	}
+	fileRegex := regexp.MustCompile(`^node.*\.log$`)
+	logLevelRegex := regexp.MustCompile(`(CRIT|PANIC|FATAL)`)
+
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && fileRegex.MatchString(info.Name()) {
+			file, err := os.Open(path)
+			if err != nil {
+				return fmt.Errorf("failed to open file %s: %w", path, err)
+			}
+			defer file.Close()
+
+			scanner := bufio.NewScanner(file)
+			lineNumber := 1
+			for scanner.Scan() {
+				line := scanner.Text()
+				if logLevelRegex.MatchString(line) {
+					return fmt.Errorf("file %s contains a matching log level at line %d: %s", path, lineNumber, line)
+				}
+				lineNumber++
+			}
+			if err := scanner.Err(); err != nil {
+				return fmt.Errorf("error reading file %s: %w", path, err)
+			}
+		}
+		return nil
+	})
+	return err
+}

framework/logs_test.go

@@ -0,0 +1,48 @@
+package framework
+
+import (
+	"path/filepath"
+	"testing"
+)
+
+// Table test for checkLogFilesForLevels
+func TestCheckLogFilesForLevels(t *testing.T) {
+	tests := []struct {
+		name        string
+		dir         string
+		content     string
+		expectError bool
+	}{
+		{
+			name:        "Clean",
+			dir:         "clean",
+			expectError: false,
+		},
+		{
+			name:        "Contains CRIT",
+			dir:         "crit",
+			expectError: true,
+		},
+		{
+			name:        "Contains PANIC",
+			dir:         "panic",
+			expectError: true,
+		},
+		{
+			name:        "Contains FATAL",
+			dir:         "fatal",
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := checkNodeLogErrors(filepath.Join("testdata", tt.dir))
+			if tt.expectError && err == nil {
+				t.Errorf("expected error but got none")
+			} else if !tt.expectError && err != nil {
+				t.Errorf("did not expect error but got: %v", err)
+			}
+		})
+	}
+}

framework/testdata/clean/node_clean.log

@@ -0,0 +1,3 @@
+2024-11-29T09:09:31.901Z [INFO]  StartUpHealthReport shutdown complete              services/health.go:47            version=2.17.0@5ebb632
+2024-11-29T09:09:31.901Z [DEBUG]  StartUpHealthReport shutdown complete              services/health.go:47            version=2.17.0@5ebb632
+2024-11-29T09:09:31.901Z [WARN]  StartUpHealthReport shutdown complete              services/health.go:47            version=2.17.0@5ebb632

framework/testdata/crit/node_crit.log

@@ -0,0 +1 @@
+2024-11-29T09:09:31.901Z [CRIT]  StartUpHealthReport shutdown complete              services/health.go:47            version=2.17.0@5ebb632

framework/testdata/fatal/node_fatal.log

@@ -0,0 +1 @@
+2024-11-29T09:09:31.901Z [FATAL]  StartUpHealthReport shutdown complete              services/health.go:47            version=2.17.0@5ebb632

framework/testdata/panic/node_panic.log

@@ -0,0 +1 @@
+2024-11-29T09:09:31.901Z [PANIC]  StartUpHealthReport shutdown complete              services/health.go:47            version=2.17.0@5ebb632