read all logs even with restarts

Author: skudasov

framework/components/simple_node_set/reload.go

@@ -1,6 +1,9 @@
 package simple_node_set
 
 import (
+	"fmt"
+	"github.com/google/uuid"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/chaos"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
 	"time"
@@ -9,6 +12,10 @@ import (
 // UpgradeNodeSet updates nodes configuration TOML files
 // this API is discouraged, however, you can use it if nodes require restart or configuration updates, temporarily!
 func UpgradeNodeSet(in *Input, bc *blockchain.Output, wait time.Duration) (*Output, error) {
+	uniq := fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, uuid.NewString()[0:4])
+	if err := framework.WriteAllContainersLogs(uniq); err != nil {
+		return nil, err
+	}
 	_, err := chaos.ExecPumba("rm --volumes=false re2:^node.*|ns-postgresql.*", wait)
 	if err != nil {
 		return nil, err

framework/config.go

@@ -130,9 +130,9 @@ func Load[X any](t *testing.T) (*X, error) {
 	t.Cleanup(func() {
 		err := Store[X](input)
 		require.NoError(t, err)
-		err = WriteAllContainersLogs()
+		err = WriteAllContainersLogs(DefaultCTFLogsDir)
 		require.NoError(t, err)
-		err = checkNodeLogErrors(DefaultCTFLogsDir)
+		err = checkAllNodeLogErrors()
 		require.NoError(t, err)
 	})
 	// TODO: not all the people have AWS access, sadly enough, uncomment when granted

framework/docker.go

@@ -189,11 +189,11 @@ func isLocalToolDockerContainer(containerName string) bool {
 }
 
 // WriteAllContainersLogs writes all Docker container logs to the default logs directory
-func WriteAllContainersLogs() error {
+func WriteAllContainersLogs(dir string) error {
 	L.Info().Msg("Writing Docker containers logs")
-	if _, err := os.Stat(DefaultCTFLogsDir); os.IsNotExist(err) {
-		if err := os.MkdirAll(DefaultCTFLogsDir, 0755); err != nil {
-			return fmt.Errorf("failed to create directory %s: %w", DefaultCTFLogsDir, err)
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return fmt.Errorf("failed to create directory %s: %w", dir, err)
 		}
 	}
 	provider, err := tc.NewDockerProvider()
@@ -220,7 +220,7 @@ func WriteAllContainersLogs() error {
 				L.Error().Err(err).Str("Container", containerName).Msg("failed to fetch logs for container")
 				return err
 			}
-			logFilePath := filepath.Join(DefaultCTFLogsDir, fmt.Sprintf("%s.log", containerName))
+			logFilePath := filepath.Join(dir, fmt.Sprintf("%s.log", containerName))
 			logFile, err := os.Create(logFilePath)
 			if err != nil {
 				L.Error().Err(err).Str("Container", containerName).Msg("failed to create container log file")

framework/logs.go

@@ -6,12 +6,44 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 )
 
 const (
 	EnvVarIgnoreCriticalLogs = "CTF_IGNORE_CRITICAL_LOGS"
 )
 
+func getLogDirectories() ([]string, error) {
+	logDirs := make([]string, 0)
+	currentDir, err := os.Getwd()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get current directory: %w", err)
+	}
+	entries, err := os.ReadDir(currentDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read directory: %w", err)
+	}
+	for _, entry := range entries {
+		if entry.IsDir() && strings.HasPrefix(entry.Name(), "logs-") {
+			logDirs = append(logDirs, filepath.Join(currentDir, entry.Name()))
+		}
+	}
+	return logDirs, nil
+}
+
+func checkAllNodeLogErrors() error {
+	dirs, err := getLogDirectories()
+	if err != nil {
+		return err
+	}
+	for _, dd := range dirs {
+		if err := checkNodeLogErrors(dd); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // checkNodeLogsErrors check Chainlink nodes logs for error levels
 func checkNodeLogErrors(dir string) error {
 	if os.Getenv(EnvVarIgnoreCriticalLogs) == "true" {