asserting logs

Author: skudasov

book/src/SUMMARY.md

@@ -11,6 +11,7 @@
   - [NodeSet (Local Docker builds)](./framework/nodeset_docker_rebuild.md)
   - [NodeSet Compat Environment](./framework/nodeset_compatibility.md)
   - [Creating your own components](./developing/developing_components.md)
+  - [Asserting Logs](./developing/asserting_logs.md)
   - [Fork Testing](./framework/fork.md)
   - [Quick Contracts Deployment](./framework/quick_deployment.md)
   - [Verifying Contracts](./framework/verify.md)

book/src/developing/asserting_logs.md

@@ -0,0 +1,35 @@
+# Asserting Container Logs
+
+You can either assert that CL nodes have no errors like that, we check `(CRIT|PANIC|FATAL)` levels by default for all the nodes
+
+```golang
+	in, err := framework.Load[Cfg](t)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		err := framework.SaveAndCheckLogs(t)
+		require.NoError(t, err)
+	})
+```
+
+or customize file assertions
+
+```golang
+	in, err := framework.Load[Cfg](t)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		// save all the logs to default directory "logs/docker-$test_name"
+		logs, err := framework.SaveContainerLogs(fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, t.Name()))
+		require.NoError(t, err)
+		// check that CL nodes has no errors (CRIT|PANIC|FATAL) levels
+		err = framework.CheckCLNodeContainerErrors()
+		require.NoError(t, err)
+		// do custom assertions
+		for _, l := range logs {
+			matches, err := framework.SearchLogFile(l, " name=HeadReporter version=\\d")
+			require.NoError(t, err)
+			_ = matches
+		}
+	})
+```
+
+Full [example](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/framework/examples/myproject/smoke_logs_test.go)

framework/.changeset/v0.4.8.md

@@ -0,0 +1,2 @@
+- Do not expose 40k ports by default, only with CTF_CLNODE_DLV flag
+- Remove default log checking API, make it simpler to customize

framework/components/clnode/clnode.go

@@ -135,20 +135,22 @@ func generateEntryPoint() []string {
 // exposes custom_ports in format "host:docker" or map 1-to-1 if only "host" port is provided
 func generatePortBindings(in *Input) ([]string, nat.PortMap, error) {
 	httpPort := fmt.Sprintf("%s/tcp", DefaultHTTPPort)
-	innerDebuggerPort := fmt.Sprintf("%d/tcp", DefaultDebuggerPort)
+	exposedPorts := []string{httpPort}
 	portBindings := nat.PortMap{
 		nat.Port(httpPort): []nat.PortBinding{
 			{
 				HostIP:   "0.0.0.0",
 				HostPort: strconv.Itoa(in.Node.HTTPPort),
 			},
 		},
-		nat.Port(innerDebuggerPort): []nat.PortBinding{
-			{
-				HostIP:   "0.0.0.0",
-				HostPort: strconv.Itoa(in.Node.DebuggerPort),
-			},
-		},
+	}
+	if os.Getenv("CTF_CLNODE_DLV") == "true" {
+		innerDebuggerPort := fmt.Sprintf("%d/tcp", DefaultDebuggerPort)
+		portBindings[nat.Port(innerDebuggerPort)] = append(portBindings[nat.Port(innerDebuggerPort)], nat.PortBinding{
+			HostIP:   "0.0.0.0",
+			HostPort: strconv.Itoa(in.Node.DebuggerPort),
+		})
+		exposedPorts = append(exposedPorts, strconv.Itoa(DefaultDebuggerPort))
 	}
 	customPorts := make([]string, 0)
 	for _, p := range in.Node.CustomPorts {
@@ -180,7 +182,6 @@ func generatePortBindings(in *Input) ([]string, nat.PortMap, error) {
 			}
 		}
 	}
-	exposedPorts := []string{httpPort, strconv.Itoa(DefaultDebuggerPort)}
 	exposedPorts = append(exposedPorts, customPorts...)
 	return exposedPorts, portBindings, nil
 }

framework/components/simple_node_set/reload.go

@@ -14,7 +14,7 @@ import (
 // this API is discouraged, however, you can use it if nodes require restart or configuration updates, temporarily!
 func UpgradeNodeSet(t *testing.T, in *Input, bc *blockchain.Output, wait time.Duration) (*Output, error) {
 	uniq := fmt.Sprintf("%s-%s-%s", framework.DefaultCTFLogsDir, t.Name(), uuid.NewString()[0:4])
-	if err := framework.WriteAllContainersLogs(uniq); err != nil {
+	if _, err := framework.SaveContainerLogs(uniq); err != nil {
 		return nil, err
 	}
 	_, err := chaos.ExecPumba("rm --volumes=false re2:^node.*|ns-postgresql.*", wait)

framework/config.go

@@ -130,10 +130,6 @@ func Load[X any](t *testing.T) (*X, error) {
 	t.Cleanup(func() {
 		err := Store[X](input)
 		require.NoError(t, err)
-		err = WriteAllContainersLogs(fmt.Sprintf("%s-%s", DefaultCTFLogsDir, t.Name()))
-		require.NoError(t, err)
-		err = checkAllNodeLogErrors()
-		require.NoError(t, err)
 	})
 	// TODO: not all the people have AWS access, sadly enough, uncomment when granted
 	//if os.Getenv(EnvVarAWSSecretsManager) == "true" {

framework/docker.go

@@ -2,12 +2,13 @@ package framework
 
 import (
 	"archive/tar"
+	"bufio"
 	"bytes"
 	"context"
 	"encoding/binary"
 	"fmt"
 	"github.com/docker/docker/api/types/container"
-	filters2 "github.com/docker/docker/api/types/filters"
+	dfilter "github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/docker/go-connections/nat"
 	"github.com/google/uuid"
@@ -17,8 +18,10 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"sync"
+	"testing"
 )
 
 const (
@@ -176,30 +179,70 @@ func (dc *DockerClient) copyToContainer(containerID, sourceFile, targetPath stri
 	return nil
 }
 
-// WriteAllContainersLogs writes all Docker container logs to the default logs directory
-func WriteAllContainersLogs(dir string) error {
+// SearchLogFile searches logfile using regex and return matches or error
+func SearchLogFile(fp string, regex string) ([]string, error) {
+	file, err := os.Open(fp)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	scanner := bufio.NewScanner(file)
+	re, err := regexp.Compile(regex)
+	if err != nil {
+		return nil, err
+	}
+	matches := make([]string, 0)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if re.MatchString(line) {
+			L.Info().Str("Regex", regex).Msg("Log match found")
+			matches = append(matches, line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return matches, err
+	}
+	return matches, nil
+}
+
+func SaveAndCheckLogs(t *testing.T) error {
+	_, err := SaveContainerLogs(fmt.Sprintf("%s-%s", DefaultCTFLogsDir, t.Name()))
+	if err != nil {
+		return err
+	}
+	err = CheckCLNodeContainerErrors()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// SaveContainerLogs writes all Docker container logs to some directory
+func SaveContainerLogs(dir string) ([]string, error) {
 	L.Info().Msg("Writing Docker containers logs")
 	if _, err := os.Stat(dir); os.IsNotExist(err) {
 		if err := os.MkdirAll(dir, 0755); err != nil {
-			return fmt.Errorf("failed to create directory %s: %w", dir, err)
+			return nil, fmt.Errorf("failed to create directory %s: %w", dir, err)
 		}
 	}
 	provider, err := tc.NewDockerProvider()
 	if err != nil {
-		return fmt.Errorf("failed to create Docker provider: %w", err)
+		return nil, fmt.Errorf("failed to create Docker provider: %w", err)
 	}
 	containers, err := provider.Client().ContainerList(context.Background(), container.ListOptions{
 		All: true,
-		Filters: filters2.NewArgs(filters2.KeyValuePair{
+		Filters: dfilter.NewArgs(dfilter.KeyValuePair{
 			Key:   "label",
 			Value: "framework=ctf",
 		}),
 	})
 	if err != nil {
-		return fmt.Errorf("failed to list Docker containers: %w", err)
+		return nil, fmt.Errorf("failed to list Docker containers: %w", err)
 	}
 
 	eg := &errgroup.Group{}
+	logFilePaths := make([]string, 0)
 
 	for _, containerInfo := range containers {
 		eg.Go(func() error {
@@ -217,6 +260,7 @@ func WriteAllContainersLogs(dir string) error {
 				L.Error().Err(err).Str("Container", containerName).Msg("failed to create container log file")
 				return err
 			}
+			logFilePaths = append(logFilePaths, logFilePath)
 			// Parse and write logs
 			header := make([]byte, 8) // Docker stream header is 8 bytes
 			for {
@@ -249,7 +293,10 @@ func WriteAllContainersLogs(dir string) error {
 			return nil
 		})
 	}
-	return eg.Wait()
+	if err := eg.Wait(); err != nil {
+		return nil, err
+	}
+	return logFilePaths, nil
 }
 
 func BuildImageOnce(once *sync.Once, dctx, dfile, nameAndTag string) error {

framework/examples/myproject/chaos_blockchain_evm_test.go

@@ -0,0 +1,59 @@
+package examples
+
+import (
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/rpc"
+	"github.com/stretchr/testify/require"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestBlockchainChaos(t *testing.T) {
+	srcChain := os.Getenv("CTF_CHAOS_SRC_CHAIN_RPC_HTTP_URL")
+	require.NotEmpty(t, srcChain, "source chain RPC must be set")
+	dstChain := os.Getenv("CTF_CHAOS_DST_CHAIN_RPC_HTTP_URL")
+	require.NotEmpty(t, dstChain, "destination chain RPC must be set")
+
+	recoveryIntervalDuration := 120 * time.Second
+
+	testCases := []struct {
+		name       string
+		chainURL   string
+		reorgDepth int
+	}{
+		{
+			name:       "Reorg src with depth: 1",
+			chainURL:   srcChain,
+			reorgDepth: 1,
+		},
+		{
+			name:       "Reorg dst with depth: 1",
+			chainURL:   dstChain,
+			reorgDepth: 1,
+		},
+		{
+			name:       "Reorg src with depth: 5",
+			chainURL:   srcChain,
+			reorgDepth: 5,
+		},
+		{
+			name:       "Reorg dst with depth: 5",
+			chainURL:   dstChain,
+			reorgDepth: 5,
+		},
+	}
+
+	// Run test cases
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Log(tc.name)
+			r := rpc.New(tc.chainURL, nil)
+			err := r.GethSetHead(tc.reorgDepth)
+			require.NoError(t, err)
+			t.Logf("Awaiting chaos recovery: %s", tc.name)
+			time.Sleep(recoveryIntervalDuration)
+
+			// Validate Chainlink product here, use load test assertions
+		})
+	}
+}