Merge branch 'main' of github.com:smartcontractkit/chainlink-testing-framework into flakeguardPRFix

Author: kalverra

framework/.changeset/v0.4.2.md

@@ -0,0 +1,3 @@
+- Name logs subdir as test name
+- Mutex for concurrent mock Record access
+- Remove Promtail only through CLI

framework/cmd/observability.go

@@ -30,7 +30,7 @@ func observabilityDown() error {
 	framework.L.Info().Msg("Removing local observability stack")
 	err := runCommand("bash", "-c", fmt.Sprintf(`
 		cd %s && \
-		docker compose down -v
+		docker compose down -v && docker rm -f promtail
 	`, "compose"))
 	if err != nil {
 		return err

framework/components/fake/record.go

@@ -6,10 +6,12 @@ import (
 	"github.com/gin-gonic/gin"
 	"io/ioutil"
 	"net/http"
+	"sync"
 )
 
 var (
-	R = NewRecords()
+	R  = NewRecords()
+	mu sync.Mutex
 )
 
 // Record is a request and response data
@@ -78,6 +80,8 @@ func recordMiddleware() gin.HandlerFunc {
 		// Capture response data
 		resBody := customWriter.body.String()
 		status := c.Writer.Status()
+		mu.Lock()
+		defer mu.Unlock()
 		if R.Data[c.Request.URL.Path] == nil {
 			R.Data[c.Request.URL.Path] = make([]*Record, 0)
 		}

framework/components/simple_node_set/reload.go

@@ -6,13 +6,14 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/chaos"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
+	"testing"
 	"time"
 )
 
 // UpgradeNodeSet updates nodes configuration TOML files
 // this API is discouraged, however, you can use it if nodes require restart or configuration updates, temporarily!
-func UpgradeNodeSet(in *Input, bc *blockchain.Output, wait time.Duration) (*Output, error) {
-	uniq := fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, uuid.NewString()[0:4])
+func UpgradeNodeSet(t *testing.T, in *Input, bc *blockchain.Output, wait time.Duration) (*Output, error) {
+	uniq := fmt.Sprintf("%s-%s-%s", framework.DefaultCTFLogsDir, t.Name(), uuid.NewString()[0:4])
 	if err := framework.WriteAllContainersLogs(uniq); err != nil {
 		return nil, err
 	}

framework/config.go

@@ -130,7 +130,7 @@ func Load[X any](t *testing.T) (*X, error) {
 	t.Cleanup(func() {
 		err := Store[X](input)
 		require.NoError(t, err)
-		err = WriteAllContainersLogs(DefaultCTFLogsDir)
+		err = WriteAllContainersLogs(fmt.Sprintf("%s-%s", DefaultCTFLogsDir, t.Name()))
 		require.NoError(t, err)
 		err = checkAllNodeLogErrors()
 		require.NoError(t, err)

framework/docker.go

@@ -7,6 +7,7 @@ import (
 	"encoding/binary"
 	"fmt"
 	"github.com/docker/docker/api/types/container"
+	filters2 "github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/docker/go-connections/nat"
 	"github.com/google/uuid"
@@ -175,23 +176,6 @@ func (dc *DockerClient) copyToContainer(containerID, sourceFile, targetPath stri
 	return nil
 }
 
-func in(s string, substrings []string) bool {
-	for _, substr := range substrings {
-		if strings.Contains(s, substr) {
-			return true
-		}
-	}
-	return false
-}
-
-func isLocalToolDockerContainer(containerName string) bool {
-	if in(containerName, []string{"/sig-provider", "/stats", "/stats-db", "/db", "/backend", "/promtail", "/compose", "/blockscout", "/frontend", "/user-ops-indexer", "/visualizer", "/redis-db", "/proxy"}) {
-		L.Debug().Str("Container", containerName).Msg("Ignoring local tool container output")
-		return true
-	}
-	return false
-}
-
 // WriteAllContainersLogs writes all Docker container logs to the default logs directory
 func WriteAllContainersLogs(dir string) error {
 	L.Info().Msg("Writing Docker containers logs")
@@ -204,7 +188,13 @@ func WriteAllContainersLogs(dir string) error {
 	if err != nil {
 		return fmt.Errorf("failed to create Docker provider: %w", err)
 	}
-	containers, err := provider.Client().ContainerList(context.Background(), container.ListOptions{All: true})
+	containers, err := provider.Client().ContainerList(context.Background(), container.ListOptions{
+		All: true,
+		Filters: filters2.NewArgs(filters2.KeyValuePair{
+			Key:   "label",
+			Value: "framework=ctf",
+		}),
+	})
 	if err != nil {
 		return fmt.Errorf("failed to list Docker containers: %w", err)
 	}
@@ -214,9 +204,6 @@ func WriteAllContainersLogs(dir string) error {
 	for _, containerInfo := range containers {
 		eg.Go(func() error {
 			containerName := containerInfo.Names[0]
-			if isLocalToolDockerContainer(containerName) {
-				return nil
-			}
 			L.Debug().Str("Container", containerName).Msg("Collecting logs")
 			logOptions := container.LogsOptions{ShowStdout: true, ShowStderr: true}
 			logs, err := provider.Client().ContainerLogs(context.Background(), containerInfo.ID, logOptions)

framework/examples/myproject/go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/blocto/solana-go-sdk v1.30.0
 	github.com/ethereum/go-ethereum v1.14.11
 	github.com/go-resty/resty/v2 v2.15.3
-	github.com/smartcontractkit/chainlink-testing-framework/framework v0.0.0-00010101000000-000000000000
+	github.com/smartcontractkit/chainlink-testing-framework/framework v0.4.1
 	github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.10
 	github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.2
 	github.com/stretchr/testify v1.10.0

framework/examples/myproject/upgrade_test.go

@@ -66,7 +66,7 @@ func TestUpgrade(t *testing.T) {
 											level = 'info'
 	`
 
-	out, err = ns.UpgradeNodeSet(in.NodeSet, bc, 3*time.Second)
+	out, err = ns.UpgradeNodeSet(t, in.NodeSet, bc, 3*time.Second)
 	require.NoError(t, err)
 
 	jobs, _, err := c[0].ReadJobs()

framework/promtail.go

@@ -127,7 +127,6 @@ func NewPromtail() error {
 		ExposedPorts: []string{"9080/tcp"},
 		Name:         "promtail",
 		Cmd:          cmd,
-		Labels:       DefaultTCLabels(),
 		Files: []testcontainers.ContainerFile{
 			{
 				HostFilePath:      pcn,

wasp/benchspy/basic.go

@@ -121,6 +121,13 @@ func (b *BasicData) Validate() error {
 		return errors.New("test end time is missing. We cannot query Loki without a time range. Please set it and try again")
 	}
 
+	if b.TestEnd.Before(b.TestStart) {
+		return errors.New("test end time is before test start time. Please set valid times and try again")
+	}
+	if b.TestEnd.Sub(b.TestStart) < time.Second {
+		return errors.New("test duration is less than a second. Please set a valid time range and try again")
+	}
+
 	if len(b.GeneratorConfigs) == 0 {
 		return errors.New("generator configs are missing. At least one is required. Please set them and try again")
 	}