stateful db for chaos tests

Author: skudasov

book/src/framework/first_test.md

@@ -55,6 +55,6 @@ Summary:
 - We defined configuration for `BlockchainNetwork`
 - We've used one CTF component in test and checked if it's working
 
-You can learn more about [component design](./components/overview.md) or proceed with another example of [connecting Chainlink node](./connecting_chainlink_node.md)
+Now let's connect the [Chainlink](./connecting_chainlink_node.md) node!
 
 Learn more about [anvil](./components/blockchains/anvil.md) component.

book/src/framework/nodeset_environment.md

@@ -72,13 +72,9 @@ Run it
 CTF_CONFIGS=smoke.toml go test -v -run TestNodeSet
 ```
 
-You'll see something like, use any URL to access CL node
+Check the logs to access the UI
 ```bash
-6:14PM INF Chainlink node url URL=http://127.0.0.1:34041
-6:14PM INF Chainlink node url URL=http://127.0.0.1:34045
-6:14PM INF Chainlink node url URL=http://127.0.0.1:34044
-6:14PM INF Chainlink node url URL=http://127.0.0.1:34042
-6:14PM INF Chainlink node url URL=http://127.0.0.1:34043
+12:41AM INF UI=["http://127.0.0.1:10000","http://127.0.0.1:10001", ...]
 ```
 
 Use credentials to authorize:

framework/cmd/blockscout.go

@@ -18,7 +18,6 @@ func blockscoutUp() error {
 	if err != nil {
 		return err
 	}
-	framework.L.Info().Msg("Done")
 	fmt.Println()
 	framework.L.Info().Msgf("Blockscout is up at: %s", "http://localhost")
 	return nil
@@ -40,6 +39,5 @@ func blockscoutDown() error {
 		rm -rf redis-data && \
 		rm -rf stats-db-data
 	`, filepath.Join("blockscout", "services")))
-	framework.L.Info().Msg("Done")
 	return nil
 }

framework/cmd/docker.go

@@ -24,7 +24,6 @@ func cleanDockerResources() error {
 	if err != nil {
 		return fmt.Errorf("error running clean command: %s", string(output))
 	}
-	framework.L.Info().Msgf("Done")
 	return nil
 }
 

framework/cmd/interactive.go

@@ -131,14 +131,14 @@ Docker Desktop (https://www.docker.com/products/docker-desktop/)
 				Options(
 					huh.NewOption("Anvil", "anvil"),
 				).
-				Value(&f.Network), // stores the selected network
+				Value(&f.Network),
 
 			huh.NewSelect[int]().
 				Title("How many nodes do you need?").
 				Options(
 					huh.NewOption("5", 5),
 				).
-				Value(&f.Nodes), // stores the selected number of nodes
+				Value(&f.Nodes),
 
 			huh.NewSelect[string]().
 				Title("Choose Chainlink node version").
@@ -147,11 +147,11 @@ Docker Desktop (https://www.docker.com/products/docker-desktop/)
 				Value(&f.CLVersion),
 			huh.NewConfirm().
 				Title("Do you need to spin up an observability stack?").
-				Value(&f.Observability), // stores the observability option
+				Value(&f.Observability),
 
 			huh.NewConfirm().
 				Title("Do you need to spin up a Blockscout stack?").
-				Value(&f.Blockscout), // stores the Blockscout option
+				Value(&f.Blockscout),
 		),
 	)
 
@@ -167,13 +167,12 @@ Docker Desktop (https://www.docker.com/products/docker-desktop/)
 	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
 	go func() {
 		<-sigs
-		fmt.Println("\nReceived Ctrl+C, starting custom cleanup...")
 		err := cleanup(f)
 		if err != nil {
 			panic(err)
 		}
 		os.Exit(0)
 	}()
-	framework.L.Info().Msg("Press Ctrl+C to remove the stack..")
+	framework.L.Info().Msg("Services are up! Press Ctrl+C to remove them..")
 	select {}
 }

framework/cmd/observability.go

@@ -17,11 +17,8 @@ func observabilityUp() error {
 	if err != nil {
 		return err
 	}
-	framework.L.Info().Msg("Done")
 	fmt.Println()
 	framework.L.Info().Msgf("Loki: %s", LocalLogsURL)
-	framework.L.Info().Msgf("All logs: %s", "{job=\"ctf\"}")
-	framework.L.Info().Msgf("By log level: %s", "{job=\"ctf\", container=~\"node.*\"} |= \"WARN|INFO|DEBUG\"")
 	framework.L.Info().Msgf("Pyroscope: %s", LocalPyroScopeURL)
 	return nil
 }
@@ -35,6 +32,5 @@ func observabilityDown() error {
 	if err != nil {
 		return err
 	}
-	framework.L.Info().Msg("Done")
 	return nil
 }

framework/components/postgres/postgres.go

@@ -3,25 +3,29 @@ package postgres
 import (
 	"context"
 	"fmt"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/go-connections/nat"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"github.com/testcontainers/testcontainers-go"
 	tcwait "github.com/testcontainers/testcontainers-go/wait"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 )
 
 const (
-	User      = "chainlink"
-	Password  = "thispasswordislongenough"
-	Port      = "5432"
-	Database  = "chainlink"
-	Databases = 5
+	User              = "chainlink"
+	Password          = "thispasswordislongenough"
+	Port              = "5432"
+	ExposedStaticPort = "13000"
+	Database          = "chainlink"
 )
 
 type Input struct {
 	Image     string  `toml:"image" validate:"required"`
+	Databases int     `toml:"databases"`
 	PullImage bool    `toml:"pull_image"`
 	Out       *Output `toml:"out"`
 }
@@ -35,11 +39,10 @@ func NewPostgreSQL(in *Input) (*Output, error) {
 	ctx := context.Background()
 
 	bindPort := fmt.Sprintf("%s/tcp", Port)
-
 	containerName := framework.DefaultTCName("postgresql")
 
 	var sqlCommands []string
-	for i := 0; i <= Databases; i++ {
+	for i := 0; i <= in.Databases; i++ {
 		sqlCommands = append(sqlCommands, fmt.Sprintf("CREATE DATABASE db_%d;", i))
 	}
 	sqlCommands = append(sqlCommands, "ALTER USER chainlink WITH SUPERUSER;")
@@ -82,7 +85,29 @@ func NewPostgreSQL(in *Input) (*Output, error) {
 		},
 		WaitingFor: tcwait.ForExec([]string{"psql", "-h", "127.0.0.1",
 			"-U", User, "-p", Port, "-c", "select", "1", "-d", Database}).
-			WithStartupTimeout(20 * time.Second),
+			WithStartupTimeout(20 * time.Second).
+			WithPollInterval(1 * time.Second),
+	}
+	wd, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+	req.HostConfigModifier = func(h *container.HostConfig) {
+		h.PortBindings = nat.PortMap{
+			nat.Port(bindPort): []nat.PortBinding{
+				{
+					HostIP:   "0.0.0.0",
+					HostPort: fmt.Sprintf("%s/tcp", ExposedStaticPort),
+				},
+			},
+		}
+		h.Mounts = []mount.Mount{
+			{
+				Type:   mount.TypeBind,
+				Source: filepath.Join(wd, "postgresql_data"),
+				Target: "/var/lib/postgresql/data",
+			},
+		}
 	}
 	c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
 		ContainerRequest: req,
@@ -95,10 +120,6 @@ func NewPostgreSQL(in *Input) (*Output, error) {
 	if err != nil {
 		return nil, err
 	}
-	mp, err := c.MappedPort(ctx, nat.Port(bindPort))
-	if err != nil {
-		return nil, err
-	}
 	return &Output{
 		DockerInternalURL: fmt.Sprintf(
 			"postgresql://%s:%s@%s:%s/%s?sslmode=disable",
@@ -113,7 +134,7 @@ func NewPostgreSQL(in *Input) (*Output, error) {
 			User,
 			Password,
 			host,
-			mp.Port(),
+			ExposedStaticPort,
 			Database,
 		),
 	}, nil

framework/components/simple_node_set/node_set.go

@@ -7,6 +7,7 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/clnode"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/postgres"
 	"golang.org/x/sync/errgroup"
+	"slices"
 	"strings"
 	"sync"
 )
@@ -16,20 +17,24 @@ const (
 	DefaultP2PStaticRangeStart      = 12000
 )
 
+// Input is a node set configuration input
 type Input struct {
 	Nodes              int             `toml:"nodes" validate:"required"`
 	HTTPPortRangeStart int             `toml:"http_port_range_start"`
 	P2PPortRangeStart  int             `toml:"p2p_port_range_start"`
 	OverrideMode       string          `toml:"override_mode" validate:"required,oneof=all each"`
-	NodeSpecs          []*clnode.Input `toml:"node_specs"`
+	NodeSpecs          []*clnode.Input `toml:"node_specs" validate:"required"`
 	Out                *Output         `toml:"out"`
 }
 
+// Output is a node set configuration output, used for caching or external components
 type Output struct {
 	UseCache bool             `toml:"use_cache"`
 	CLNodes  []*clnode.Output `toml:"cl_nodes"`
 }
 
+// NewSharedDBNodeSet create a new node set with a shared database instance
+// all the nodes have their own isolated database
 func NewSharedDBNodeSet(in *Input, bcOut *blockchain.Output, fakeUrl string) (*Output, error) {
 	if in.Out != nil && in.Out.UseCache {
 		return in.Out, nil
@@ -39,6 +44,7 @@ func NewSharedDBNodeSet(in *Input, bcOut *blockchain.Output, fakeUrl string) (*O
 		err error
 	)
 	defer func() {
+		printURLs(out)
 		in.Out = out
 	}()
 	if len(in.NodeSpecs) != in.Nodes && in.OverrideMode == "each" {
@@ -56,23 +62,29 @@ func NewSharedDBNodeSet(in *Input, bcOut *blockchain.Output, fakeUrl string) (*O
 			return nil, err
 		}
 	}
-	printOut(out)
 	return out, nil
 }
 
-func printOut(out *Output) {
-	for i, n := range out.CLNodes {
-		framework.L.Info().Str(fmt.Sprintf("Node-%d", i), n.Node.HostURL).Msg("Chainlink node url")
+func printURLs(out *Output) {
+	httpURLs, p2pURLs := make([]string, 0), make([]string, 0)
+	for _, n := range out.CLNodes {
+		httpURLs = append(httpURLs, n.Node.HostURL)
+		p2pURLs = append(p2pURLs, n.Node.HostP2PURL)
 	}
+	framework.L.Info().Any("UI", httpURLs).Send()
+	framework.L.Debug().Any("P2P", p2pURLs).Send()
 }
 
 func sharedDBSetup(in *Input, bcOut *blockchain.Output, fakeUrl string, overrideEach bool) (*Output, error) {
+	in.NodeSpecs[0].DbInput.Databases = in.Nodes
 	dbOut, err := postgres.NewPostgreSQL(in.NodeSpecs[0].DbInput)
 	if err != nil {
 		return nil, err
 	}
 	nodeOuts := make([]*clnode.Output, 0)
 
+	// to make it easier for chaos testing we use static ports
+	// there is no need to check them in advance since testcontainers-go returns a nice error
 	var (
 		httpPortRangeStart = DefaultHTTPPortStaticRangeStart
 		p2pPortRangeStart  = DefaultP2PStaticRangeStart
@@ -144,8 +156,21 @@ func sharedDBSetup(in *Input, bcOut *blockchain.Output, fakeUrl string, override
 	if err := eg.Wait(); err != nil {
 		return nil, err
 	}
+	sortNodeOutsByHostPort(nodeOuts)
 	return &Output{
 		UseCache: true,
 		CLNodes:  nodeOuts,
 	}, nil
 }
+
+func sortNodeOutsByHostPort(nodes []*clnode.Output) {
+	slices.SortFunc[[]*clnode.Output, *clnode.Output](nodes, func(a, b *clnode.Output) int {
+		aa := strings.Split(a.Node.HostURL, ":")
+		bb := strings.Split(b.Node.HostURL, ":")
+		if aa[len(aa)-1] < bb[len(bb)-1] {
+			return -1
+		} else {
+			return 1
+		}
+	})
+}

framework/examples/.gitignore

@@ -33,3 +33,5 @@ promtail-config.yml
 **/blockscout
 **/blockscout/*
 **/*cache.toml
+**/postgresql_data
+**/postgresql_data/*