sync with main

iljapavlovs · iljapavlovs · commit a84b1cbdf111 · 2024-11-19T05:33:08.000+02:00
diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md
@@ -18,6 +18,7 @@
     - [Components Cleanup](framework/components/cleanup.md)
     - [Components Caching](framework/components/caching.md)
     - [Mocking Services](framework/components/mocking.md)
+    - [Copying Files](framework/copying_files.md)
     - [External Environment](framework/components/external.md)
     - [Secrets]()
     - [Observability Stack](framework/observability/observability_stack.md)
diff --git a/book/src/framework/components/state.md b/book/src/framework/components/state.md
@@ -9,6 +9,7 @@ When deploying a component, you can explicitly configure port ranges if the defa
 Defaults are:
 - [NodeSet](../components/chainlink/nodeset.md) (Node HTTP API): `10000..100XX`
 - [NodeSet](../components/chainlink/nodeset.md) (Node P2P API): `12000..120XX`
+- Shared `PostgreSQL` volume is called `postgresql_data`
 ```
 [nodeset]
   # HTTP API port range start, each new node get port incremented (host machine)
diff --git a/book/src/framework/copying_files.md b/book/src/framework/copying_files.md
@@ -0,0 +1,15 @@
+# Copying Files
+
+You can copy files to containers by using the `ContainerName` from the output and specifying the source `src` and destination `dst` paths.
+
+However, using this API is discouraged and will be **deprecated** in the future, as it violates the principles of "black-box" testing. If your service relies on this functionality, consider designing a configuration or API to address the requirement instead.
+
+```go
+	bc, err := blockchain.NewBlockchainNetwork(&blockchain.Input{
+	    ...
+	})
+	require.NoError(t, err)
+
+	err = dockerClient.CopyFile(bc.ContainerName, "local_file.txt", "/home")
+	require.NoError(t, err)
+```
diff --git a/framework/.changeset/v0.2.6.md b/framework/.changeset/v0.2.6.md
@@ -0,0 +1 @@
+- Reflect DB changes in interactive mode
diff --git a/framework/.changeset/v0.2.7.md b/framework/.changeset/v0.2.7.md
@@ -0,0 +1,3 @@
+- Do not expose P2P URL to prevent confusion
+- Add container names into outputs
+- Add `CopyFile` command for Docker
diff --git a/framework/cmd/interactive.go b/framework/cmd/interactive.go
@@ -52,9 +52,6 @@ func createComponentsFromForm(form *nodeSetForm) error {
 		nspecs := make([]*clnode.Input, 0)
 		for i := 0; i < form.Nodes; i++ {
 			nspecs = append(nspecs, &clnode.Input{
-				DbInput: &postgres.Input{
-					Image: "postgres:15.6",
-				},
 				Node: &clnode.NodeInput{
 					Image: form.CLVersion,
 				},
@@ -64,7 +61,10 @@ func createComponentsFromForm(form *nodeSetForm) error {
 			_, err = simple_node_set.NewSharedDBNodeSet(&simple_node_set.Input{
 				Nodes:        5,
 				OverrideMode: "all",
-				NodeSpecs:    nspecs,
+				DbInput: &postgres.Input{
+					Image: "postgres:15.6",
+				},
+				NodeSpecs: nspecs,
 			}, bc)
 		}
 		err = spinner.New().
diff --git a/framework/components/blockchain/anvil.go b/framework/components/blockchain/anvil.go
@@ -55,8 +55,9 @@ func deployAnvil(in *Input) (*Output, error) {
 		return nil, err
 	}
 	return &Output{
-		UseCache: true,
-		ChainID:  in.ChainID,
+		UseCache:      true,
+		ChainID:       in.ChainID,
+		ContainerName: containerName,
 		Nodes: []*Node{
 			{
 				HostWSUrl:             fmt.Sprintf("ws://%s:%s", host, mp.Port()),
diff --git a/framework/components/blockchain/blockchain.go b/framework/components/blockchain/blockchain.go
@@ -17,9 +17,10 @@ type Input struct {
 
 // Output is a blockchain network output, ChainID and one or more nodes that forms the network
 type Output struct {
-	UseCache bool    `toml:"use_cache"`
-	ChainID  string  `toml:"chain_id"`
-	Nodes    []*Node `toml:"nodes"`
+	UseCache      bool    `toml:"use_cache"`
+	ContainerName string  `toml:"container_name"`
+	ChainID       string  `toml:"chain_id"`
+	Nodes         []*Node `toml:"nodes"`
 }
 
 // Node represents blockchain node output, URLs required for connection locally and inside docker network
diff --git a/framework/components/clnode/clnode.go b/framework/components/clnode/clnode.go
@@ -67,8 +67,8 @@ type Output struct {
 type NodeOut struct {
 	APIAuthUser     string `toml:"api_auth_user"`
 	APIAuthPassword string `toml:"api_auth_password"`
+	ContainerName   string `toml:"container_name"`
 	HostURL         string `toml:"url"`
-	HostP2PURL      string `toml:"p2p_url"`
 	DockerURL       string `toml:"docker_internal_url"`
 	DockerP2PUrl    string `toml:"p2p_docker_internal_url"`
 }
@@ -124,7 +124,7 @@ func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
 	if err != nil {
 		return nil, err
 	}
-	cfgPath, err := writeDefaultConfig(in)
+	cfgPath, err := writeDefaultConfig()
 	if err != nil {
 		return nil, err
 	}
@@ -150,7 +150,6 @@ func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
 	}
 
 	httpPort := fmt.Sprintf("%s/tcp", DefaultHTTPPort)
-	p2pPort := fmt.Sprintf("%s/udp", DefaultP2PPort)
 	var containerName string
 	if in.Node.Name != "" {
 		containerName = in.Node.Name
@@ -161,7 +160,7 @@ func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
 	for _, p := range in.Node.CustomPorts {
 		customPorts = append(customPorts, fmt.Sprintf("%d/tcp", p))
 	}
-	exposedPorts := []string{httpPort, p2pPort}
+	exposedPorts := []string{httpPort}
 	exposedPorts = append(exposedPorts, customPorts...)
 
 	portBindings := nat.PortMap{
@@ -171,12 +170,6 @@ func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
 				HostPort: fmt.Sprintf("%d/tcp", in.Node.HTTPPort),
 			},
 		},
-		nat.Port(p2pPort): []nat.PortBinding{
-			{
-				HostIP:   "0.0.0.0",
-				HostPort: fmt.Sprintf("%d/udp", in.Node.P2PPort),
-			},
-		},
 	}
 	for _, p := range customPorts {
 		portBindings[nat.Port(p)] = []nat.PortBinding{
@@ -286,13 +279,12 @@ func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
 	}
 
 	mp := nat.Port(fmt.Sprintf("%d/tcp", in.Node.HTTPPort))
-	mpP2P := nat.Port(fmt.Sprintf("%d/udp", in.Node.P2PPort))
 
 	return &NodeOut{
 		APIAuthUser:     DefaultAPIUser,
 		APIAuthPassword: DefaultAPIPassword,
+		ContainerName:   containerName,
 		HostURL:         fmt.Sprintf("http://%s:%s", host, mp.Port()),
-		HostP2PURL:      fmt.Sprintf("http://%s:%s", host, mpP2P.Port()),
 		DockerURL:       fmt.Sprintf("http://%s:%s", containerName, DefaultHTTPPort),
 		DockerP2PUrl:    fmt.Sprintf("http://%s:%s", containerName, DefaultP2PPort),
 	}, nil
@@ -303,7 +295,7 @@ type DefaultCLNodeConfig struct {
 	SecureCookies bool
 }
 
-func generateDefaultConfig(in *Input) (string, error) {
+func generateDefaultConfig() (string, error) {
 	config := DefaultCLNodeConfig{
 		HTTPPort:      DefaultHTTPPort,
 		SecureCookies: false,
@@ -351,8 +343,8 @@ func writeDefaultSecrets(pgOut *postgres.Output) (*os.File, error) {
 	return WriteTmpFile(secretsOverrides, "secrets.toml")
 }
 
-func writeDefaultConfig(in *Input) (*os.File, error) {
-	cfg, err := generateDefaultConfig(in)
+func writeDefaultConfig() (*os.File, error) {
+	cfg, err := generateDefaultConfig()
 	if err != nil {
 		return nil, err
 	}
diff --git a/framework/components/clnode/clnode_test.go b/framework/components/clnode/clnode_test.go
@@ -23,7 +23,7 @@ func checkBasicOutputs(t *testing.T, output *clnode.Output) {
 	require.Contains(t, output.Node.DockerP2PUrl, "cl-node")
 	require.NotNil(t, output.PostgreSQL)
 	require.Contains(t, output.PostgreSQL.Url, "postgresql://chainlink:thispasswordislongenough@127.0.0.1")
-	require.Contains(t, output.PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@postgresql-")
+	require.Contains(t, output.PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@ns-postgresql")
 }
 
 func TestComponentDockerNodeWithSharedDB(t *testing.T) {
diff --git a/framework/components/postgres/postgres.go b/framework/components/postgres/postgres.go
@@ -38,6 +38,7 @@ type Input struct {
 
 type Output struct {
 	Url               string `toml:"url"`
+	ContainerName     string `toml:"container_name"`
 	DockerInternalURL string `toml:"docker_internal_url"`
 }
 
@@ -134,6 +135,7 @@ func NewPostgreSQL(in *Input) (*Output, error) {
 		return nil, err
 	}
 	return &Output{
+		ContainerName: containerName,
 		DockerInternalURL: fmt.Sprintf(
 			"postgresql://%s:%s@%s:%s/%s?sslmode=disable",
 			User,
diff --git a/framework/components/simple_node_set/node_set.go b/framework/components/simple_node_set/node_set.go
@@ -68,7 +68,6 @@ func printURLs(out *Output) {
 	httpURLs, p2pURLs, pgURLs := make([]string, 0), make([]string, 0), make([]string, 0)
 	for _, n := range out.CLNodes {
 		httpURLs = append(httpURLs, n.Node.HostURL)
-		p2pURLs = append(p2pURLs, n.Node.HostP2PURL)
 		pgURLs = append(pgURLs, n.PostgreSQL.Url)
 	}
 	framework.L.Info().Any("UI", httpURLs).Send()
diff --git a/framework/components/simple_node_set/nodeset_test.go b/framework/components/simple_node_set/nodeset_test.go
@@ -13,7 +13,6 @@ import (
 
 type testCase struct {
 	name         string
-	fakeURL      string
 	funding      float64
 	bcInput      *blockchain.Input
 	nodeSetInput *ns.Input
@@ -25,13 +24,13 @@ func checkBasicOutputs(t *testing.T, output *ns.Output) {
 	require.NotNil(t, output.CLNodes)
 	require.Len(t, output.CLNodes, 2)
 	require.Contains(t, output.CLNodes[0].PostgreSQL.Url, "postgresql://chainlink:thispasswordislongenough@127.0.0.1")
-	require.Contains(t, output.CLNodes[0].PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@postgresql-")
+	require.Contains(t, output.CLNodes[0].PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@ns-postgresql-")
 	require.Contains(t, output.CLNodes[0].Node.HostURL, "127.0.0.1")
 	require.Contains(t, output.CLNodes[0].Node.DockerURL, "node")
 	require.Contains(t, output.CLNodes[0].Node.DockerP2PUrl, "node")
 
 	require.Contains(t, output.CLNodes[1].PostgreSQL.Url, "postgresql://chainlink:thispasswordislongenough@127.0.0.1")
-	require.Contains(t, output.CLNodes[1].PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@postgresql-")
+	require.Contains(t, output.CLNodes[1].PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@ns-postgresql-")
 	require.Contains(t, output.CLNodes[1].Node.HostURL, "127.0.0.1")
 	require.Contains(t, output.CLNodes[1].Node.DockerURL, "node")
 	require.Contains(t, output.CLNodes[1].Node.DockerP2PUrl, "node")
@@ -40,8 +39,7 @@ func checkBasicOutputs(t *testing.T, output *ns.Output) {
 func TestComponentDockerNodeSetSharedDB(t *testing.T) {
 	testCases := []testCase{
 		{
-			name:    "2 nodes cluster, override mode 'all'",
-			fakeURL: "http://example.com",
+			name: "2 nodes cluster, override mode 'all'",
 			bcInput: &blockchain.Input{
 				Type:    "anvil",
 				Image:   "f4hrenh9it/foundry",
@@ -56,9 +54,6 @@ func TestComponentDockerNodeSetSharedDB(t *testing.T) {
 				},
 				NodeSpecs: []*clnode.Input{
 					{
-						DbInput: &postgres.Input{
-							Image: "postgres:15.6",
-						},
 						Node: &clnode.NodeInput{
 							Image: "public.ecr.aws/chainlink/chainlink:v2.17.0",
 							Name:  "cl-node",
@@ -71,8 +66,7 @@ func TestComponentDockerNodeSetSharedDB(t *testing.T) {
 			},
 		},
 		{
-			name:    "2 nodes cluster, override mode 'each'",
-			fakeURL: "http://example.com",
+			name: "2 nodes cluster, override mode 'each'",
 			bcInput: &blockchain.Input{
 				Type:    "anvil",
 				Image:   "f4hrenh9it/foundry",
@@ -90,10 +84,6 @@ func TestComponentDockerNodeSetSharedDB(t *testing.T) {
 				},
 				NodeSpecs: []*clnode.Input{
 					{
-						DbInput: &postgres.Input{
-							Image: "postgres:15.6",
-							Port:  14000,
-						},
 						Node: &clnode.NodeInput{
 							Image: "public.ecr.aws/chainlink/chainlink:v2.17.0",
 							Name:  "cl-node-1",
@@ -104,9 +94,6 @@ level = 'info'
 						},
 					},
 					{
-						DbInput: &postgres.Input{
-							Image: "postgres:15.6",
-						},
 						Node: &clnode.NodeInput{
 							Image: "public.ecr.aws/chainlink/chainlink:v2.17.0",
 							Name:  "cl-node-2",
diff --git a/framework/docker.go b/framework/docker.go
@@ -1,13 +1,17 @@
 package framework
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/client"
 	"github.com/docker/go-connections/nat"
 	"github.com/google/uuid"
 	tc "github.com/testcontainers/testcontainers-go"
+	"io"
 	"os"
 	"os/exec"
 	"strings"
@@ -132,3 +136,86 @@ func RebuildDockerImage(once *sync.Once, dockerfile string, buildContext string,
 	}
 	return fmt.Sprintf("localhost:5050/%s:latest", imageName), nil
 }
+
+// DockerClient wraps a Docker API client and provides convenience methods
+type DockerClient struct {
+	cli *client.Client
+}
+
+// NewDockerClient creates a new instance of DockerClient
+func NewDockerClient() (*DockerClient, error) {
+	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Docker client: %w", err)
+	}
+	return &DockerClient{cli: cli}, nil
+}
+
+// CopyFile copies a file into a container by name
+func (dc *DockerClient) CopyFile(containerName, sourceFile, targetPath string) error {
+	ctx := context.Background()
+	containerID, err := dc.findContainerIDByName(ctx, containerName)
+	if err != nil {
+		return fmt.Errorf("failed to find container ID by name: %s", containerName)
+	}
+	return dc.copyToContainer(containerID, sourceFile, targetPath)
+}
+
+// findContainerIDByName finds a container ID by its name
+func (dc *DockerClient) findContainerIDByName(ctx context.Context, containerName string) (string, error) {
+	containers, err := dc.cli.ContainerList(ctx, container.ListOptions{
+		All: true,
+	})
+	if err != nil {
+		return "", fmt.Errorf("failed to list containers: %w", err)
+	}
+	for _, c := range containers {
+		for _, name := range c.Names {
+			if name == "/"+containerName {
+				return c.ID, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("container with name %s not found", containerName)
+}
+
+// copyToContainer copies a file into a container
+func (dc *DockerClient) copyToContainer(containerID, sourceFile, targetPath string) error {
+	ctx := context.Background()
+	src, err := os.Open(sourceFile)
+	if err != nil {
+		return fmt.Errorf("could not open source file: %w", err)
+	}
+	defer src.Close()
+
+	// Create a tar archive containing the file
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+	info, err := src.Stat()
+	if err != nil {
+		return fmt.Errorf("could not stat source file: %w", err)
+	}
+
+	// Add file to tar
+	header := &tar.Header{
+		Name: info.Name(),
+		Size: info.Size(),
+		Mode: int64(info.Mode()),
+	}
+	if err := tw.WriteHeader(header); err != nil {
+		return fmt.Errorf("could not write tar header: %w", err)
+	}
+	if _, err := io.Copy(tw, src); err != nil {
+		return fmt.Errorf("could not write file to tar archive: %w", err)
+	}
+	tw.Close()
+
+	// Copy the tar archive to the container
+	err = dc.cli.CopyToContainer(ctx, containerID, targetPath, &buf, container.CopyToContainerOptions{
+		AllowOverwriteDirWithFile: true,
+	})
+	if err != nil {
+		return fmt.Errorf("could not copy file to container: %w", err)
+	}
+	return nil
+}
diff --git a/framework/examples/myproject/.envrc b/framework/examples/myproject/.envrc
@@ -1,3 +1,4 @@
 export LOKI_TENANT_ID=promtail
 export LOKI_URL=http://localhost:3030/loki/api/v1/push
 export TESTCONTAINERS_RYUK_DISABLED=true
+export CTF_LOG_LEVEL=info
diff --git a/framework/examples/myproject/go.mod b/framework/examples/myproject/go.mod
diff --git a/framework/examples/myproject/smoke_test.go b/framework/examples/myproject/smoke_test.go
diff --git a/framework/examples/myproject/upgrade_test.go b/framework/examples/myproject/upgrade_test.go
diff --git a/tools/flakeguard/cmd/aggregate_all.go b/tools/flakeguard/cmd/aggregate_all.go
diff --git a/tools/flakeguard/cmd/aggregate_failed.go b/tools/flakeguard/cmd/aggregate_failed.go
diff --git a/tools/flakeguard/reports/reports.go b/tools/flakeguard/reports/reports.go

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+- Reflect DB changes in interactive mode`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+- Do not expose P2P URL to prevent confusion`
	`2`	`+- Add container names into outputs`
	`3`	+- Add `CopyFile` command for Docker
Original file line number	Diff line number	Diff line change
`@@ -55,8 +55,9 @@ func deployAnvil(in Input) (Output, error) {`
`55`	`55`	`return nil, err`
`56`	`56`	`}`
`57`	`57`	`return &Output{`
`58`		`- UseCache: true,`
`59`		`- ChainID: in.ChainID,`
	`58`	`+ UseCache: true,`
	`59`	`+ ChainID: in.ChainID,`
	`60`	`+ ContainerName: containerName,`
`60`	`61`	`Nodes: []*Node{`
`61`	`62`	`{`
`62`	`63`	`HostWSUrl: fmt.Sprintf("ws://%s:%s", host, mp.Port()),`
Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,7 @@ func checkBasicOutputs(t testing.T, output clnode.Output) {`
`23`	`23`	`require.Contains(t, output.Node.DockerP2PUrl, "cl-node")`
`24`	`24`	`require.NotNil(t, output.PostgreSQL)`
`25`	`25`	`require.Contains(t, output.PostgreSQL.Url, "postgresql://chainlink:[email protected]")`
`26`		`- require.Contains(t, output.PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@postgresql-")`
	`26`	`+ require.Contains(t, output.PostgreSQL.DockerInternalURL, "postgresql://chainlink:thispasswordislongenough@ns-postgresql")`
`27`	`27`	`}`
`28`	`28`
`29`	`29`	`func TestComponentDockerNodeWithSharedDB(t *testing.T) {`