Blockscout multinet (#1398)

skudasov · web-flow · commit 24b91498a69e · 2024-11-27T20:48:17.000+01:00
* blockscout urls and docs, run logs download in parallel and ignore local stuff

* remove unused vars

* blockscout docs

* changelog
diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md
@@ -23,6 +23,7 @@
     - [Mocking Services](framework/components/mocking.md)
     - [Copying Files](framework/copying_files.md)
     - [External Environment](framework/components/external.md)
+    - [Troubleshooting](framework/components/troubleshooting.md)
     - [Secrets]()
     - [Observability Stack](framework/observability/observability_stack.md)
       - [Metrics](framework/observability/metrics.md)
diff --git a/book/src/framework/components/troubleshooting.md b/book/src/framework/components/troubleshooting.md
@@ -0,0 +1,12 @@
+# Troubleshooting
+
+## Can't run `anvil`, issue with `Rosetta`
+```
+2024/11/27 15:20:27 ⏳ Waiting for container id 79f8a68c07cc image: f4hrenh9it/foundry:latest. Waiting for: &{Port:8546 timeout:0x14000901278 PollInterval:100ms skipInternalCheck:false}
+2024/11/27 15:20:27 container logs (all exposed ports, [8546/tcp], were not mapped in 5s: port 8546/tcp is not mapped yet
+wait until ready: check target: retries: 1, port: "", last err: container exited with code 133):
+rosetta error: Rosetta is only intended to run on Apple Silicon with a macOS host using Virtualization.framework with Rosetta mode enabled
+```
+#### Solution
+
+Update your docker to `Docker version 27.3.1, build ce12230`
diff --git a/book/src/framework/getting_started.md b/book/src/framework/getting_started.md
@@ -4,6 +4,12 @@
 - `Docker` [OrbStack](https://orbstack.dev/) or [Docker Desktop](https://www.docker.com/products/docker-desktop/), we recommend OrbStack (faster, smaller memory footprint)
 - [Golang](https://go.dev/doc/install)
 
+Tested with
+```
+Docker version 27.3.1
+OrbStack Version: 1.8.2 (1080200)
+```
+
 ## Test setup
 
 To start writing tests create a directory for your project with `go.mod` and add a package
diff --git a/book/src/framework/observability/blockscout.md b/book/src/framework/observability/blockscout.md
@@ -12,6 +12,14 @@ To remove it, we also clean up all Blockscout databases to prevent stale data wh
 ctf bs down
 ```
 
+## Selecting Blockchain Node
+
+By default, we connect to the first `anvil` node, but you can select the node explicitly
+```
+ctf bs -r http://host.docker.internal:8545 d
+ctf bs -r http://host.docker.internal:8555 d
+```
+
 <div class="warning">
 
 Blockscout isn’t ideal for local, ephemeral environments, as it won’t re-index blocks and transactions on test reruns. The easiest approach is to set up Blockscout first, initialize the test environment, switch to the [cache](../components/caching.md) config, and run tests without restarting RPC nodes. 
diff --git a/framework/.changeset/v0.2.12.md b/framework/.changeset/v0.2.12.md
@@ -0,0 +1,2 @@
+- Omit all local tooling container logs
+- Allow to connect Blockscout to any node
diff --git a/framework/cmd/blockscout.go b/framework/cmd/blockscout.go
@@ -3,14 +3,16 @@ package main
 import (
 	"fmt"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+	"os"
 	"path/filepath"
 )
 
-func blockscoutUp() error {
+func blockscoutUp(url string) error {
 	framework.L.Info().Msg("Creating local Blockscout stack")
 	if err := extractAllFiles("observability"); err != nil {
 		return err
 	}
+	os.Setenv("BLOCKSCOUT_RPC_URL", url)
 	err := runCommand("bash", "-c", fmt.Sprintf(`
 		cd %s && \
 		docker compose up -d
@@ -23,8 +25,9 @@ func blockscoutUp() error {
 	return nil
 }
 
-func blockscoutDown() error {
+func blockscoutDown(url string) error {
 	framework.L.Info().Msg("Removing local Blockscout stack")
+	os.Setenv("BLOCKSCOUT_RPC_URL", url)
 	err := runCommand("bash", "-c", fmt.Sprintf(`
 		cd %s && \
 		docker compose down -v
diff --git a/framework/cmd/interactive.go b/framework/cmd/interactive.go
@@ -16,11 +16,12 @@ import (
 )
 
 type nodeSetForm struct {
-	Network       string
-	CLVersion     string
-	Nodes         int
-	Observability bool
-	Blockscout    bool
+	Network          string
+	CLVersion        string
+	Nodes            int
+	Observability    bool
+	Blockscout       bool
+	BlockscoutRPCURL string
 }
 
 func createComponentsFromForm(form *nodeSetForm) error {
@@ -83,7 +84,7 @@ func createComponentsFromForm(form *nodeSetForm) error {
 	}
 	switch form.Blockscout {
 	case true:
-		if err := blockscoutUp(); err != nil {
+		if err := blockscoutUp(form.BlockscoutRPCURL); err != nil {
 			return err
 		}
 	}
@@ -107,7 +108,7 @@ func cleanup(form *nodeSetForm) error {
 	}
 	switch form.Blockscout {
 	case true:
-		if err := blockscoutDown(); err != nil {
+		if err := blockscoutDown(form.BlockscoutRPCURL); err != nil {
 			return err
 		}
 	}
@@ -156,6 +157,13 @@ Docker Desktop (https://www.docker.com/products/docker-desktop/)
 			huh.NewConfirm().
 				Title("Do you need to spin up a Blockscout stack?").
 				Value(&f.Blockscout),
+			huh.NewSelect[string]().
+				Title("To which blockchain node you want Blockscout to connect?").
+				Options(
+					huh.NewOption("Network 1", "http://host.docker.internal:8545"),
+					huh.NewOption("Network 2", "http://host.docker.internal:8550"),
+				).
+				Value(&f.BlockscoutRPCURL),
 		),
 	)
 
diff --git a/framework/cmd/main.go b/framework/cmd/main.go
@@ -114,31 +114,44 @@ func main() {
 				Name:    "blockscout",
 				Aliases: []string{"bs"},
 				Usage:   "Controls local Blockscout stack",
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:    "rpc",
+						Aliases: []string{"r"},
+						Usage:   "RPC URL for blockchain node to index",
+						Value:   "http://host.docker.internal:8545",
+					},
+				},
 				Subcommands: []*cli.Command{
 					{
 						Name:        "up",
 						Usage:       "ctf bs up",
 						Aliases:     []string{"u"},
 						Description: "Spins up Blockscout stack",
-						Action:      func(c *cli.Context) error { return blockscoutUp() },
+						Action: func(c *cli.Context) error {
+							return blockscoutUp(c.String("rpc"))
+						},
 					},
 					{
 						Name:        "down",
 						Usage:       "ctf bs down",
 						Aliases:     []string{"d"},
 						Description: "Removes Blockscout stack, wipes all Blockscout databases data",
-						Action:      func(c *cli.Context) error { return blockscoutDown() },
+						Action: func(c *cli.Context) error {
+							return blockscoutDown(c.String("rpc"))
+						},
 					},
 					{
 						Name:        "reboot",
 						Usage:       "ctf bs reboot or ctf bs r",
 						Aliases:     []string{"r"},
 						Description: "Reboots Blockscout stack",
 						Action: func(c *cli.Context) error {
-							if err := blockscoutDown(); err != nil {
+							rpc := c.String("rpc")
+							if err := blockscoutDown(rpc); err != nil {
 								return err
 							}
-							return blockscoutUp()
+							return blockscoutUp(rpc)
 						},
 					},
 				},
diff --git a/framework/cmd/observability/blockscout/docker-compose.yml b/framework/cmd/observability/blockscout/docker-compose.yml
@@ -28,6 +28,8 @@ services:
       - db:database
     environment:
         ETHEREUM_JSONRPC_VARIANT: 'geth'
+        ETHEREUM_JSONRPC_HTTP_URL: ${BLOCKSCOUT_RPC_URL}
+        ETHEREUM_JSONRPC_TRACE_URL: ${BLOCKSCOUT_RPC_URL}
 
   visualizer:
     extends:
diff --git a/framework/components/simple_node_set/node_set.go b/framework/components/simple_node_set/node_set.go
@@ -62,13 +62,12 @@ func printURLs(out *Output) {
 	if out == nil {
 		return
 	}
-	httpURLs, p2pURLs, pgURLs := make([]string, 0), make([]string, 0), make([]string, 0)
+	httpURLs, _, pgURLs := make([]string, 0), make([]string, 0), make([]string, 0)
 	for _, n := range out.CLNodes {
 		httpURLs = append(httpURLs, n.Node.HostURL)
 		pgURLs = append(pgURLs, n.PostgreSQL.Url)
 	}
 	framework.L.Info().Any("UI", httpURLs).Send()
-	framework.L.Debug().Any("P2P", p2pURLs).Send()
 	framework.L.Debug().Any("DB", pgURLs).Send()
 }
 
diff --git a/framework/docker.go b/framework/docker.go
@@ -12,6 +12,7 @@ import (
 	"github.com/docker/go-connections/nat"
 	"github.com/google/uuid"
 	tc "github.com/testcontainers/testcontainers-go"
+	"golang.org/x/sync/errgroup"
 	"io"
 	"os"
 	"os/exec"
@@ -230,6 +231,23 @@ func (dc *DockerClient) copyToContainer(containerID, sourceFile, targetPath stri
 	return nil
 }
 
+func in(s string, substrings []string) bool {
+	for _, substr := range substrings {
+		if strings.Contains(s, substr) {
+			return true
+		}
+	}
+	return false
+}
+
+func isLocalToolDockerContainer(containerName string) bool {
+	if in(containerName, []string{"/sig-provider", "/stats", "/stats-db", "/db", "/backend", "/promtail", "/compose", "/blockscout", "/frontend", "/user-ops-indexer", "/visualizer", "/redis-db", "/proxy"}) {
+		L.Debug().Str("Container", containerName).Msg("Ignoring local tool container output")
+		return true
+	}
+	return false
+}
+
 // WriteAllContainersLogs writes all Docker container logs to the default logs directory
 func WriteAllContainersLogs() error {
 	L.Info().Msg("Writing Docker containers logs")
@@ -247,49 +265,58 @@ func WriteAllContainersLogs() error {
 		return fmt.Errorf("failed to list Docker containers: %w", err)
 	}
 
+	eg := &errgroup.Group{}
+
 	for _, containerInfo := range containers {
-		containerName := containerInfo.Names[0]
-		logOptions := container.LogsOptions{ShowStdout: true, ShowStderr: true}
-		logs, err := provider.Client().ContainerLogs(context.Background(), containerInfo.ID, logOptions)
-		if err != nil {
-			L.Error().Err(err).Str("Container", containerName).Msg("failed to fetch logs for container")
-			continue
-		}
-		logFilePath := filepath.Join(DefaultCTFLogsDir, fmt.Sprintf("%s.log", containerName))
-		logFile, err := os.Create(logFilePath)
-		if err != nil {
-			L.Error().Err(err).Str("Container", containerName).Msg("failed to create container log file")
-			continue
-		}
-		// Parse and write logs
-		header := make([]byte, 8) // Docker stream header is 8 bytes
-		for {
-			_, err := io.ReadFull(logs, header)
-			if err == io.EOF {
-				break
+		eg.Go(func() error {
+			containerName := containerInfo.Names[0]
+			if isLocalToolDockerContainer(containerName) {
+				return nil
 			}
+			L.Debug().Str("Container", containerName).Msg("Collecting logs")
+			logOptions := container.LogsOptions{ShowStdout: true, ShowStderr: true}
+			logs, err := provider.Client().ContainerLogs(context.Background(), containerInfo.ID, logOptions)
 			if err != nil {
-				L.Error().Err(err).Str("Container", containerName).Msg("failed to read log stream header")
-				break
+				L.Error().Err(err).Str("Container", containerName).Msg("failed to fetch logs for container")
+				return err
 			}
-
-			// Extract log message size
-			msgSize := binary.BigEndian.Uint32(header[4:8])
-
-			// Read the log message
-			msg := make([]byte, msgSize)
-			_, err = io.ReadFull(logs, msg)
+			logFilePath := filepath.Join(DefaultCTFLogsDir, fmt.Sprintf("%s.log", containerName))
+			logFile, err := os.Create(logFilePath)
 			if err != nil {
-				L.Error().Err(err).Str("Container", containerName).Msg("failed to read log message")
-				break
+				L.Error().Err(err).Str("Container", containerName).Msg("failed to create container log file")
+				return err
 			}
+			// Parse and write logs
+			header := make([]byte, 8) // Docker stream header is 8 bytes
+			for {
+				_, err := io.ReadFull(logs, header)
+				if err == io.EOF {
+					break
+				}
+				if err != nil {
+					L.Error().Err(err).Str("Container", containerName).Msg("failed to read log stream header")
+					break
+				}
 
-			// Write the log message to the file
-			if _, err := logFile.Write(msg); err != nil {
-				L.Error().Err(err).Str("Container", containerName).Msg("failed to write log message to file")
-				break
+				// Extract log message size
+				msgSize := binary.BigEndian.Uint32(header[4:8])
+
+				// Read the log message
+				msg := make([]byte, msgSize)
+				_, err = io.ReadFull(logs, msg)
+				if err != nil {
+					L.Error().Err(err).Str("Container", containerName).Msg("failed to read log message")
+					break
+				}
+
+				// Write the log message to the file
+				if _, err := logFile.Write(msg); err != nil {
+					L.Error().Err(err).Str("Container", containerName).Msg("failed to write log message to file")
+					break
+				}
 			}
-		}
+			return nil
+		})
 	}
-	return nil
+	return eg.Wait()
 }
diff --git a/framework/examples/myproject/fork.toml b/framework/examples/myproject/fork.toml
@@ -3,16 +3,16 @@
 
 [blockchain_dst]
   chain_id = "2337"
-#  docker_cmd_params = ["--fork-url", "wss://avalanche-fuji-c-chain-rpc.publicnode.com", "--auto-impersonate"]
+#  docker_cmd_params = ["--fork-url", "wss://avalanche-fuji-c-chain-rpc.publicnode.com", "--auto-impersonate", "-b", "1"]
+  docker_cmd_params = ["-b", "1"]
   image = "f4hrenh9it/foundry:latest"
   port = "8545"
   type = "anvil"
-  docker_cmd_params = ["-b", "1"]
 
 [blockchain_src]
   chain_id = "3337"
-#  docker_cmd_params = ["--fork-url", "wss://avalanche-fuji-c-chain-rpc.publicnode.com", "--auto-impersonate"]
+#  docker_cmd_params = ["--fork-url", "wss://avalanche-fuji-c-chain-rpc.publicnode.com", "--auto-impersonate", "-b", "1"]
+  docker_cmd_params = ["-b", "1"]
   image = "f4hrenh9it/foundry:latest"
   port = "8555"
   type = "anvil"
-  docker_cmd_params = ["-b", "1"]
diff --git a/framework/examples/myproject/fork_plus_offchain.toml b/framework/examples/myproject/fork_plus_offchain.toml
@@ -1,21 +1,23 @@
-[contracts_src]
-[contracts_dst]
 
 [blockchain_dst]
   chain_id = "2337"
-#  docker_cmd_params = ["--fork-url", "wss://avalanche-fuji-c-chain-rpc.publicnode.com", "--auto-impersonate"]
+#  docker_cmd_params = ["--fork-url", "wss://avalanche-fuji-c-chain-rpc.publicnode.com", "--auto-impersonate", "-b", "1"]
+  docker_cmd_params = ["-b", "1"]
   image = "f4hrenh9it/foundry:latest"
   port = "8545"
   type = "anvil"
-  docker_cmd_params = ["-b", "1"]
 
 [blockchain_src]
   chain_id = "3337"
-#  docker_cmd_params = ["--fork-url", "wss://avalanche-fuji-c-chain-rpc.publicnode.com", "--auto-impersonate"]
+#  docker_cmd_params = ["--fork-url", "wss://avalanche-fuji-c-chain-rpc.publicnode.com", "--auto-impersonate", "-b", "1"]
+  docker_cmd_params = ["-b", "1"]
   image = "f4hrenh9it/foundry:latest"
   port = "8555"
   type = "anvil"
-  docker_cmd_params = ["-b", "1"]
+
+[contracts_dst]
+
+[contracts_src]
 
 [nodeset]
   nodes = 5
diff --git a/framework/examples/myproject/fork_plus_offchain_test.go b/framework/examples/myproject/fork_plus_offchain_test.go
@@ -71,12 +71,14 @@ func TestOffChainAndFork(t *testing.T) {
 		WithTracing(seth.TracingLevel_All, []string{seth.TraceOutput_Console}).
 		WithPrivateKeys([]string{blockchain.DefaultAnvilPrivateKey}).
 		Build()
+	require.NoError(t, err)
 	scDst, err := seth.NewClientBuilder().
 		WithRpcUrl(bcDst.Nodes[0].HostWSUrl).
 		WithGasPriceEstimations(true, 0, seth.Priority_Fast).
 		WithTracing(seth.TracingLevel_All, []string{seth.TraceOutput_Console}).
 		WithPrivateKeys([]string{blockchain.DefaultAnvilPrivateKey}).
 		Build()
+	require.NoError(t, err)
 
 	// deploy 2 example product contracts
 	// you should replace it with chainlink-deployments
diff --git a/framework/examples/myproject/fork_test.go b/framework/examples/myproject/fork_test.go
diff --git a/framework/examples/myproject/go.mod b/framework/examples/myproject/go.mod
diff --git a/framework/examples/myproject/go.sum b/framework/examples/myproject/go.sum

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+- Omit all local tooling container logs`
	`2`	`+- Allow to connect Blockscout to any node`
Original file line number	Diff line number	Diff line change
`@@ -16,11 +16,12 @@ import (`
`16`	`16`	`)`
`17`	`17`
`18`	`18`	`type nodeSetForm struct {`
`19`		`- Network string`
`20`		`- CLVersion string`
`21`		`- Nodes int`
`22`		`- Observability bool`
`23`		`- Blockscout bool`
	`19`	`+ Network string`
	`20`	`+ CLVersion string`
	`21`	`+ Nodes int`
	`22`	`+ Observability bool`
	`23`	`+ Blockscout bool`
	`24`	`+ BlockscoutRPCURL string`
`24`	`25`	`}`
`25`	`26`
`26`	`27`	`func createComponentsFromForm(form *nodeSetForm) error {`
`@@ -83,7 +84,7 @@ func createComponentsFromForm(form *nodeSetForm) error {`
`83`	`84`	`}`
`84`	`85`	`switch form.Blockscout {`
`85`	`86`	`case true:`
`86`		`- if err := blockscoutUp(); err != nil {`
	`87`	`+ if err := blockscoutUp(form.BlockscoutRPCURL); err != nil {`
`87`	`88`	`return err`
`88`	`89`	`}`
`89`	`90`	`}`
`@@ -107,7 +108,7 @@ func cleanup(form *nodeSetForm) error {`
`107`	`108`	`}`
`108`	`109`	`switch form.Blockscout {`
`109`	`110`	`case true:`
`110`		`- if err := blockscoutDown(); err != nil {`
	`111`	`+ if err := blockscoutDown(form.BlockscoutRPCURL); err != nil {`
`111`	`112`	`return err`
`112`	`113`	`}`
`113`	`114`	`}`
`@@ -156,6 +157,13 @@ Docker Desktop (https://www.docker.com/products/docker-desktop/)`
`156`	`157`	`huh.NewConfirm().`
`157`	`158`	`Title("Do you need to spin up a Blockscout stack?").`
`158`	`159`	`Value(&f.Blockscout),`
	`160`	`+ huh.NewSelect[string]().`
	`161`	`+ Title("To which blockchain node you want Blockscout to connect?").`
	`162`	`+ Options(`
	`163`	`+ huh.NewOption("Network 1", "http://host.docker.internal:8545"),`
	`164`	`+ huh.NewOption("Network 2", "http://host.docker.internal:8550"),`
	`165`	`+ ).`
	`166`	`+ Value(&f.BlockscoutRPCURL),`
`159`	`167`	`),`
`160`	`168`	`)`
`161`	`169`
Original file line number	Diff line number	Diff line change
`@@ -62,13 +62,12 @@ func printURLs(out *Output) {`
`62`	`62`	`if out == nil {`
`63`	`63`	`return`
`64`	`64`	`}`
`65`		`- httpURLs, p2pURLs, pgURLs := make([]string, 0), make([]string, 0), make([]string, 0)`
	`65`	`+ httpURLs, _, pgURLs := make([]string, 0), make([]string, 0), make([]string, 0)`
`66`	`66`	`for _, n := range out.CLNodes {`
`67`	`67`	`httpURLs = append(httpURLs, n.Node.HostURL)`
`68`	`68`	`pgURLs = append(pgURLs, n.PostgreSQL.Url)`
`69`	`69`	`}`
`70`	`70`	`framework.L.Info().Any("UI", httpURLs).Send()`
`71`		`- framework.L.Debug().Any("P2P", p2pURLs).Send()`
`72`	`71`	`framework.L.Debug().Any("DB", pgURLs).Send()`
`73`	`72`	`}`
`74`	`73`