Merge #4121

4121: [Testing] Cleanup integration test bootstrap r=peterargue a=peterargue

Refactor integration tests to simplify config and hopefully make them more stable. The main improvements:
* Add healthchecks to all nodes via the admin server
* Cleanup container dirs so they are removed at the end of the test
* Consolidate access to ports into the `Container` object. This simplifies the interface and mades logic more explicit about which container is used
* Add helper methods to get various grpc clients configured to interact with a node
* Refactor observer setup so it's more consistent with other nodes
* Refactor observer suite to reduce the number of network setup/teardowns
* Fix flakiness in the observer integration suite

Co-authored-by: Peter Argue <[email protected]>

Author: bors[bot]

admin/command_runner.go

@@ -76,9 +76,15 @@ func NewCommandRunnerBootstrapper() *CommandRunnerBootstrapper {
 func (r *CommandRunnerBootstrapper) Bootstrap(logger zerolog.Logger, bindAddress string, opts ...CommandRunnerOption) *CommandRunner {
 	handlers := make(map[string]CommandHandler)
 	commands := make([]interface{}, 0, len(r.handlers))
+
+	r.RegisterHandler("ping", func(ctx context.Context, req *CommandRequest) (interface{}, error) {
+		return "pong", nil
+	})
+
 	r.RegisterHandler("list-commands", func(ctx context.Context, req *CommandRequest) (interface{}, error) {
 		return commands, nil
 	})
+
 	for command, handler := range r.handlers {
 		handlers[command] = handler
 		commands = append(commands, command)

insecure/cmd/corrupted_builder.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"fmt"
 	"net"
-	"strconv"
 
 	"github.com/spf13/pflag"
 
@@ -19,7 +18,7 @@ import (
 )
 
 // CorruptNetworkPort is the port number that gRPC server of the corrupt networking layer of the corrupted nodes is listening on.
-const CorruptNetworkPort = 4300
+const CorruptNetworkPort = "4300"
 
 // CorruptedNodeBuilder creates a general flow node builder with corrupt network.
 type CorruptedNodeBuilder struct {
@@ -133,7 +132,7 @@ func (cnb *CorruptedNodeBuilder) enqueueNetworkingLayer() {
 			return nil, fmt.Errorf("could not extract host address: %w", err)
 		}
 
-		address := net.JoinHostPort(host, strconv.Itoa(CorruptNetworkPort))
+		address := net.JoinHostPort(host, CorruptNetworkPort)
 		ccf := corruptnet.NewCorruptConduitFactory(cnb.FlowNodeBuilder.Logger, cnb.FlowNodeBuilder.RootChainID)
 
 		cnb.Logger.Info().Hex("node_id", logging.ID(cnb.NodeID)).Msg("corrupted conduit factory initiated")

integration/client/admin_client.go

@@ -0,0 +1,108 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+)
+
+// AdminClient is a simple client for interacting with the Flow admin server
+type AdminClient struct {
+	client *http.Client
+	url    string
+}
+
+// Request is the request to the admin server.
+type Request struct {
+	CommandName string `json:"commandName"`
+	Data        any    `json:"data,omitempty"`
+}
+
+// Response is the response from the admin server.
+type Response struct {
+	Output any `json:"output"`
+}
+
+// AdminClientOption is a function that configures an admin client.
+type AdminClientOption func(c *AdminClient)
+
+// WithHTTPClient configures the admin client to use the provided HTTP client.
+func WithHTTPClient(client *http.Client) AdminClientOption {
+	return func(c *AdminClient) {
+		c.client = client
+	}
+}
+
+// WithTLS configures the admin client to use TLS when sending requests.
+func WithTLS(enabled bool) AdminClientOption {
+	return func(c *AdminClient) {
+		c.url = strings.Replace(c.url, "http://", "https://", 1)
+	}
+}
+
+// NewAdminClient creates a new admin client.
+func NewAdminClient(serverAddr string, opts ...AdminClientOption) *AdminClient {
+	c := &AdminClient{
+		client: &http.Client{},
+		url:    fmt.Sprintf("http://%s/admin/run_command", serverAddr),
+	}
+
+	for _, apply := range opts {
+		apply(c)
+	}
+
+	return c
+}
+
+// Ping sends a ping command to the server and returns an error if the response is not "pong".
+func (c *AdminClient) Ping(ctx context.Context) error {
+	response, err := c.send(ctx, Request{
+		CommandName: "ping",
+	})
+	if err != nil {
+		return err
+	}
+
+	if response.Output != "pong" {
+		return fmt.Errorf("unexpected response: %v", response.Output)
+	}
+
+	return nil
+}
+
+// RunCommand sends a command to the server and returns the response.
+func (c *AdminClient) RunCommand(ctx context.Context, commandName string, data any) (*Response, error) {
+	response, err := c.send(ctx, Request{
+		CommandName: commandName,
+		Data:        data,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return response, nil
+}
+
+func (c *AdminClient) send(ctx context.Context, req Request) (*Response, error) {
+	reqBody, err := json.Marshal(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal request body: %w", err)
+	}
+
+	resp, err := c.client.Post(c.url, "application/json", bytes.NewBuffer(reqBody))
+	if err != nil {
+		return nil, fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	var result Response
+	err = json.NewDecoder(resp.Body).Decode(&result)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode response body: %w", err)
+	}
+
+	return &result, nil
+}

integration/localnet/.gitignore

@@ -4,3 +4,4 @@
 /trie/
 docker-compose.nodes.yml
 targets.nodes.json
+ports.nodes.json

integration/localnet/Makefile

@@ -46,7 +46,7 @@ else
 	go run -tags relic \
 		-ldflags="-X 'github.com/onflow/flow-go/cmd/build.commit=${COMMIT}' \
 		-X  'github.com/onflow/flow-go/cmd/build.semver=${VERSION}'" \
-		bootstrap.go \
+		builder/*.go \
 		-loglevel=$(LOGLEVEL) \
 		-collection=$(COLLECTION) \
 		-consensus=$(CONSENSUS) \