[DX-1118] make Chip Ingress more robust, pin Red Panda image versions (#1946)

Author: Tofel

framework/.changeset/v0.9.9.md

@@ -0,0 +1 @@
+- Make Chip Ingress more robust, pin Red Panda image versions

framework/components/dockercompose/chip_ingress_set/chip_ingress.go

@@ -11,8 +11,10 @@ import (
 
 	networkTypes "github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
+	"github.com/docker/go-connections/nat"
 	"github.com/pkg/errors"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/modules/compose"
 	"github.com/testcontainers/testcontainers-go/wait"
 )
@@ -111,7 +113,9 @@ func New(in *Input) (*Output, error) {
 			wait.ForListeningPort(DEFAULT_CHIP_INGRESS_GRPC_PORT),
 		).WithDeadline(1*time.Minute),
 	).WaitForService(DEFAULT_RED_PANDA_SERVICE_NAME,
-		wait.ForListeningPort(DEFAULT_RED_PANDA_KAFKA_PORT).WithStartupTimeout(1*time.Minute),
+		wait.ForAll(
+			wait.ForListeningPort(DEFAULT_RED_PANDA_SCHEMA_REGISTRY_PORT).WithStartupTimeout(1*time.Minute),
+			wait.ForListeningPort(DEFAULT_RED_PANDA_KAFKA_PORT).WithStartupTimeout(1*time.Minute)),
 	).WaitForService(DEFAULT_RED_PANDA_CONSOLE_SERVICE_NAME,
 		wait.ForListeningPort(DEFAULT_RED_PANDA_CONSOLE_PORT).WithStartupTimeout(1*time.Minute),
 	)
@@ -130,41 +134,17 @@ func New(in *Input) (*Output, error) {
 	}
 	defer cli.Close()
 
-	timeout := time.After(1 * time.Minute)
-	tick := time.Tick(500 * time.Millisecond)
-
 	// so let's try to connect to a Docker network a couple of times, there must be a race condition in Docker
 	// and even when network sandbox has been created and container is running, this call can still fail
 	// retrying is simpler than trying to figure out how to correctly wait for the network sandbox to be ready
-	var connectNetwork = func(networkName string) error {
-		for {
-			select {
-			case <-timeout:
-				return fmt.Errorf("timeout while trying to connect chip-ingress to default network")
-			case <-tick:
-				if networkErr := cli.NetworkConnect(
-					ctx,
-					networkName,
-					chipIngressContainer.ID,
-					&networkTypes.EndpointSettings{
-						Aliases: []string{identifier},
-					},
-				); networkErr != nil && !strings.Contains(networkErr.Error(), "already exists in network") {
-					framework.L.Trace().Msgf("failed to connect to default network: %v", networkErr)
-					continue
-				}
-				framework.L.Trace().Msgf("connected to %s network", networkName)
-				return nil
-			}
-		}
-	}
-
 	networks := []string{framework.DefaultNetworkName}
 	networks = append(networks, in.ExtraDockerNetworks...)
 
 	for _, networkName := range networks {
 		framework.L.Debug().Msgf("Connecting chip-ingress to %s network", networkName)
-		if connectErr := connectNetwork(networkName); connectErr != nil {
+		connectContex, connectCancel := context.WithTimeout(ctx, 30*time.Second)
+		defer connectCancel()
+		if connectErr := connectNetwork(connectContex, 30*time.Second, cli, chipIngressContainer.ID, networkName, identifier); connectErr != nil {
 			return nil, errors.Wrapf(connectErr, "failed to connect chip-ingress to %s network", networkName)
 		}
 		// verify that the container is connected to framework's network
@@ -177,14 +157,17 @@ func New(in *Input) (*Output, error) {
 		if !ok {
 			return nil, fmt.Errorf("container %s is NOT on network %s", chipIngressContainer.ID, networkName)
 		}
+
+		framework.L.Debug().Msgf("Container %s is connected to network %s", chipIngressContainer.ID, networkName)
 	}
 
 	// get hosts and ports for chip-ingress and redpanda
 	chipIngressExternalHost, chipIngressExternalHostErr := chipIngressContainer.Host(ctx)
 	if chipIngressExternalHostErr != nil {
 		return nil, errors.Wrap(chipIngressExternalHostErr, "failed to get host for Chip Ingress")
 	}
-	chipIngressExternalPort, chipIngressExternalPortErr := chipIngressContainer.MappedPort(ctx, DEFAULT_CHIP_INGRESS_GRPC_PORT)
+	// for some magical reason mapped port sometimes cannot be found, even though we wait for it to be ready, when starting the services
+	chipIngressExternalPort, chipIngressExternalPortErr := findMappdePort(ctx, 20*time.Second, chipIngressContainer, DEFAULT_CHIP_INGRESS_GRPC_PORT)
 	if chipIngressExternalPortErr != nil {
 		return nil, errors.Wrap(chipIngressExternalPortErr, "failed to get mapped port for Chip Ingress")
 	}
@@ -198,11 +181,11 @@ func New(in *Input) (*Output, error) {
 	if redpandaExternalHostErr != nil {
 		return nil, errors.Wrap(redpandaExternalHostErr, "failed to get host for Red Panda")
 	}
-	redpandaExternalKafkaPort, redpandaExternalKafkaPortErr := redpandaContainer.MappedPort(ctx, DEFAULT_RED_PANDA_KAFKA_PORT)
+	redpandaExternalKafkaPort, redpandaExternalKafkaPortErr := findMappdePort(ctx, 20*time.Second, redpandaContainer, DEFAULT_RED_PANDA_KAFKA_PORT)
 	if redpandaExternalKafkaPortErr != nil {
 		return nil, errors.Wrap(redpandaExternalKafkaPortErr, "failed to get mapped port for Red Panda")
 	}
-	redpandaExternalSchemaRegistryPort, redpandaExternalSchemaRegistryPortErr := redpandaContainer.MappedPort(ctx, DEFAULT_RED_PANDA_SCHEMA_REGISTRY_PORT)
+	redpandaExternalSchemaRegistryPort, redpandaExternalSchemaRegistryPortErr := findMappdePort(ctx, 20*time.Second, redpandaContainer, DEFAULT_RED_PANDA_SCHEMA_REGISTRY_PORT)
 	if redpandaExternalSchemaRegistryPortErr != nil {
 		return nil, errors.Wrap(redpandaExternalSchemaRegistryPortErr, "failed to get mapped port for Red Panda")
 	}
@@ -211,12 +194,11 @@ func New(in *Input) (*Output, error) {
 	if redpandaConsoleErr != nil {
 		return nil, errors.Wrap(redpandaConsoleErr, "failed to get redpanda-console container")
 	}
-
 	redpandaExternalConsoleHost, redpandaExternalConsoleHostErr := redpandaConsoleContainer.Host(ctx)
 	if redpandaExternalConsoleHostErr != nil {
 		return nil, errors.Wrap(redpandaExternalConsoleHostErr, "failed to get host for Red Panda Console")
 	}
-	redpandaExternalConsolePort, redpandaExternalConsolePortErr := redpandaConsoleContainer.MappedPort(ctx, DEFAULT_RED_PANDA_CONSOLE_PORT)
+	redpandaExternalConsolePort, redpandaExternalConsolePortErr := findMappdePort(ctx, 20*time.Second, redpandaConsoleContainer, DEFAULT_RED_PANDA_CONSOLE_PORT)
 	if redpandaExternalConsolePortErr != nil {
 		return nil, errors.Wrap(redpandaExternalConsolePortErr, "failed to get mapped port for Red Panda Console")
 	}
@@ -266,3 +248,59 @@ func composeFilePath(rawFilePath string) (string, error) {
 
 	return tempFile.Name(), nil
 }
+
+func findMappdePort(ctx context.Context, timeout time.Duration, container *testcontainers.DockerContainer, port nat.Port) (nat.Port, error) {
+	forCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	tickerInterval := 5 * time.Second
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-forCtx.Done():
+			return "", fmt.Errorf("timeout while waiting for mapped port for %s", port)
+		case <-ticker.C:
+			portCtx, portCancel := context.WithTimeout(ctx, tickerInterval)
+			defer portCancel()
+			mappedPort, mappedPortErr := container.MappedPort(portCtx, port)
+			if mappedPortErr != nil {
+				return "", errors.Wrapf(mappedPortErr, "failed to get mapped port for %s", port)
+			}
+			if mappedPort.Port() == "" {
+				return "", fmt.Errorf("mapped port for %s is empty", port)
+			}
+			return mappedPort, nil
+		}
+	}
+}
+
+func connectNetwork(connCtx context.Context, timeout time.Duration, dockerClient *client.Client, containerID, networkName, stackIdentifier string) error {
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+
+	networkCtx, networkCancel := context.WithTimeout(connCtx, timeout)
+	defer networkCancel()
+
+	for {
+		select {
+		case <-networkCtx.Done():
+			return fmt.Errorf("timeout while trying to connect chip-ingress to default network")
+		case <-ticker.C:
+			if networkErr := dockerClient.NetworkConnect(
+				connCtx,
+				networkName,
+				containerID,
+				&networkTypes.EndpointSettings{
+					Aliases: []string{stackIdentifier},
+				},
+			); networkErr != nil && !strings.Contains(networkErr.Error(), "already exists in network") {
+				framework.L.Trace().Msgf("failed to connect to default network: %v", networkErr)
+				continue
+			}
+			framework.L.Trace().Msgf("connected to %s network", networkName)
+			return nil
+		}
+	}
+}

framework/components/dockercompose/chip_ingress_set/docker-compose.yml

@@ -2,7 +2,8 @@
 services:
 
   chip-ingress:
-    image: chip-ingress:qa-latest
+    # added image (instead of build) to use image created by a setup script
+    image: ${CHIP_INGRESS_IMAGE:-chip-ingress:local-cre}
     container_name: chip-ingress
     depends_on:
       - redpanda-0
@@ -42,7 +43,8 @@ services:
       retries: 10
 
   redpanda-0:
-    image: docker.redpanda.com/redpandadata/redpanda
+    # using a specific version of the redpanda image to exclude potentially breaking changes
+    image: docker.redpanda.com/redpandadata/redpanda:v24.3.16
     container_name: redpanda-0
     hostname: redpanda-0
     command:
@@ -72,8 +74,9 @@ services:
       start_period: 1s
 
   redpanda-console:
+    # using a specific version of the console image to ensure compatibility, because v3.x.x uses incompatible configuration format
     container_name: redpanda-console
-    image: docker.redpanda.com/redpandadata/console
+    image: docker.redpanda.com/redpandadata/console:v2.8.6
     entrypoint: /bin/sh
     command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
     environment:

framework/components/dockercompose/go.mod

@@ -7,6 +7,7 @@ replace github.com/smartcontractkit/chainlink-testing-framework/framework => ../
 require (
 	github.com/confluentinc/confluent-kafka-go v1.9.2
 	github.com/docker/docker v28.0.4+incompatible
+	github.com/docker/go-connections v0.5.0
 	github.com/google/go-github/v72 v72.0.0
 	github.com/pkg/errors v0.9.1
 	github.com/smartcontractkit/chainlink-testing-framework/framework v0.0.0-00010101000000-000000000000
@@ -64,7 +65,6 @@ require (
 	github.com/docker/distribution v2.8.3+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.8.2 // indirect
 	github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
-	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/ebitengine/purego v0.8.2 // indirect