JD, custom n:m ports mapping syntax (#1430)

add jd and n:m port mapping

Author: skudasov

.github/workflows/framework-golden-tests.yml

@@ -10,9 +10,6 @@ jobs:
     env:
       LOKI_TENANT_ID: promtail
       LOKI_URL: http://localhost:3030/loki/api/v1/push
-      # this is not the best practice, and it must be fixed, run your tests WITHOUT IT!
-      # however, on current latest image we must use this flag
-      CTF_IGNORE_CRITICAL_LOGS: true
     runs-on: ubuntu-latest
     permissions:
       id-token: write

.github/workflows/framework.yml

@@ -9,6 +9,8 @@ jobs:
     defaults:
       run:
         working-directory: framework
+    env:
+      CTF_JD_IMAGE: ${{ secrets.CTF_JD_IMAGE }}
     runs-on: ubuntu-latest
     permissions:
       id-token: write

book/src/framework/components/state.md

@@ -40,7 +40,7 @@ One node set is enough for any kind of testing, if you need more nodes consider
 
 ## Custom ports
 
-You can also define a custom set of ports for any node
+You can also define a custom set of ports for any node.
 ```toml
 [nodeset]
   nodes = 5
@@ -53,6 +53,7 @@ You can also define a custom set of ports for any node
 
     [nodeset.node_specs.node]
       # here we defined 2 new ports to listen and mapped them to our host machine
-      custom_ports = [14000, 14001]
+      # syntax is "host:docker", if you provide only host port then we map 1-to-1
+      custom_ports = ["14000:15000", "20000"]
       image = "public.ecr.aws/chainlink/chainlink:v2.16.0"
 ```

framework/.changeset/v0.3.2.md

@@ -0,0 +1,5 @@
+- Allow exposing CL node ports in host:docker format
+- Expose default test private keys as framework constants
+- Rename CHAINLINK_IMAGE to CTF_CHAINLINK_IMAGE to avoid CI collisions
+- Add CTF_JD_IMAGE env var
+- Add JobDistributor component

framework/Makefile

@@ -29,5 +29,5 @@ tidy:
 go_mod:
 	go mod download
 
-test_unit: go_mod
-	go test -timeout 5m -count 1 -run TestDocker
+test_component: go_mod
+	go test -timeout 5m -count 1 -run TestComponentDocker ./...

framework/components/clnode/clnode.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"text/template"
 	"time"
@@ -20,9 +21,10 @@ import (
 )
 
 const (
-	DefaultHTTPPort = "6688"
-	DefaultP2PPort  = "6690"
-	TmpImageName    = "chainlink-tmp:latest"
+	DefaultHTTPPort     = "6688"
+	DefaultP2PPort      = "6690"
+	TmpImageName        = "chainlink-tmp:latest"
+	CustomPortSeparator = ":"
 )
 
 var (
@@ -51,7 +53,7 @@ type NodeInput struct {
 	UserSecretsOverrides    string   `toml:"user_secrets_overrides"`
 	HTTPPort                int      `toml:"port"`
 	P2PPort                 int      `toml:"p2p_port"`
-	CustomPorts             []int    `toml:"custom_ports"`
+	CustomPorts             []string `toml:"custom_ports"`
 }
 
 // Output represents Chainlink node output, nodes and databases connection URLs
@@ -111,6 +113,54 @@ func NewNode(in *Input, pgOut *postgres.Output) (*Output, error) {
 	return out, nil
 }
 
+// generatePortBindings generates exposed ports and port bindings
+// exposes default CL node port
+// exposes custom_ports in format "host:docker" or map 1-to-1 if only "host" port is provided
+func generatePortBindings(in *Input) ([]string, nat.PortMap, error) {
+	httpPort := fmt.Sprintf("%s/tcp", DefaultHTTPPort)
+	portBindings := nat.PortMap{
+		nat.Port(httpPort): []nat.PortBinding{
+			{
+				HostIP:   "0.0.0.0",
+				HostPort: fmt.Sprintf("%d/tcp", in.Node.HTTPPort),
+			},
+		},
+	}
+	customPorts := make([]string, 0)
+	for _, p := range in.Node.CustomPorts {
+		if strings.Contains(p, CustomPortSeparator) {
+			pp := strings.Split(p, CustomPortSeparator)
+			if len(pp) != 2 {
+				return nil, nil, errors.New("custom_ports has ':' but you must provide both ports")
+			}
+			customPorts = append(customPorts, fmt.Sprintf("%s/tcp", pp[1]))
+
+			dockerPort := nat.Port(fmt.Sprintf("%s/tcp", pp[1]))
+			hostPort := fmt.Sprintf("%s/tcp", pp[0])
+			portBindings[dockerPort] = []nat.PortBinding{
+				{
+					HostIP:   "0.0.0.0",
+					HostPort: hostPort,
+				},
+			}
+		} else {
+			customPorts = append(customPorts, fmt.Sprintf("%s/tcp", p))
+
+			dockerPort := nat.Port(fmt.Sprintf("%s/tcp", p))
+			hostPort := fmt.Sprintf("%s/tcp", p)
+			portBindings[dockerPort] = []nat.PortBinding{
+				{
+					HostIP:   "0.0.0.0",
+					HostPort: hostPort,
+				},
+			}
+		}
+	}
+	exposedPorts := []string{httpPort}
+	exposedPorts = append(exposedPorts, customPorts...)
+	return exposedPorts, portBindings, nil
+}
+
 func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
 	ctx := context.Background()
 
@@ -147,37 +197,17 @@ func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
 		return nil, err
 	}
 
-	httpPort := fmt.Sprintf("%s/tcp", DefaultHTTPPort)
 	var containerName string
 	if in.Node.Name != "" {
 		containerName = in.Node.Name
 	} else {
 		containerName = framework.DefaultTCName("node")
 	}
-	customPorts := make([]string, 0)
-	for _, p := range in.Node.CustomPorts {
-		customPorts = append(customPorts, fmt.Sprintf("%d/tcp", p))
-	}
-	exposedPorts := []string{httpPort}
-	exposedPorts = append(exposedPorts, customPorts...)
 
-	portBindings := nat.PortMap{
-		nat.Port(httpPort): []nat.PortBinding{
-			{
-				HostIP:   "0.0.0.0",
-				HostPort: fmt.Sprintf("%d/tcp", in.Node.HTTPPort),
-			},
-		},
-	}
-	for _, p := range customPorts {
-		portBindings[nat.Port(p)] = []nat.PortBinding{
-			{
-				HostIP:   "0.0.0.0",
-				HostPort: p,
-			},
-		}
+	exposedPorts, portBindings, err := generatePortBindings(in)
+	if err != nil {
+		return nil, err
 	}
-
 	req := tc.ContainerRequest{
 		AlwaysPullImage: in.Node.PullImage,
 		Image:           in.Node.Image,

framework/components/jd/jd.go

@@ -0,0 +1,113 @@
+package jd
+
+import (
+	"context"
+	"fmt"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/go-connections/nat"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+	tc "github.com/testcontainers/testcontainers-go"
+	tcwait "github.com/testcontainers/testcontainers-go/wait"
+	"os"
+)
+
+const (
+	TmpImageName            = "jd-local"
+	GRPCPort         string = "42242"
+	CSAEncryptionKey string = "!PASsword000!"
+	WSRPCPort        string = "8080"
+)
+
+type Input struct {
+	Image            string  `toml:"image"`
+	GRPCPort         string  `toml:"grpc_port"`
+	WSRPCPort        string  `toml:"wsrpc_port"`
+	DBURL            string  `toml:"db_url"`
+	CSAEncryptionKey string  `toml:"csa_encryption_key"`
+	DockerFilePath   string  `toml:"docker_file"`
+	DockerContext    string  `toml:"docker_ctx"`
+	Out              *Output `toml:"out"`
+}
+
+type Output struct {
+	UseCache       bool   `toml:"use_cache"`
+	HostGRPCUrl    string `toml:"grpc_url"`
+	DockerGRPCUrl  string `toml:"docker_internal_grpc_url"`
+	HostWSRPCUrl   string `toml:"wsrpc_url"`
+	DockerWSRPCUrl string `toml:"docker_internal_wsrpc_url"`
+}
+
+func defaults(in *Input) {
+	if in.GRPCPort == "" {
+		in.GRPCPort = GRPCPort
+	}
+	if in.WSRPCPort == "" {
+		in.WSRPCPort = WSRPCPort
+	}
+	if in.CSAEncryptionKey == "" {
+		in.CSAEncryptionKey = CSAEncryptionKey
+	}
+}
+
+func NewJD(in *Input) (*Output, error) {
+	if in.Out != nil && in.Out.UseCache {
+		return in.Out, nil
+	}
+	ctx := context.Background()
+	defaults(in)
+	jdImg := os.Getenv("CTF_JD_IMAGE")
+	if jdImg != "" {
+		in.Image = jdImg
+	}
+	containerName := framework.DefaultTCName("jd")
+	bindPort := fmt.Sprintf("%s/tcp", in.GRPCPort)
+	req := tc.ContainerRequest{
+		Name:     containerName,
+		Image:    in.Image,
+		Labels:   framework.DefaultTCLabels(),
+		Networks: []string{framework.DefaultNetworkName},
+		NetworkAliases: map[string][]string{
+			framework.DefaultNetworkName: {containerName},
+		},
+		ExposedPorts: []string{bindPort},
+		HostConfigModifier: func(h *container.HostConfig) {
+			h.PortBindings = framework.MapTheSamePort(bindPort)
+		},
+		Env: map[string]string{
+			"DATABASE_URL":              in.DBURL,
+			"PORT":                      in.GRPCPort,
+			"NODE_RPC_PORT":             in.WSRPCPort,
+			"CSA_KEY_ENCRYPTION_SECRET": in.CSAEncryptionKey,
+		},
+		WaitingFor: tcwait.ForAll(
+			tcwait.ForListeningPort(nat.Port(fmt.Sprintf("%s/tcp", in.GRPCPort))),
+		),
+	}
+	if req.Image == "" {
+		req.Image = TmpImageName
+		if err := framework.BuildImage(in.DockerContext, in.DockerFilePath, req.Image); err != nil {
+			return nil, err
+		}
+		req.KeepImage = false
+	}
+	c, err := tc.GenericContainer(ctx, tc.GenericContainerRequest{
+		ContainerRequest: req,
+		Started:          true,
+	})
+	if err != nil {
+		return nil, err
+	}
+	host, err := framework.GetHost(c)
+	if err != nil {
+		return nil, err
+	}
+	out := &Output{
+		UseCache:       true,
+		HostGRPCUrl:    fmt.Sprintf("http://%s:%s", host, in.GRPCPort),
+		DockerGRPCUrl:  fmt.Sprintf("http://%s:%s", containerName, in.GRPCPort),
+		HostWSRPCUrl:   fmt.Sprintf("ws://%s:%s", host, in.WSRPCPort),
+		DockerWSRPCUrl: fmt.Sprintf("ws://%s:%s", containerName, in.WSRPCPort),
+	}
+	in.Out = out
+	return out, nil
+}

framework/components/jd/jd_test.go

@@ -0,0 +1,31 @@
+package jd_test
+
+import (
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/jd"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/postgres"
+	"github.com/stretchr/testify/require"
+	"os"
+	"sync"
+	"testing"
+)
+
+// here we only test that we can boot up JD
+// client examples are under "examples" dir
+// since JD is private this env var should be set locally and in CI
+// TODO: add ComponentDocker prefix to turn this on when we'll have access to ECRs
+func TestJD(t *testing.T) {
+	err := framework.DefaultNetwork(&sync.Once{})
+	require.NoError(t, err)
+	pgOut, err := postgres.NewPostgreSQL(&postgres.Input{
+		Image:      "postgres:12.0",
+		Port:       14402,
+		VolumeName: "c",
+	})
+	require.NoError(t, err)
+	_, err = jd.NewJD(&jd.Input{
+		DBURL: pgOut.JDDockerInternalURL,
+		Image: os.Getenv("CTF_JD_IMAGE"),
+	})
+	require.NoError(t, err)
+}

framework/components/postgres/postgres.go

@@ -32,9 +32,11 @@ type Input struct {
 }
 
 type Output struct {
-	Url               string `toml:"url"`
-	ContainerName     string `toml:"container_name"`
-	DockerInternalURL string `toml:"docker_internal_url"`
+	Url                 string `toml:"url"`
+	ContainerName       string `toml:"container_name"`
+	DockerInternalURL   string `toml:"docker_internal_url"`
+	JDUrl               string `toml:"jd_url"`
+	JDDockerInternalURL string `toml:"jd_docker_internal_url"`
 }
 
 func NewPostgreSQL(in *Input) (*Output, error) {
@@ -47,6 +49,7 @@ func NewPostgreSQL(in *Input) (*Output, error) {
 	for i := 0; i <= in.Databases; i++ {
 		sqlCommands = append(sqlCommands, fmt.Sprintf("CREATE DATABASE db_%d;", i))
 	}
+	sqlCommands = append(sqlCommands, "CREATE DATABASE jd;")
 	sqlCommands = append(sqlCommands, "ALTER USER chainlink WITH SUPERUSER;")
 	initSQL := strings.Join(sqlCommands, "\n")
 	initFile, err := os.CreateTemp("", "init-*.sql")
@@ -144,5 +147,21 @@ func NewPostgreSQL(in *Input) (*Output, error) {
 			portToExpose,
 			Database,
 		),
+		JDDockerInternalURL: fmt.Sprintf(
+			"postgresql://%s:%s@%s:%s/%s?sslmode=disable",
+			User,
+			Password,
+			containerName,
+			Port,
+			"jd",
+		),
+		JDUrl: fmt.Sprintf(
+			"postgresql://%s:%s@%s:%d/%s?sslmode=disable",
+			User,
+			Password,
+			host,
+			portToExpose,
+			"jd",
+		),
 	}, nil
 }

framework/components/simple_node_set/node_set.go

@@ -32,6 +32,7 @@ type Input struct {
 // Output is a node set configuration output, used for caching or external components
 type Output struct {
 	UseCache bool             `toml:"use_cache"`
+	DBOut    *postgres.Output `toml:"db_out"`
 	CLNodes  []*clnode.Output `toml:"cl_nodes"`
 }
 
@@ -170,6 +171,7 @@ func sharedDBSetup(in *Input, bcOut *blockchain.Output) (*Output, error) {
 	sortNodeOutsByHostPort(nodeOuts)
 	return &Output{
 		UseCache: true,
+		DBOut:    dbOut,
 		CLNodes:  nodeOuts,
 	}, nil
 }