DNS isolation for JD and CLNodes (#1798)

* add chaosmesh debug example with Havoc for local and stage

* WIP: add chaosmesh debug example with Havoc for local and stage

* JD, CL node DNS isolation, update troubleshooting

* review fixes

Author: skudasov

book/src/SUMMARY.md

@@ -25,6 +25,7 @@
   - [Components Cleanup](framework/components/cleanup.md)
   - [Components Caching](framework/components/caching.md)
   - [Components Resources](framework/components/resources.md)
+  - [Containers Network Isolation](framework/components/network_isolation.md)
   - [Mocking Services](framework/components/mocking.md)
   - [Copying Files](framework/copying_files.md)
   - [External Environment](framework/components/external.md)

book/src/framework/components/network_isolation.md

@@ -0,0 +1,12 @@
+# Containers Network Isolation
+
+Some components can be isolated from internet. Since some of them doesn't have `iptables` and must be accessible from local host for debugging we isolate network on DNS level.
+
+JobDistributor is isolated from internet by default to prevent applying manifest changes when run in tests.
+
+NodeSet DNS isolation can be controlled with a flag
+```
+[[nodesets]]
+  # NodeSet DNS can be isolated if needed
+  no_dns = true
+```

book/src/framework/components/troubleshooting.md

@@ -1,5 +1,18 @@
 # Troubleshooting
 
+## Can't start `ctf obs u`
+```
+Error response from daemon: error while creating mount source path '/host_mnt/Users/fsldkfs/Downloads/compose/conf/provisioning/dashboards/cadvisor/cadvisor.json': mkdir /host_mnt/Users/sdfjskj/Downloads/compose/conf/provisioning: operation not permitted
+exit status 1
+```
+
+#### Solution
+Enable Docker to access your directory
+```
+Docker Desktop -> Settings -> Resources -> File Sharing
+```
+
+
 ## Can't run `anvil`, issue with `Rosetta`
 ```
 2024/11/27 15:20:27 ⏳ Waiting for container id 79f8a68c07cc image: f4hrenh9it/foundry:latest. Waiting for: &{Port:8546 timeout:0x14000901278 PollInterval:100ms skipInternalCheck:false}

framework/.changeset/v0.7.6.md

@@ -0,0 +1,2 @@
+- Add troubleshooting for observability stack and file access
+- Add DNS isolation for CL node and JD + test

framework/components/clnode/clnode.go

@@ -36,6 +36,7 @@ var (
 
 // Input represents Chainlink node input
 type Input struct {
+	NoDNS   bool            `toml:"no_dns"`
 	DbInput *postgres.Input `toml:"db" validate:"required"`
 	Node    *NodeInput      `toml:"node" validate:"required"`
 	Out     *Output         `toml:"out"`
@@ -252,6 +253,7 @@ func newNode(in *Input, pgOut *postgres.Output) (*NodeOut, error) {
 	}
 	if in.Node.HTTPPort != 0 && in.Node.P2PPort != 0 {
 		req.HostConfigModifier = func(h *container.HostConfig) {
+			framework.NoDNS(in.NoDNS, h)
 			h.PortBindings = portBindings
 			framework.ResourceLimitsFunc(h, in.Node.ContainerResources)
 		}

framework/components/jd/jd.go

@@ -93,6 +93,8 @@ func NewJD(in *Input) (*Output, error) {
 		},
 		ExposedPorts: []string{bindPort},
 		HostConfigModifier: func(h *container.HostConfig) {
+			// JobDistributor service is isolated from internet by default!
+			framework.NoDNS(true, h)
 			h.PortBindings = framework.MapTheSamePort(bindPort)
 		},
 		Env: map[string]string{

framework/components/networktest/nettest.go

@@ -0,0 +1,52 @@
+package networktest
+
+/*
+This component exists purely for Docker debug purposes
+*/
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/wait"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+)
+
+type Input struct {
+	Name          string
+	NoDNS         bool
+	CustomNetwork bool
+}
+
+type Output struct{}
+
+// NewNetworkTest creates a minimal Alpine Linux container for network testing
+func NewNetworkTest(in Input) error {
+	req := testcontainers.ContainerRequest{
+		Name:     in.Name,
+		Image:    "alpine:latest",
+		Networks: []string{framework.DefaultNetworkName},
+		NetworkAliases: map[string][]string{
+			framework.DefaultNetworkName: {"networktest"},
+		},
+		Labels:     framework.DefaultTCLabels(),
+		WaitingFor: wait.ForLog(""),
+		Cmd:        []string{"/bin/sh", "-c", "while true; do sleep 30; done;"},
+	}
+	req.HostConfigModifier = func(hc *container.HostConfig) {
+		// Remove external DNS
+		framework.NoDNS(in.NoDNS, hc)
+	}
+
+	_, err := testcontainers.GenericContainer(context.Background(), testcontainers.GenericContainerRequest{
+		ContainerRequest: req,
+		Started:          true,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to start alpine container: %w", err)
+	}
+	return nil
+}

framework/components/networktest/nettest_test.go

@@ -0,0 +1,36 @@
+package networktest_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/networktest"
+)
+
+func TestNetworkIsolationRules(t *testing.T) {
+	t.Run("can work without DNS isolation", func(t *testing.T) {
+		name := "nettest1"
+		err := networktest.NewNetworkTest(networktest.Input{NoDNS: false, Name: name})
+		require.NoError(t, err)
+		dc, err := framework.NewDockerClient()
+		require.NoError(t, err)
+		sOut, err := dc.ExecContainer(name, []string{"ping", "-c", "1", "google.com"})
+		require.NoError(t, err)
+		require.NotContains(t, sOut, "bad address")
+		fmt.Println(sOut)
+	})
+	t.Run("DNS isolation works", func(t *testing.T) {
+		name := "nettest2"
+		err := networktest.NewNetworkTest(networktest.Input{NoDNS: true, Name: name})
+		require.NoError(t, err)
+		dc, err := framework.NewDockerClient()
+		require.NoError(t, err)
+		sOut, err := dc.ExecContainer(name, []string{"ping", "-c", "1", "google.com"})
+		require.NoError(t, err)
+		require.Contains(t, sOut, "bad address")
+		fmt.Println(sOut)
+	})
+}

framework/components/simple_node_set/node_set.go

@@ -30,6 +30,7 @@ type Input struct {
 	OverrideMode       string          `toml:"override_mode" validate:"required,oneof=all each"`
 	DbInput            *postgres.Input `toml:"db" validate:"required"`
 	NodeSpecs          []*clnode.Input `toml:"node_specs" validate:"required"`
+	NoDNS              bool            `toml:"no_dns"`
 	Out                *Output         `toml:"out"`
 }
 
@@ -135,6 +136,7 @@ func sharedDBSetup(in *Input, bcOut *blockchain.Output) (*Output, error) {
 			nodeWithNodeSetPrefixName := fmt.Sprintf("%s-%s", in.Name, nodeName)
 
 			nodeSpec := &clnode.Input{
+				NoDNS:   in.NoDNS,
 				DbInput: in.DbInput,
 				Node: &clnode.NodeInput{
 					HTTPPort:                httpPortRangeStart + i,

framework/docker.go

@@ -412,6 +412,7 @@ func ResourceLimitsFunc(h *container.HostConfig, resources *ContainerResources)
 	}
 }
 
+// GenerateCustomPortsData generate custom ports data: exposed and forwarded port map
 func GenerateCustomPortsData(portsProvided []string) ([]string, nat.PortMap, error) {
 	exposedPorts := make([]string, 0)
 	portBindings := nat.PortMap{}
@@ -438,3 +439,10 @@ func GenerateCustomPortsData(portsProvided []string) ([]string, nat.PortMap, err
 	exposedPorts = append(exposedPorts, customPorts...)
 	return exposedPorts, portBindings, nil
 }
+
+// NoDNS removes default DNS server and sets it to localhost
+func NoDNS(noDNS bool, hc *container.HostConfig) {
+	if noDNS {
+		hc.DNS = []string{"127.0.0.1"}
+	}
+}