Merge branch 'main' into etb/update-testing-framework-billing

EasterTheBunny · web-flow · commit a143eb3499ac · 2025-10-16T12:50:27.000-05:00
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -25,6 +25,9 @@ jobs:
           - path: framework
             vm: ubuntu-latest
             regex: TestStorageMutations
+          - path: framework
+            vm: ubuntu-latest
+            regex: TestDX
           - path: wasp
             vm: ubuntu22.04-16cores-64GB # ghv-ignore!
             regex: TestSmoke
diff --git a/framework/.changeset/v0.10.33.md b/framework/.changeset/v0.10.33.md
diff --git a/framework/.changeset/v0.10.35.md b/framework/.changeset/v0.10.35.md
@@ -1 +1 @@
-- Add function that exports OCR2 keys to the clnode's REST client
+- Add function that exports OCR2 keys to the clnode's REST client
diff --git a/framework/.changeset/v0.10.36.md b/framework/.changeset/v0.10.36.md
@@ -0,0 +1,2 @@
+
+- Allow to execute arbitrary `ExecConfig` in Docker using our client
diff --git a/framework/.changeset/v0.11.0.md b/framework/.changeset/v0.11.0.md
@@ -0,0 +1 @@
+- Add DX tracking package
diff --git a/framework/.changeset/v0.11.1.md b/framework/.changeset/v0.11.1.md
@@ -0,0 +1 @@
+- Add a function for streaming Docker logs from containers and a predefined filter to exited/dead CTF containers
diff --git a/framework/.changeset/v0.11.2.md b/framework/.changeset/v0.11.2.md
@@ -0,0 +1 @@
+- Fix concurrent map write when streaming Docker logs
diff --git a/framework/components/dockercompose/.changeset/v0.1.16.md b/framework/components/dockercompose/.changeset/v0.1.16.md
@@ -0,0 +1,3 @@
+- Fix prefix stripping from proto package names
+- Add `_` to valid character in proto package name (fix for `bridge_status` package)
+- Update `chainlink-protos` SHA used in tests to `latest`
diff --git a/framework/components/dockercompose/chip_ingress_set/protos.go b/framework/components/dockercompose/chip_ingress_set/protos.go
@@ -139,15 +139,16 @@ func RegisterAndFetchProtos(ctx context.Context, client *github.Client, protoSch
 		// or in other words, we treat "workflows/" folder as the root folder for all protos in this schema set and strip it from the paths derived from the repository structure.
 		prefixesToStrip := determineFolderPrefixesToStrip(protoSchemaSet.Folders)
 
+		strippedProtdMap := make(map[string]string)
+		strippedSubjects := make(map[string]string)
+
 		for path := range protoMap {
 			strippedPath := stripFolderPrefix(path, prefixesToStrip)
-			protoMap[strippedPath] = protoMap[path]
-			subjects[strippedPath] = subjects[path]
-			delete(protoMap, path)
-			delete(subjects, path)
+			strippedProtdMap[strippedPath] = protoMap[path]
+			strippedSubjects[strippedPath] = subjects[path]
 		}
 
-		registerErr := registerAllWithTopologicalSorting(schemaRegistryURL, protoMap, subjects)
+		registerErr := registerAllWithTopologicalSorting(schemaRegistryURL, strippedProtdMap, strippedSubjects)
 		if registerErr != nil {
 			return errors.Wrapf(registerErr, "failed to register protos from %s", protoSchemaSet.URI)
 		}
@@ -174,7 +175,7 @@ func DefaultSubjectNamingStrategy(subjectPrefix string, proto protoFile, protoSc
 // extractPackageNameWithRegex extracts the package name from a proto source file using regex.
 // It returns an error if no package name is found.
 func extractPackageNameWithRegex(protoSrc string) (string, error) {
-	matches := regexp.MustCompile(`(?m)^\s*package\s+([a-zA-Z0-9.]+)\s*;`).FindStringSubmatch(protoSrc)
+	matches := regexp.MustCompile(`(?m)^\s*package\s+([a-zA-Z0-9._]+)\s*;`).FindStringSubmatch(protoSrc)
 	if len(matches) < 2 {
 		return "", fmt.Errorf("no package name found in proto source")
 	}
diff --git a/framework/components/dockercompose/go.mod b/framework/components/dockercompose/go.mod
@@ -25,7 +25,7 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
 	github.com/BurntSushi/toml v1.4.0 // indirect
 	github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e // indirect
-	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/Masterminds/semver/v3 v3.4.0 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
diff --git a/framework/components/dockercompose/go.sum b/framework/components/dockercompose/go.sum
@@ -15,8 +15,8 @@ github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0
 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e h1:rd4bOvKmDIx0WeTv9Qz+hghsgyjikFiPrseXHlKepO0=
 github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw=
-github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
-github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
+github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/Microsoft/hcsshim v0.12.9 h1:2zJy5KA+l0loz1HzEGqyNnjd3fyZA31ZBCGKacp6lLg=
diff --git a/framework/docker.go b/framework/docker.go
@@ -114,7 +114,18 @@ func NewDockerClient() (*DockerClient, error) {
 
 // ExecContainer executes a command inside a running container by name and returns the combined stdout/stderr.
 func (dc *DockerClient) ExecContainer(containerName string, command []string) (string, error) {
-	L.Info().Strs("Command", command).Str("ContainerName", containerName).Msg("Executing command")
+	execConfig := container.ExecOptions{
+		Cmd:          command,
+		AttachStdout: true,
+		AttachStderr: true,
+	}
+
+	return dc.ExecContainerOptions(containerName, execConfig)
+}
+
+// ExecContainer executes a command inside a running container by name and returns the combined stdout/stderr.
+func (dc *DockerClient) ExecContainerOptions(containerName string, execConfig container.ExecOptions) (string, error) {
+	L.Info().Strs("Command", execConfig.Cmd).Str("ContainerName", containerName).Msg("Executing command")
 	ctx := context.Background()
 	containers, err := dc.cli.ContainerList(ctx, container.ListOptions{
 		All: true,
@@ -135,11 +146,6 @@ func (dc *DockerClient) ExecContainer(containerName string, command []string) (s
 		return "", fmt.Errorf("container with name '%s' not found", containerName)
 	}
 
-	execConfig := container.ExecOptions{
-		Cmd:          command,
-		AttachStdout: true,
-		AttachStderr: true,
-	}
 	execID, err := dc.cli.ContainerExecCreate(ctx, containerID, execConfig)
 	if err != nil {
 		return "", fmt.Errorf("failed to create exec instance: %w", err)
@@ -273,34 +279,23 @@ func SaveContainerLogs(dir string) ([]string, error) {
 			return nil, fmt.Errorf("failed to create directory %s: %w", dir, err)
 		}
 	}
-	provider, err := tc.NewDockerProvider()
-	if err != nil {
-		return nil, fmt.Errorf("failed to create Docker provider: %w", err)
-	}
-	containers, err := provider.Client().ContainerList(context.Background(), container.ListOptions{
+
+	logStream, lErr := StreamContainerLogs(container.ListOptions{
 		All: true,
 		Filters: dfilter.NewArgs(dfilter.KeyValuePair{
 			Key:   "label",
 			Value: "framework=ctf",
 		}),
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to list Docker containers: %w", err)
+	}, container.LogsOptions{ShowStdout: true, ShowStderr: true})
+
+	if lErr != nil {
+		return nil, lErr
 	}
 
 	eg := &errgroup.Group{}
 	logFilePaths := make([]string, 0)
-
-	for _, containerInfo := range containers {
+	for containerName, reader := range logStream {
 		eg.Go(func() error {
-			containerName := containerInfo.Names[0]
-			L.Debug().Str("Container", containerName).Msg("Collecting logs")
-			logOptions := container.LogsOptions{ShowStdout: true, ShowStderr: true}
-			logs, err := provider.Client().ContainerLogs(context.Background(), containerInfo.ID, logOptions)
-			if err != nil {
-				L.Error().Err(err).Str("Container", containerName).Msg("failed to fetch logs for container")
-				return err
-			}
 			logFilePath := filepath.Join(dir, fmt.Sprintf("%s.log", containerName))
 			logFile, err := os.Create(logFilePath)
 			if err != nil {
@@ -311,7 +306,7 @@ func SaveContainerLogs(dir string) ([]string, error) {
 			// Parse and write logs
 			header := make([]byte, 8) // Docker stream header is 8 bytes
 			for {
-				_, err := io.ReadFull(logs, header)
+				_, err := io.ReadFull(reader, header)
 				if err == io.EOF {
 					break
 				}
@@ -325,7 +320,7 @@ func SaveContainerLogs(dir string) ([]string, error) {
 
 				// Read the log message
 				msg := make([]byte, msgSize)
-				_, err = io.ReadFull(logs, msg)
+				_, err = io.ReadFull(reader, msg)
 				if err != nil {
 					L.Error().Err(err).Str("Container", containerName).Msg("failed to read log message")
 					break
@@ -346,6 +341,57 @@ func SaveContainerLogs(dir string) ([]string, error) {
 	return logFilePaths, nil
 }
 
+var ExitedCtfContainersListOpts = container.ListOptions{
+	All: true,
+	Filters: dfilter.NewArgs(dfilter.KeyValuePair{
+		Key:   "label",
+		Value: "framework=ctf",
+	},
+		dfilter.KeyValuePair{
+			Key:   "status",
+			Value: "exited"},
+		dfilter.KeyValuePair{
+			Key:   "status",
+			Value: "dead"}),
+}
+
+func StreamContainerLogs(listOptions container.ListOptions, logOptions container.LogsOptions) (map[string]io.ReadCloser, error) {
+	L.Info().Msg("Streaming Docker containers logs")
+	provider, err := tc.NewDockerProvider()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Docker provider: %w", err)
+	}
+	containers, err := provider.Client().ContainerList(context.Background(), listOptions)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list Docker containers: %w", err)
+	}
+
+	eg := &errgroup.Group{}
+	logMap := make(map[string]io.ReadCloser)
+	var mutex sync.Mutex
+
+	for _, containerInfo := range containers {
+		eg.Go(func() error {
+			containerName := containerInfo.Names[0]
+			L.Debug().Str("Container", containerName).Msg("Collecting logs")
+			logs, err := provider.Client().ContainerLogs(context.Background(), containerInfo.ID, logOptions)
+			if err != nil {
+				L.Error().Err(err).Str("Container", containerName).Msg("failed to fetch logs for container")
+				return err
+			}
+			mutex.Lock()
+			defer mutex.Unlock()
+			logMap[containerName] = logs
+			return nil
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		return nil, err
+	}
+
+	return logMap, nil
+}
+
 func BuildImageOnce(once *sync.Once, dctx, dfile, nameAndTag string, buildArgs map[string]string) error {
 	var err error
 	once.Do(func() {
diff --git a/framework/examples/myproject/go.mod b/framework/examples/myproject/go.mod
@@ -138,7 +138,7 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
-	github.com/Masterminds/semver/v3 v3.3.1 // indirect
+	github.com/Masterminds/semver/v3 v3.4.0 // indirect
 	github.com/Masterminds/sprig/v3 v3.3.0 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
diff --git a/framework/examples/myproject/go.sum b/framework/examples/myproject/go.sum
@@ -72,8 +72,8 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob
 github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
-github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4=
-github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
+github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
+github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs=
 github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
diff --git a/framework/examples/myproject/smoke_chip_ingress_test.go b/framework/examples/myproject/smoke_chip_ingress_test.go
@@ -34,7 +34,7 @@ func TestChipIngressSmoke(t *testing.T) {
 		err := chipingressset.DefaultRegisterAndFetchProtos(ctx, nil, []chipingressset.ProtoSchemaSet{
 			{
 				URI:           "https://github.com/smartcontractkit/chainlink-protos",
-				Ref:           "49a301ac36cad84ac8dd7fae6749d3a124b6819b",
+				Ref:           "a653ed4c82a02ec6c0d501dd5af80d02a00009db",
 				Folders:       []string{"workflows"},
 				SubjectPrefix: "cre-",
 			},
diff --git a/framework/go.mod b/framework/go.mod
@@ -4,6 +4,7 @@ go 1.24.2
 
 require (
 	dario.cat/mergo v1.0.1
+	github.com/Masterminds/semver/v3 v3.4.0
 	github.com/avast/retry-go/v4 v4.6.1
 	github.com/block-vision/sui-go-sdk v1.0.6
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
diff --git a/framework/go.sum b/framework/go.sum
@@ -8,6 +8,8 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg6
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
 github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
+github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
+github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
diff --git a/framework/tracking/dx.go b/framework/tracking/dx.go
diff --git a/framework/tracking/dx_test.go b/framework/tracking/dx_test.go
diff --git a/lib/.changeset/v1.54.6.md b/lib/.changeset/v1.54.6.md
diff --git a/lib/docker/test_env/nethermind_base.go b/lib/docker/test_env/nethermind_base.go
diff --git a/lib/docker/test_env/nethermind_eth1.go b/lib/docker/test_env/nethermind_eth1.go
diff --git a/lib/docker/test_env/nethermind_eth2.go b/lib/docker/test_env/nethermind_eth2.go

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-- Add function that exports OCR2 keys to the clnode's REST client`
	`1`	`+- Add function that exports OCR2 keys to the clnode's REST client`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+`
	`2`	+- Allow to execute arbitrary `ExecConfig` in Docker using our client
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+- Add a function for streaming Docker logs from containers and a predefined filter to exited/dead CTF containers`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+- Fix concurrent map write when streaming Docker logs`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+- Fix prefix stripping from proto package names`
	`2`	+- Add `_` to valid character in proto package name (fix for `bridge_status` package)
	`3`	+- Update `chainlink-protos` SHA used in tests to `latest`