Adding ECS Observer translator (#1755)

Paramadon · dependabot[bot] · agarakan · commit ec4989c49a4e · 2025-08-19T20:03:24.000Z
Signed-off-by: dependabot[bot] &lt;support@github.com&gt;
Co-authored-by: dependabot[bot] &lt;49699333+dependabot[bot]@users.noreply.github.com&gt;
diff --git a/translator/tocwconfig/sampleConfig/prometheus_combined_config_linux.yaml b/translator/tocwconfig/sampleConfig/prometheus_combined_config_linux.yaml
@@ -131,6 +131,44 @@ extensions:
             usage_flags:
                 mode: EC2
                 region_type: ACJ
+    ecs_observer:
+        cluster_name: ecs-cluster-a
+        cluster_region: us-west-2
+        docker_labels:
+            - job_name: ""
+              job_name_label: ECS_PROMETHEUS_JOB_NAME_1
+              metrics_path: ""
+              metrics_path_label: ECS_PROMETHEUS_METRICS_PATH
+              port_label: ECS_PROMETHEUS_EXPORTER_PORT_SUBSET
+        job_label_name: ""
+        refresh_interval: 1m0s
+        result_file: "{ecsSdFileName}"
+        services:
+            - container_name_pattern: nginx-prometheus-exporter
+              job_name: service_name_1
+              metrics_path: /metrics
+              metrics_ports:
+                - 9113
+              name_pattern: .*-application-stack
+            - container_name_pattern: ""
+              job_name: ""
+              metrics_path: /stats/metrics
+              metrics_ports:
+                - 9114
+              name_pattern: run-application-stack
+        task_definitions:
+            - arn_pattern: .*task_def_1:[0-9]+
+              container_name_pattern: ""
+              job_name: task_def_1
+              metrics_path: /stats/metrics
+              metrics_ports:
+                - 9901
+            - arn_pattern: task_def_2
+              container_name_pattern: ^envoy$
+              job_name: ""
+              metrics_path: /metrics
+              metrics_ports:
+                - 9902
     entitystore:
         mode: ec2
         region: us-west-2
@@ -244,6 +282,7 @@ service:
     extensions:
         - agenthealth/metrics
         - agenthealth/statuscode
+        - ecs_observer
         - sigv4auth
         - agenthealth/logs
         - entitystore
diff --git a/translator/tocwconfig/sampleConfig/prometheus_config_linux.conf b/translator/tocwconfig/sampleConfig/prometheus_config_linux.conf
@@ -20,6 +20,6 @@
     endpoint_override = "https://fake_endpoint"
     force_flush_interval = "30s"
     log_stream_name = "host_name_from_env"
+    mode = "EC2"
     region = "us-east-1"
-
-[processors]
+    region_type = "ACJ"
diff --git a/translator/tocwconfig/sampleConfig/prometheus_config_linux.yaml b/translator/tocwconfig/sampleConfig/prometheus_config_linux.yaml
@@ -86,6 +86,44 @@ extensions:
             usage_flags:
                 mode: EC2
                 region_type: ACJ
+    ecs_observer:
+        cluster_name: ecs-cluster-a
+        cluster_region: us-west-1
+        docker_labels:
+            - job_name: ""
+              job_name_label: ECS_PROMETHEUS_JOB_NAME_1
+              metrics_path: ""
+              metrics_path_label: ECS_PROMETHEUS_METRICS_PATH
+              port_label: ECS_PROMETHEUS_EXPORTER_PORT_SUBSET
+        job_label_name: ""
+        refresh_interval: 1m0s
+        result_file: "{ecsSdFileName}"
+        services:
+            - container_name_pattern: nginx-prometheus-exporter
+              job_name: service_name_1
+              metrics_path: /metrics
+              metrics_ports:
+                - 9113
+              name_pattern: .*-application-stack
+            - container_name_pattern: ""
+              job_name: ""
+              metrics_path: /stats/metrics
+              metrics_ports:
+                - 9114
+              name_pattern: run-application-stack
+        task_definitions:
+            - arn_pattern: .*task_def_1:[0-9]+
+              container_name_pattern: ""
+              job_name: task_def_1
+              metrics_path: /stats/metrics
+              metrics_ports:
+                - 9901
+            - arn_pattern: task_def_2
+              container_name_pattern: ^envoy$
+              job_name: ""
+              metrics_path: /metrics
+              metrics_ports:
+                - 9902
     entitystore:
         mode: ec2
         region: us-east-1
@@ -138,6 +176,7 @@ service:
     extensions:
         - agenthealth/logs
         - agenthealth/statuscode
+        - ecs_observer
         - entitystore
     pipelines:
         metrics/prometheus/cloudwatchlogs:
diff --git a/translator/tocwconfig/sampleConfig/prometheus_config_windows.conf b/translator/tocwconfig/sampleConfig/prometheus_config_windows.conf
@@ -20,6 +20,6 @@
     endpoint_override = "https://fake_endpoint"
     force_flush_interval = "5s"
     log_stream_name = "host_name_from_env"
+    mode = "EC2"
     region = "us-east-1"
-
-[processors]
+    region_type = "ACJ"
diff --git a/translator/tocwconfig/sampleConfig/prometheus_config_windows.yaml b/translator/tocwconfig/sampleConfig/prometheus_config_windows.yaml
@@ -68,6 +68,44 @@ extensions:
             usage_flags:
                 mode: EC2
                 region_type: ACJ
+    ecs_observer:
+        cluster_name: ecs-cluster-a
+        cluster_region: us-west-1
+        docker_labels:
+            - job_name: ""
+              job_name_label: ECS_PROMETHEUS_JOB_NAME_1
+              metrics_path: ""
+              metrics_path_label: ECS_PROMETHEUS_METRICS_PATH
+              port_label: ECS_PROMETHEUS_EXPORTER_PORT_SUBSET
+        job_label_name: ""
+        refresh_interval: 1m0s
+        result_file: "{ecsSdFileName}"
+        services:
+            - container_name_pattern: nginx-prometheus-exporter
+              job_name: service_name_1
+              metrics_path: /metrics
+              metrics_ports:
+                - 9113
+              name_pattern: .*-application-stack
+            - container_name_pattern: ""
+              job_name: ""
+              metrics_path: /stats/metrics
+              metrics_ports:
+                - 9114
+              name_pattern: run-application-stack
+        task_definitions:
+            - arn_pattern: .*task_def_1:[0-9]+
+              container_name_pattern: ""
+              job_name: task_def_1
+              metrics_path: /stats/metrics
+              metrics_ports:
+                - 9901
+            - arn_pattern: task_def_2
+              container_name_pattern: ^envoy$
+              job_name: ""
+              metrics_path: /metrics
+              metrics_ports:
+                - 9902
     entitystore:
         mode: ec2
         region: us-east-1
@@ -120,6 +158,7 @@ service:
     extensions:
         - agenthealth/logs
         - agenthealth/statuscode
+        - ecs_observer
         - entitystore
     pipelines:
         metrics/prometheus/cloudwatchlogs:
diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go
@@ -924,7 +924,7 @@ func verifyToTomlTranslation(t *testing.T, input interface{}, desiredTomlPath st
 	_, decodeError2 := toml.Decode(tomlStr, &actual)
 	assert.NoError(t, decodeError2)
 
-	//assert.NoError(t, os.WriteFile(desiredTomlPath, []byte(tomlStr), 0644)) // useful for regenerating TOML
+	// assert.NoError(t, os.WriteFile(desiredTomlPath, []byte(tomlStr), 0644)) // useful for regenerating TOML
 
 	// This less function sort the content of string slice in alphabetical order so the
 	// cmp.Equal method will compare the two struct with slices in them, regardless the elements within the slices
@@ -956,8 +956,7 @@ func verifyToYamlTranslation(t *testing.T, input interface{}, expectedYamlFilePa
 		require.NoError(t, err)
 		yamlStr := toyamlconfig.ToYamlConfig(yamlConfig)
 		require.NoError(t, yaml.Unmarshal([]byte(yamlStr), &actual))
-
-		//assert.NoError(t, os.WriteFile(expectedYamlFilePath, []byte(yamlStr), 0644)) // useful for regenerating YAML
+		// assert.NoError(t, os.WriteFile(expectedYamlFilePath, []byte(yamlStr), 0644)) // useful for regenerating YAML
 		opt := cmpopts.SortSlices(func(x, y interface{}) bool {
 			return pretty.Sprint(x) < pretty.Sprint(y)
 		})
diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go
@@ -31,6 +31,7 @@ const (
 	TracesCollectedKey                 = "traces_collected"
 	MetricsDestinationsKey             = "metrics_destinations"
 	ECSKey                             = "ecs"
+	ECSServiceDiscovery                = "ecs_service_discovery"
 	KubernetesKey                      = "kubernetes"
 	CloudWatchKey                      = "cloudwatch"
 	CloudWatchLogsKey                  = "cloudwatchlogs"
diff --git a/translator/translate/otel/extension/ecsobserver/translator.go b/translator/translate/otel/extension/ecsobserver/translator.go
@@ -0,0 +1,172 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+package ecsobserver
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/ecsobserver"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/confmap"
+
+	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common"
+)
+
+const (
+	defaultMetricsPath      = "/metrics"
+	defaultPortLabel        = "ECS_PROMETHEUS_EXPORTER_PORT"
+	defaultMetricsPathLabel = "ECS_PROMETHEUS_METRICS_PATH"
+	defaultJobNameLabel     = ""
+)
+
+var ecsSDKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.PrometheusKey, common.ECSServiceDiscovery)
+
+type translator struct {
+	factory component.Factory
+	name    string
+}
+
+var _ common.ComponentTranslator = (*translator)(nil)
+var _ common.NameSetter = (*translator)(nil)
+
+func NewTranslator(opts ...common.TranslatorOption) common.ComponentTranslator {
+	t := &translator{
+		factory: ecsobserver.NewFactory(),
+	}
+	for _, opt := range opts {
+		opt(t)
+	}
+	return t
+}
+
+func (t *translator) ID() component.ID {
+	return component.NewIDWithName(t.factory.Type(), t.name)
+}
+
+func (t *translator) SetName(name string) {
+	t.name = name
+}
+
+func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) {
+	if conf == nil || !conf.IsSet(ecsSDKey) {
+		return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: ecsSDKey}
+	}
+
+	ecsSDValue := conf.Get(ecsSDKey)
+	ecsSD, ok := ecsSDValue.(map[string]interface{})
+	if !ok {
+		return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: ecsSDKey}
+	}
+
+	requiredFields := []string{"sd_target_cluster", "sd_cluster_region", "sd_result_file"}
+	for _, field := range requiredFields {
+		if _, ok := ecsSD[field]; !ok {
+			return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: field}
+		}
+	}
+
+	refreshDuration, err := time.ParseDuration(getStringWithDefault(ecsSD, "sd_frequency", "10s"))
+	if err != nil {
+		return nil, fmt.Errorf("invalid refresh interval: %w", err)
+	}
+
+	cfg := &ecsobserver.Config{
+		RefreshInterval: refreshDuration,
+		ClusterName:     getString(ecsSD, "sd_target_cluster"),
+		ClusterRegion:   getString(ecsSD, "sd_cluster_region"),
+		ResultFile:      getString(ecsSD, "sd_result_file"),
+	}
+	// Docker label based service discovery
+	if dockerLabel, ok := ecsSD["docker_label"].(map[string]interface{}); ok {
+		dockerConfig := ecsobserver.DockerLabelConfig{
+			MetricsPathLabel: getStringWithDefault(dockerLabel, "sd_metrics_path_label", defaultMetricsPath),
+			PortLabel:        getStringWithDefault(dockerLabel, "sd_port_label", defaultPortLabel),
+			JobNameLabel:     getString(dockerLabel, "sd_job_name_label"),
+		}
+		cfg.DockerLabels = []ecsobserver.DockerLabelConfig{dockerConfig} // Initialize as slice with single element
+	}
+
+	// Task definition based service discovery
+	if taskDefs, ok := ecsSD["task_definition_list"].([]interface{}); ok {
+		for _, td := range taskDefs {
+			if tdMap, ok := td.(map[string]interface{}); ok {
+				ports := parseMetricsPorts(getString(tdMap, "sd_metrics_ports"))
+				taskConfig := ecsobserver.TaskDefinitionConfig{
+					CommonExporterConfig: ecsobserver.CommonExporterConfig{
+						JobName:      getString(tdMap, "sd_job_name"),
+						MetricsPath:  getStringWithDefault(tdMap, "sd_metrics_path", defaultMetricsPath),
+						MetricsPorts: convertStringPortsToInt(ports),
+					},
+					ArnPattern:           getString(tdMap, "sd_task_definition_arn_pattern"),
+					ContainerNamePattern: getString(tdMap, "sd_container_name_pattern"),
+				}
+				cfg.TaskDefinitions = append(cfg.TaskDefinitions, taskConfig)
+			}
+		}
+	}
+
+	// Service name based service discovery
+	if services, ok := ecsSD["service_name_list_for_tasks"].([]interface{}); ok {
+		for _, svc := range services {
+			if svcMap, ok := svc.(map[string]interface{}); ok {
+				ports := parseMetricsPorts(getString(svcMap, "sd_metrics_ports"))
+				serviceConfig := ecsobserver.ServiceConfig{
+					CommonExporterConfig: ecsobserver.CommonExporterConfig{
+						JobName:      getString(svcMap, "sd_job_name"),
+						MetricsPath:  getStringWithDefault(svcMap, "sd_metrics_path", defaultMetricsPath),
+						MetricsPorts: convertStringPortsToInt(ports),
+					},
+					NamePattern:          getString(svcMap, "sd_service_name_pattern"),
+					ContainerNamePattern: getString(svcMap, "sd_container_name_pattern"),
+				}
+				cfg.Services = append(cfg.Services, serviceConfig)
+			}
+		}
+	}
+
+	return cfg, nil
+}
+
+// Add helper function to convert string ports to int
+func convertStringPortsToInt(ports []string) []int {
+	result := make([]int, 0, len(ports))
+	for _, port := range ports {
+		if p, err := strconv.Atoi(port); err == nil {
+			result = append(result, p)
+		}
+	}
+	return result
+}
+
+// Helper functions
+func parseMetricsPorts(ports string) []string {
+	if ports == "" {
+		return nil
+	}
+	portList := strings.Split(ports, ";")
+	var cleanPorts []string
+	for _, port := range portList {
+		if trimmed := strings.TrimSpace(port); trimmed != "" {
+			cleanPorts = append(cleanPorts, trimmed)
+		}
+	}
+	return cleanPorts
+}
+
+func getString(m map[string]interface{}, key string) string {
+	if val, ok := m[key].(string); ok {
+		return val
+	}
+	return ""
+}
+
+func getStringWithDefault(m map[string]interface{}, key, defaultValue string) string {
+	if val := getString(m, key); val != "" {
+		return val
+	}
+	return defaultValue
+}
diff --git a/translator/translate/otel/extension/ecsobserver/translator_test.go b/translator/translate/otel/extension/ecsobserver/translator_test.go
diff --git a/translator/translate/otel/pipeline/prometheus/translator.go b/translator/translate/otel/pipeline/prometheus/translator.go
