Adding cloudwatch as destination for prometheus (#1742)

Paramadon · agarakan · commit f7bcd28813c8 · 2025-08-19T20:03:24.000Z
diff --git a/translator/tocwconfig/sampleConfig/prometheus_combined_config_linux.yaml b/translator/tocwconfig/sampleConfig/prometheus_combined_config_linux.yaml
@@ -1,4 +1,13 @@
 exporters:
+    awscloudwatch:
+        force_flush_interval: 1m0s
+        max_datums_per_call: 1000
+        max_values_per_datum: 150
+        middleware: agenthealth/metrics
+        namespace: CWAgent
+        region: us-west-2
+        resource_to_telemetry_conversion:
+            enabled: true
     awsemf/prometheus:
         add_entity: false
         certificate_file_path: ""
@@ -107,6 +116,14 @@ extensions:
             usage_flags:
                 mode: EC2
                 region_type: ACJ
+    agenthealth/metrics:
+        is_usage_data_enabled: true
+        stats:
+            operations:
+                - PutMetricData
+            usage_flags:
+                mode: EC2
+                region_type: ACJ
     agenthealth/statuscode:
         is_status_code_enabled: true
         is_usage_data_enabled: true
@@ -127,11 +144,23 @@ processors:
         send_batch_max_size: 0
         send_batch_size: 8192
         timeout: 1m0s
+    batch/prometheus/cloudwatch:
+        metadata_cardinality_limit: 1000
+        send_batch_max_size: 0
+        send_batch_size: 8192
+        timeout: 1m0s
     batch/prometheus/cloudwatchlogs:
         metadata_cardinality_limit: 1000
         send_batch_max_size: 0
         send_batch_size: 8192
         timeout: 30s
+    cumulativetodelta/prometheus/cloudwatch:
+        exclude:
+            match_type: ""
+        include:
+            match_type: ""
+        initial_value: 2
+        max_staleness: 0s
     deltatocumulative/prometheus/amp:
         max_stale: 336h0m0s
         max_streams: 9223372036854775807
@@ -213,9 +242,10 @@ receivers:
         use_start_time_metric: false
 service:
     extensions:
-        - agenthealth/logs
+        - agenthealth/metrics
         - agenthealth/statuscode
         - sigv4auth
+        - agenthealth/logs
         - entitystore
     pipelines:
         metrics/prometheus/amp:
@@ -226,6 +256,14 @@ service:
                 - deltatocumulative/prometheus/amp
             receivers:
                 - prometheus
+        metrics/prometheus/cloudwatch:
+            exporters:
+                - awscloudwatch
+            processors:
+                - batch/prometheus/cloudwatch
+                - cumulativetodelta/prometheus/cloudwatch
+            receivers:
+                - prometheus
         metrics/prometheus/cloudwatchlogs:
             exporters:
                 - awsemf/prometheus
diff --git a/translator/tocwconfig/sampleConfig/prometheus_cwa_config.yaml b/translator/tocwconfig/sampleConfig/prometheus_cwa_config.yaml
@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 5s
+  scrape_timeout: 5s
+scrape_configs:
+  - job_name: 'prometheus_test_job'
+    static_configs:
+      - targets: ['localhost:8101']
diff --git a/translator/tocwconfig/sampleConfig/prometheus_pmd_config.conf b/translator/tocwconfig/sampleConfig/prometheus_pmd_config.conf
@@ -0,0 +1,19 @@
+[agent]
+  collection_jitter = "0s"
+  debug = false
+  flush_interval = "1s"
+  flush_jitter = "0s"
+  hostname = "host_name_from_env"
+  interval = "15s"
+  logfile = ""
+  logtarget = "lumberjack"
+  metric_batch_size = 1000
+  metric_buffer_limit = 10000
+  omit_hostname = false
+  precision = ""
+  quiet = false
+  round_interval = false
+
+[outputs]
+
+  [[outputs.cloudwatch]]
diff --git a/translator/tocwconfig/sampleConfig/prometheus_pmd_config.json b/translator/tocwconfig/sampleConfig/prometheus_pmd_config.json
@@ -0,0 +1,28 @@
+{
+  "agent": {
+    "metrics_collection_interval": 15
+  },
+  "metrics": {
+    "namespace": "CWAgent",
+    "metrics_destinations": {
+      "cloudwatch": {}
+    },
+    "metrics_collected": {
+      "prometheus": {
+        "prometheus_config_path": "{prometheusFileName}"
+      }
+    },
+    "append_dimensions": {
+      "ImageId": "${aws:ImageId}",
+      "InstanceId": "${aws:InstanceId}",
+      "InstanceType": "${aws:InstanceType}",
+      "AutoScalingGroupName": "${aws:AutoScalingGroupName}"
+    },
+    "aggregation_dimensions": [
+      [
+        "InstanceId",
+        "InstanceType"
+      ]
+    ]
+  }
+}
diff --git a/translator/tocwconfig/sampleConfig/prometheus_pmd_config.yaml b/translator/tocwconfig/sampleConfig/prometheus_pmd_config.yaml
@@ -0,0 +1,126 @@
+exporters:
+    awscloudwatch:
+        force_flush_interval: 1m0s
+        max_datums_per_call: 1000
+        max_values_per_datum: 150
+        middleware: agenthealth/metrics
+        namespace: CWAgent
+        region: us-west-2
+        resource_to_telemetry_conversion:
+            enabled: true
+        rollup_dimensions:
+            - - InstanceId
+              - InstanceType
+extensions:
+    agenthealth/metrics:
+        is_usage_data_enabled: true
+        stats:
+            operations:
+                - PutMetricData
+            usage_flags:
+                mode: EC2
+                region_type: ACJ
+    agenthealth/statuscode:
+        is_status_code_enabled: true
+        is_usage_data_enabled: true
+        stats:
+            usage_flags:
+                mode: EC2
+                region_type: ACJ
+    entitystore:
+        mode: ec2
+        region: us-west-2
+processors:
+    batch/prometheus/cloudwatch:
+        metadata_cardinality_limit: 1000
+        send_batch_max_size: 0
+        send_batch_size: 8192
+        timeout: 1m0s
+    cumulativetodelta/prometheus/cloudwatch:
+        exclude:
+            match_type: ""
+        include:
+            match_type: ""
+        initial_value: 2
+        max_staleness: 0s
+    ec2tagger:
+        ec2_instance_tag_keys:
+            - AutoScalingGroupName
+        ec2_metadata_tags:
+            - InstanceType
+            - ImageId
+            - InstanceId
+        imds_retries: 1
+        middleware: agenthealth/statuscode
+        refresh_tags_interval: 0s
+        refresh_volumes_interval: 0s
+    rollup:
+        attribute_groups:
+            - - InstanceId
+              - InstanceType
+        cache_size: 1000
+receivers:
+    prometheus:
+        config:
+            global:
+                evaluation_interval: 1m
+                scrape_interval: 5s
+                scrape_protocols:
+                    - OpenMetricsText1.0.0
+                    - OpenMetricsText0.0.1
+                    - PrometheusText1.0.0
+                    - PrometheusText0.0.4
+                scrape_timeout: 5s
+            scrape_configs:
+                - enable_compression: true
+                  enable_http2: true
+                  fallback_scrape_protocol: PrometheusText0.0.4
+                  follow_redirects: true
+                  honor_timestamps: true
+                  job_name: prometheus_test_job
+                  metrics_path: /metrics
+                  scheme: http
+                  scrape_interval: 5s
+                  scrape_protocols:
+                    - OpenMetricsText1.0.0
+                    - OpenMetricsText0.0.1
+                    - PrometheusText1.0.0
+                    - PrometheusText0.0.4
+                  scrape_timeout: 5s
+                  static_configs:
+                    - targets:
+                        - localhost:8101
+                  track_timestamps_staleness: false
+        report_extra_scrape_metrics: false
+        start_time_metric_regex: ""
+        trim_metric_suffixes: false
+        use_start_time_metric: false
+service:
+    extensions:
+        - agenthealth/metrics
+        - agenthealth/statuscode
+        - entitystore
+    pipelines:
+        metrics/prometheus/cloudwatch:
+            exporters:
+                - awscloudwatch
+            processors:
+                - batch/prometheus/cloudwatch
+                - cumulativetodelta/prometheus/cloudwatch
+                - rollup
+                - ec2tagger
+            receivers:
+                - prometheus
+    telemetry:
+        logs:
+            encoding: console
+            level: info
+            sampling:
+                enabled: true
+                initial: 2
+                thereafter: 500
+                tick: 10s
+        metrics:
+            level: None
+        traces:
+            level: None
diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go
@@ -51,6 +51,9 @@ const (
 //go:embed sampleConfig/prometheus_config.yaml
 var prometheusConfig string
 
+//go:embed sampleConfig/prometheus_cwa_config.yaml
+var prometheusPMDConfig string
+
 type testCase struct {
 	filename        string
 	targetPlatform  string
@@ -439,6 +442,26 @@ func TestPrometheusConfig(t *testing.T) {
 	checkTranslation(t, "prometheus_config_linux", "linux", expectedEnvVars, "", tokenReplacements)
 	checkTranslation(t, "prometheus_config_windows", "windows", nil, "", tokenReplacements)
 }
+
+func TestPrometheusPMDConfig(t *testing.T) {
+	resetContext(t)
+	context.CurrentContext().SetRunInContainer(true)
+	context.CurrentContext().SetMode(config.ModeEC2)
+	t.Setenv(config.HOST_NAME, "host_name_from_env")
+
+	temp := t.TempDir()
+	prometheusConfigFileName := filepath.Join(temp, "prometheus.yaml")
+	err := os.WriteFile(prometheusConfigFileName, []byte(prometheusPMDConfig), 0600)
+	require.NoError(t, err)
+
+	tokenReplacements := map[string]string{
+		prometheusFileNameToken: strings.ReplaceAll(prometheusConfigFileName, "\\", "\\\\"),
+	}
+
+	expectedEnvVars := map[string]string{}
+	checkTranslation(t, "prometheus_pmd_config", "linux", expectedEnvVars, "", tokenReplacements)
+}
+
 func TestPrometheusConfigwithTargetAllocator(t *testing.T) {
 	resetContext(t)
 	context.CurrentContext().SetRunInContainer(true)
diff --git a/translator/translate/otel/pipeline/prometheus/translator.go b/translator/translate/otel/pipeline/prometheus/translator.go
@@ -5,17 +5,21 @@ package prometheus
 
 import (
 	"fmt"
+	"log"
 
 	"go.opentelemetry.io/collector/confmap"
 	"go.opentelemetry.io/collector/pipeline"
 
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common"
+	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awscloudwatch"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsemf"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/prometheusremotewrite"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/agenthealth"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/sigv4auth"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/batchprocessor"
+	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/cumulativetodeltaprocessor"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/deltatocumulativeprocessor"
+	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/ec2taggerprocessor"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/prometheusadapter"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/rollupprocessor"
 	otelprom "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/prometheus"
@@ -56,10 +60,38 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators
 	}
 
 	// return pipeline based on destination to keep source/destination combinations clearly separated
+	// otel_prometheus - cloudwatch
 	// telegraf_prometheus - cloudwatch
 	// otel_prometheus - AMP
 	// this could change in future releases to support different source/destination combinations
 	switch t.Destination() {
+	case common.DefaultDestination, common.CloudWatchKey:
+		if !conf.IsSet(MetricsKey) {
+			return nil, fmt.Errorf("pipeline (%s) is missing prometheus configuration under metrics section with destination (%s)", t.name, t.Destination())
+		}
+		translators := &common.ComponentTranslators{
+			Receivers: common.NewTranslatorMap(otelprom.NewTranslator()),
+			Processors: common.NewTranslatorMap(
+				batchprocessor.NewTranslatorWithNameAndSection(t.name, common.MetricsKey),
+				cumulativetodeltaprocessor.NewTranslator(common.WithName(t.name), cumulativetodeltaprocessor.WithDefaultKeys()),
+			),
+			Exporters: common.NewTranslatorMap(awscloudwatch.NewTranslator()),
+			Extensions: common.NewTranslatorMap(
+				agenthealth.NewTranslator(agenthealth.MetricsName, []string{agenthealth.OperationPutMetricData}),
+				agenthealth.NewTranslatorWithStatusCode(agenthealth.StatusCodeName, nil, true),
+			),
+		}
+
+		if conf.IsSet(common.MetricsAggregationDimensionsKey) {
+			translators.Processors.Set(rollupprocessor.NewTranslator())
+		}
+
+		if conf.IsSet(common.ConfigKey(common.MetricsKey, common.AppendDimensionsKey)) {
+			log.Printf("D! ec2tagger processor required because append_dimensions is set")
+			translators.Processors.Set(ec2taggerprocessor.NewTranslator())
+		}
+
+		return translators, nil
 	case common.CloudWatchLogsKey:
 		if !conf.IsSet(LogsKey) {
 			return nil, fmt.Errorf("pipeline (%s) is missing prometheus configuration under logs section with destination (%s)", t.name, t.Destination())
diff --git a/translator/translate/otel/pipeline/prometheus/translators.go b/translator/translate/otel/pipeline/prometheus/translators.go
@@ -12,13 +12,11 @@ import (
 
 func NewTranslators(conf *confmap.Conf) common.PipelineTranslatorMap {
 	translators := common.NewTranslatorMap[*common.ComponentTranslators, pipeline.ID]()
-	var destinations []string
+
+	destinations := common.GetMetricsDestinations(conf)
 	if conf.IsSet(LogsKey) {
 		destinations = append(destinations, common.CloudWatchLogsKey)
 	}
-	if conf.IsSet(MetricsKey) {
-		destinations = append(destinations, common.AMPKey)
-	}
 
 	for _, destination := range destinations {
 		translators.Set(NewTranslator(common.WithDestination(destination)))
diff --git a/translator/translate/otel/pipeline/prometheus/translators_test.go b/translator/translate/otel/pipeline/prometheus/translators_test.go
diff --git a/translator/translate/otel/processor/cumulativetodeltaprocessor/translator.go b/translator/translate/otel/processor/cumulativetodeltaprocessor/translator.go
diff --git a/translator/translate/otel/processor/cumulativetodeltaprocessor/translator_test.go b/translator/translate/otel/processor/cumulativetodeltaprocessor/translator_test.go
