Adding Prometheus Adapter processor to prometheus pipeline (#1767)

Author: Paramadon

plugins/processors/prometheusadapter/internal/delta_calculator.go

@@ -8,10 +8,11 @@ import (
 	"math"
 	"time"
 
-	"github.com/aws/amazon-cloudwatch-agent/internal/mapWithExpiry"
 	"github.com/prometheus/prometheus/model/value"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/pmetric"
+
+	"github.com/aws/amazon-cloudwatch-agent/internal/mapWithExpiry"
 )
 
 const (

plugins/processors/prometheusadapter/processor.go

@@ -13,10 +13,11 @@ import (
 	semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
 	"go.uber.org/zap"
 
-	"github.com/aws/amazon-cloudwatch-agent/internal/util/collections"
-	"github.com/aws/amazon-cloudwatch-agent/plugins/processors/prometheusadapter/internal"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus"
 	"github.com/prometheus/common/model"
+
+	"github.com/aws/amazon-cloudwatch-agent/internal/util/collections"
+	"github.com/aws/amazon-cloudwatch-agent/plugins/processors/prometheusadapter/internal"
 )
 
 var extraneousMetrics = collections.NewSet(

translator/tocwconfig/sampleConfig/prometheus_combined_config_linux.yaml

@@ -202,6 +202,7 @@ processors:
     deltatocumulative/prometheus/amp:
         max_stale: 336h0m0s
         max_streams: 9223372036854775807
+    prometheusadapter/prometheus/cloudwatch: {}
     prometheusadapter/prometheus/cloudwatchlogs: {}
 receivers:
     prometheus:
@@ -241,6 +242,43 @@ receivers:
         start_time_metric_regex: ""
         trim_metric_suffixes: false
         use_start_time_metric: false
+    prometheus/prometheus/cloudwatch:
+        config:
+            global:
+                evaluation_interval: 1m
+                scrape_interval: 5m
+                scrape_protocols:
+                    - OpenMetricsText1.0.0
+                    - OpenMetricsText0.0.1
+                    - PrometheusText1.0.0
+                    - PrometheusText0.0.4
+                scrape_timeout: 5s
+            scrape_configs:
+                - enable_compression: true
+                  enable_http2: true
+                  fallback_scrape_protocol: PrometheusText0.0.4
+                  file_sd_configs:
+                    - files:
+                        - {ecsSdFileName}
+                      refresh_interval: 5m
+                  follow_redirects: true
+                  honor_timestamps: true
+                  job_name: cwagent-ecs-file-sd-config
+                  metrics_path: /metrics
+                  sample_limit: 10000
+                  scheme: http
+                  scrape_interval: 5m
+                  scrape_protocols:
+                    - OpenMetricsText1.0.0
+                    - OpenMetricsText0.0.1
+                    - PrometheusText1.0.0
+                    - PrometheusText0.0.4
+                  scrape_timeout: 5s
+                  track_timestamps_staleness: false
+        report_extra_scrape_metrics: false
+        start_time_metric_regex: ""
+        trim_metric_suffixes: false
+        use_start_time_metric: false
     prometheus/prometheus/cloudwatchlogs:
         config:
             global:
@@ -282,9 +320,9 @@ service:
     extensions:
         - agenthealth/metrics
         - agenthealth/statuscode
-        - ecs_observer
         - sigv4auth
         - agenthealth/logs
+        - ecs_observer
         - entitystore
     pipelines:
         metrics/prometheus/amp:
@@ -299,10 +337,11 @@ service:
             exporters:
                 - awscloudwatch
             processors:
+                - prometheusadapter/prometheus/cloudwatch
                 - batch/prometheus/cloudwatch
                 - cumulativetodelta/prometheus/cloudwatch
             receivers:
-                - prometheus
+                - prometheus/prometheus/cloudwatch
         metrics/prometheus/cloudwatchlogs:
             exporters:
                 - awsemf/prometheus

translator/tocwconfig/sampleConfig/prometheus_pmd_config.yaml

@@ -47,20 +47,21 @@ processors:
         ec2_instance_tag_keys:
             - AutoScalingGroupName
         ec2_metadata_tags:
-            - InstanceType
             - ImageId
             - InstanceId
+            - InstanceType
         imds_retries: 1
         middleware: agenthealth/statuscode
         refresh_tags_interval: 0s
         refresh_volumes_interval: 0s
+    prometheusadapter/prometheus/cloudwatch: {}
     rollup:
         attribute_groups:
             - - InstanceId
               - InstanceType
         cache_size: 1000
 receivers:
-    prometheus:
+    prometheus/prometheus/cloudwatch:
         config:
             global:
                 evaluation_interval: 1m
@@ -105,12 +106,13 @@ service:
             exporters:
                 - awscloudwatch
             processors:
+                - prometheusadapter/prometheus/cloudwatch
                 - batch/prometheus/cloudwatch
                 - cumulativetodelta/prometheus/cloudwatch
                 - rollup
                 - ec2tagger
             receivers:
-                - prometheus
+                - prometheus/prometheus/cloudwatch
     telemetry:
         logs:
             encoding: console

translator/translate/otel/pipeline/prometheus/translator.go

@@ -72,8 +72,9 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators
 			return nil, fmt.Errorf("pipeline (%s) is missing prometheus configuration under metrics section with destination (%s)", t.name, t.Destination())
 		}
 		translators := &common.ComponentTranslators{
-			Receivers: common.NewTranslatorMap(otelprom.NewTranslator()),
+			Receivers: common.NewTranslatorMap(otelprom.NewTranslator(otelprom.WithName(t.name), otelprom.WithConfigKey(common.ConfigKey(common.MetricsKey, common.MetricsCollectedKey, common.PrometheusKey)))),
 			Processors: common.NewTranslatorMap(
+				prometheusadapter.NewTranslatorWithName(t.name),
 				batchprocessor.NewTranslatorWithNameAndSection(t.name, common.MetricsKey),
 				cumulativetodeltaprocessor.NewTranslator(common.WithName(t.name), cumulativetodeltaprocessor.WithDefaultKeys()),
 			),

translator/translate/otel/translate_otel_test.go

@@ -16,7 +16,6 @@ import (
 	_ "github.com/aws/amazon-cloudwatch-agent/translator/registerrules"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/agent"
 	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common"
-	"github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/prometheus"
 	"github.com/aws/amazon-cloudwatch-agent/translator/util/eksdetector"
 )
 
@@ -206,7 +205,7 @@ func TestTranslator(t *testing.T) {
 					},
 				},
 			},
-			wantErrContains: common.ConfigKey(prometheus.MetricsKey, common.PrometheusConfigPathKey),
+			wantErrContains: common.ConfigKey(common.PrometheusConfigPathKey),
 		},
 	}
 	for name, testCase := range testCases {