rename to exportAllDataPoints and added test

woehrl01 · woehrl01 · commit 1d3536c05083 · 2025-03-07T08:23:31.000+01:00
Signed-off-by: Lukas Wöhrl &lt;lukas.woehrl@plentymarkets.com&gt;
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -140,9 +140,10 @@ statistics:
 # Enables the inclusion of past metric data points from the CloudWatch response if available.
 # This is useful when a metric is configured with a 60-second period and a 300-second duration, ensuring that all
 # five data points are exposed at the metrics endpoint instead of only the latest one.
-# Note: This option requires `addCloudwatchTimestamp` to be enabled
+# Note: This option requires `addCloudwatchTimestamp` to be enabled.
+# The metric destination must support out of order timestamps, see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tsdb
 # (General Setting for all metrics in this job)
-[ addHistoricalMetrics: <boolean> ]
+[ exportAllDataPoints: <boolean> ]
 
 # List of metric definitions
 metrics:
@@ -282,9 +283,10 @@ statistics:
 # Enables the inclusion of past metric data points from the CloudWatch response if available.
 # This is useful when a metric is configured with a 60-second period and a 300-second duration, ensuring that all
 # five data points are exposed at the metrics endpoint instead of only the latest one.
-# Note: This option requires `addCloudwatchTimestamp` to be enabled
+# Note: This option requires `addCloudwatchTimestamp` to be enabled.
+# The metric destination must support out of order timestamps, see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tsdb
 # (General Setting for all metrics in this job)
-[ addHistoricalMetrics: <boolean> ]
+[ exportAllDataPoints: <boolean> ]
 
 # List of metric definitions
 metrics:
@@ -349,7 +351,7 @@ Notes:
 
 - Available statistics: `Maximum`, `Minimum`, `Sum`, `SampleCount`, `Average`, `pXX` (e.g. `p90`).
 
-- Watch out using `addCloudwatchTimestamp` for sparse metrics, e.g from S3, since Prometheus won't scrape metrics containing timestamps older than 2-3 hours. Also the same applies when enabling `addHistoricalMetrics` in any metric
+- Watch out using `addCloudwatchTimestamp` for sparse metrics, e.g from S3, since Prometheus won't scrape metrics containing timestamps older than 2-3 hours. Also the same applies when enabling `exportAllDataPoints` in any metric
 
 ### `exported_tags_config`
 
diff --git a/examples/historic-data.yml b/examples/historic-data.yml
@@ -7,7 +7,7 @@ discovery:
       period: 60
       length: 300
       addCloudwatchTimestamp: true
-      addHistoricalMetrics: true
+      exportAllDataPoints: true
       metrics:
         - name: NumberOfMessagesSent
           statistics: [Sum]
diff --git a/pkg/clients/cloudwatch/v1/client.go b/pkg/clients/cloudwatch/v1/client.go
@@ -147,10 +147,7 @@ func toMetricDataResult(resp cloudwatch.GetMetricDataOutput) []cloudwatch_client
 			ID:         *metricDataResult.Id,
 			Datapoints: make([]cloudwatch_client.DatapointWithTimestamp, 0, len(metricDataResult.Timestamps))}
 		for i := 0; i < len(metricDataResult.Timestamps); i++ {
-			mappedResult.Datapoints = append(mappedResult.Datapoints, cloudwatch_client.DatapointWithTimestamp{
-				Datapoint: metricDataResult.Values[i],
-				Timestamp: *metricDataResult.Timestamps[i],
-			})
+			mappedResult.Datapoints = append(mappedResult.Datapoints, cloudwatch_client.NewDataPoint(metricDataResult.Values[i], *metricDataResult.Timestamps[i]))
 		}
 		output = append(output, mappedResult)
 	}
diff --git a/pkg/clients/cloudwatch/v2/client.go b/pkg/clients/cloudwatch/v2/client.go
@@ -154,10 +154,7 @@ func toMetricDataResult(resp cloudwatch.GetMetricDataOutput) []cloudwatch_client
 			Datapoints: make([]cloudwatch_client.DatapointWithTimestamp, 0, len(metricDataResult.Timestamps)),
 		}
 		for i := 0; i < len(metricDataResult.Timestamps); i++ {
-			mappedResult.Datapoints = append(mappedResult.Datapoints, cloudwatch_client.DatapointWithTimestamp{
-				Datapoint: &metricDataResult.Values[i],
-				Timestamp: metricDataResult.Timestamps[i],
-			})
+			mappedResult.Datapoints = append(mappedResult.Datapoints, cloudwatch_client.NewDataPoint(&metricDataResult.Values[i], metricDataResult.Timestamps[i]))
 		}
 		output = append(output, mappedResult)
 	}
diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -52,7 +52,7 @@ type JobLevelMetricFields struct {
 	Delay                  int64    `yaml:"delay"`
 	NilToZero              *bool    `yaml:"nilToZero"`
 	AddCloudwatchTimestamp *bool    `yaml:"addCloudwatchTimestamp"`
-	AddHistoricalMetrics   *bool    `yaml:"addHistoricalMetrics"`
+	ExportAllDataPoints    *bool    `yaml:"exportAllDataPoints"`
 }
 
 type Job struct {
@@ -100,7 +100,7 @@ type Metric struct {
 	Delay                  int64    `yaml:"delay"`
 	NilToZero              *bool    `yaml:"nilToZero"`
 	AddCloudwatchTimestamp *bool    `yaml:"addCloudwatchTimestamp"`
-	AddHistoricalMetrics   *bool    `yaml:"addHistoricalMetrics"`
+	ExportAllDataPoints    *bool    `yaml:"exportAllDataPoints"`
 }
 
 type Dimension struct {
@@ -389,17 +389,17 @@ func (m *Metric) validateMetric(logger *slog.Logger, metricIdx int, parent strin
 		}
 	}
 
-	mAddHistoricalMetrics := m.AddHistoricalMetrics
-	if mAddHistoricalMetrics == nil {
-		if discovery != nil && discovery.AddHistoricalMetrics != nil {
-			mAddHistoricalMetrics = discovery.AddHistoricalMetrics
+	mExportAllDataPoints := m.ExportAllDataPoints
+	if mExportAllDataPoints == nil {
+		if discovery != nil && discovery.ExportAllDataPoints != nil {
+			mExportAllDataPoints = discovery.ExportAllDataPoints
 		} else {
-			mAddHistoricalMetrics = aws.Bool(false)
+			mExportAllDataPoints = aws.Bool(false)
 		}
 	}
 
-	if aws.BoolValue(mAddHistoricalMetrics) && !aws.BoolValue(mAddCloudwatchTimestamp) {
-		return fmt.Errorf("Metric [%s/%d] in %v: AddHistoricalMetrics can only be enabled if AddCloudwatchTimestamp is enabled", m.Name, metricIdx, parent)
+	if aws.BoolValue(mExportAllDataPoints) && !aws.BoolValue(mAddCloudwatchTimestamp) {
+		return fmt.Errorf("Metric [%s/%d] in %v: ExportAllDataPoints can only be enabled if AddCloudwatchTimestamp is enabled", m.Name, metricIdx, parent)
 	}
 
 	if mLength < mPeriod {
@@ -413,7 +413,7 @@ func (m *Metric) validateMetric(logger *slog.Logger, metricIdx int, parent strin
 	m.Delay = mDelay
 	m.NilToZero = mNilToZero
 	m.AddCloudwatchTimestamp = mAddCloudwatchTimestamp
-	m.AddHistoricalMetrics = mAddHistoricalMetrics
+	m.ExportAllDataPoints = mExportAllDataPoints
 	m.Statistics = mStatistics
 
 	return nil
@@ -535,7 +535,7 @@ func toModelMetricConfig(metrics []*Metric) []*model.MetricConfig {
 			Delay:                  m.Delay,
 			NilToZero:              aws.BoolValue(m.NilToZero),
 			AddCloudwatchTimestamp: aws.BoolValue(m.AddCloudwatchTimestamp),
-			AddHistoricalMetrics:   aws.BoolValue(m.AddHistoricalMetrics),
+			ExportAllDataPoints:    aws.BoolValue(m.ExportAllDataPoints),
 		})
 	}
 	return ret
diff --git a/pkg/job/custom.go b/pkg/job/custom.go
@@ -86,7 +86,7 @@ func getMetricDataForQueriesForCustomNamespace(
 							MetricMigrationParams: model.MetricMigrationParams{
 								NilToZero:              metric.NilToZero,
 								AddCloudwatchTimestamp: metric.AddCloudwatchTimestamp,
-								AddHistoricalMetrics:   metric.AddHistoricalMetrics,
+								ExportAllDataPoints:    metric.ExportAllDataPoints,
 							},
 							Tags:                      nil,
 							GetMetricDataResult:       nil,
diff --git a/pkg/job/discovery.go b/pkg/job/discovery.go
@@ -179,7 +179,7 @@ func getFilteredMetricDatas(
 				MetricMigrationParams: model.MetricMigrationParams{
 					NilToZero:              m.NilToZero,
 					AddCloudwatchTimestamp: m.AddCloudwatchTimestamp,
-					AddHistoricalMetrics:   m.AddHistoricalMetrics,
+					ExportAllDataPoints:    m.ExportAllDataPoints,
 				},
 				Tags:                      metricTags,
 				GetMetricDataResult:       nil,
diff --git a/pkg/job/getmetricdata/processor_test.go b/pkg/job/getmetricdata/processor_test.go
@@ -259,7 +259,7 @@ func getSampleMetricDatas(id string) *model.CloudwatchData {
 		MetricMigrationParams: model.MetricMigrationParams{
 			NilToZero:              false,
 			AddCloudwatchTimestamp: false,
-			AddHistoricalMetrics:   false,
+			ExportAllDataPoints:    false,
 		},
 		GetMetricDataProcessingParams: &model.GetMetricDataProcessingParams{
 			Period:    60,
diff --git a/pkg/model/model.go b/pkg/model/model.go
@@ -80,7 +80,7 @@ type MetricConfig struct {
 	Delay                  int64
 	NilToZero              bool
 	AddCloudwatchTimestamp bool
-	AddHistoricalMetrics   bool
+	ExportAllDataPoints    bool
 }
 
 type DimensionsRegexp struct {
@@ -199,7 +199,7 @@ type GetMetricDataProcessingParams struct {
 type MetricMigrationParams struct {
 	NilToZero              bool
 	AddCloudwatchTimestamp bool
-	AddHistoricalMetrics   bool
+	ExportAllDataPoints    bool
 }
 
 type GetMetricDataResult struct {
diff --git a/pkg/promutil/migrate.go b/pkg/promutil/migrate.go
@@ -143,8 +143,8 @@ func BuildMetrics(results []model.CloudwatchMetricResult, labelsSnakeCase bool,
 						IncludeTimestamp: metric.MetricMigrationParams.AddCloudwatchTimestamp,
 					})
 
-					if !metric.MetricMigrationParams.AddHistoricalMetrics {
-						// If we're not adding historical metrics, we can skip the rest of the statistics for this metric
+					if !metric.MetricMigrationParams.ExportAllDataPoints {
+						// If we're not adding historical metrics, we can skip the rest of the datapoints for this metric
 						break
 					}
 				}
@@ -326,7 +326,7 @@ func EnsureLabelConsistencyAndRemoveDuplicates(metrics []*PrometheusMetric, obse
 
 		// We are including the timestamp in the metric key to ensure that we don't have duplicate metrics
 		// if we have AddCloudwatchTimestamp enabled its the real timestamp, otherwise its a zero value
-		// the timestamp is needed to ensure valid date created by AddHistoricalMetrics
+		// the timestamp is needed to ensure valid date created by ExportAllDataPoints
 		metricKey := fmt.Sprintf("%s-%d-%d", metric.Name, prom_model.LabelsToSignature(metric.Labels), metric.Timestamp.Unix())
 		if _, exists := metricKeys[metricKey]; !exists {
 			metricKeys[metricKey] = struct{}{}
diff --git a/pkg/promutil/migrate_test.go b/pkg/promutil/migrate_test.go
@@ -316,8 +316,7 @@ func TestBuildMetrics(t *testing.T) {
 						},
 						Namespace: "AWS/ElastiCache",
 						GetMetricDataResult: &model.GetMetricDataResult{
-							Statistic: "Average",
-
+							Statistic:  "Average",
 							Datapoints: model.SingleDataPoint(aws.Float64(1), ts),
 						},
 						Dimensions: []model.Dimension{
@@ -390,7 +389,7 @@ func TestBuildMetrics(t *testing.T) {
 						MetricMigrationParams: model.MetricMigrationParams{
 							NilToZero:              true,
 							AddCloudwatchTimestamp: true,
-							AddHistoricalMetrics:   true,
+							ExportAllDataPoints:    true,
 						},
 						Namespace: "AWS/ElastiCache",
 						Dimensions: []model.Dimension{
@@ -409,6 +408,31 @@ func TestBuildMetrics(t *testing.T) {
 						},
 						ResourceName: "arn:aws:elasticache:us-east-1:123456789012:cluster:redis-cluster",
 					},
+					{
+						//Check that if there is no data point for the latest timestamp it exports the previous one
+						MetricName: "NetworkPacketsOut",
+						MetricMigrationParams: model.MetricMigrationParams{
+							NilToZero:              true,
+							AddCloudwatchTimestamp: true,
+							ExportAllDataPoints:    false,
+						},
+						Namespace: "AWS/ElastiCache",
+						Dimensions: []model.Dimension{
+							{
+								Name:  "CacheClusterId",
+								Value: "redis-cluster",
+							},
+						},
+						GetMetricDataResult: &model.GetMetricDataResult{
+							Statistic: "Average",
+							Datapoints: []model.DatapointWithTimestamp{
+								model.NewDataPoint(nil, ts),
+								model.NewDataPoint(aws.Float64(5), ts.Add(-1*time.Minute)),
+								model.NewDataPoint(aws.Float64(6), ts.Add(-2*time.Minute)),
+							},
+						},
+						ResourceName: "arn:aws:elasticache:us-east-1:123456789012:cluster:redis-cluster",
+					},
 				},
 			}},
 			labelsSnakeCase: false,
@@ -494,6 +518,18 @@ func TestBuildMetrics(t *testing.T) {
 						"dimension_CacheClusterId": "redis-cluster",
 					},
 				},
+				{
+					Name:             "aws_elasticache_network_packets_out_average",
+					Value:            5,
+					Timestamp:        ts.Add(-1 * time.Minute),
+					IncludeTimestamp: true,
+					Labels: map[string]string{
+						"account_id":               "123456789012",
+						"name":                     "arn:aws:elasticache:us-east-1:123456789012:cluster:redis-cluster",
+						"region":                   "us-east-1",
+						"dimension_CacheClusterId": "redis-cluster",
+					},
+				},
 			},
 			expectedLabels: map[string]model.LabelSet{
 				"aws_elasticache_cpuutilization_average": {
@@ -526,6 +562,12 @@ func TestBuildMetrics(t *testing.T) {
 					"region":                   {},
 					"dimension_CacheClusterId": {},
 				},
+				"aws_elasticache_network_packets_out_average": {
+					"account_id":               {},
+					"name":                     {},
+					"region":                   {},
+					"dimension_CacheClusterId": {},
+				},
 			},
 			expectedErr: nil,
 		},

Original file line number	Diff line number	Diff line change
`@@ -147,10 +147,7 @@ func toMetricDataResult(resp cloudwatch.GetMetricDataOutput) []cloudwatch_client`
`147`	`147`	`ID: *metricDataResult.Id,`
`148`	`148`	`Datapoints: make([]cloudwatch_client.DatapointWithTimestamp, 0, len(metricDataResult.Timestamps))}`
`149`	`149`	`for i := 0; i < len(metricDataResult.Timestamps); i++ {`
`150`		`- mappedResult.Datapoints = append(mappedResult.Datapoints, cloudwatch_client.DatapointWithTimestamp{`
`151`		`- Datapoint: metricDataResult.Values[i],`
`152`		`- Timestamp: *metricDataResult.Timestamps[i],`
`153`		`- })`
	`150`	`+ mappedResult.Datapoints = append(mappedResult.Datapoints, cloudwatch_client.NewDataPoint(metricDataResult.Values[i], *metricDataResult.Timestamps[i]))`
`154`	`151`	`}`
`155`	`152`	`output = append(output, mappedResult)`
`156`	`153`	`}`
Original file line number	Diff line number	Diff line change
`@@ -154,10 +154,7 @@ func toMetricDataResult(resp cloudwatch.GetMetricDataOutput) []cloudwatch_client`
`154`	`154`	`Datapoints: make([]cloudwatch_client.DatapointWithTimestamp, 0, len(metricDataResult.Timestamps)),`
`155`	`155`	`}`
`156`	`156`	`for i := 0; i < len(metricDataResult.Timestamps); i++ {`
`157`		`- mappedResult.Datapoints = append(mappedResult.Datapoints, cloudwatch_client.DatapointWithTimestamp{`
`158`		`- Datapoint: &metricDataResult.Values[i],`
`159`		`- Timestamp: metricDataResult.Timestamps[i],`
`160`		`- })`
	`157`	`+ mappedResult.Datapoints = append(mappedResult.Datapoints, cloudwatch_client.NewDataPoint(&metricDataResult.Values[i], metricDataResult.Timestamps[i]))`
`161`	`158`	`}`
`162`	`159`	`output = append(output, mappedResult)`
`163`	`160`	`}`