Skip to content

Commit a084066

Browse files
woehrl01woehrl01
committed
feat: reimplement history data export
Signed-off-by: Lukas Wöhrl <[email protected]>
1 parent 993a40b commit a084066

File tree

15 files changed

+302
-199
lines changed

15 files changed

+302
-199
lines changed

‎docs/configuration.md‎

Lines changed: 52 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -13,23 +13,23 @@ Command-line flags are used to configure settings of the exporter which cannot b
1313

1414
All flags may be prefixed with either one hypen or two (i.e., both `-config.file` and `--config.file` are valid).
1515

16-
| Flag | Description | Default value |
17-
| --- | --- | --- |
18-
| `-listen-address` | Network address to listen to | `127.0.0.1:5000` |
19-
| `-config.file` | Path to the configuration file | `config.yml` |
20-
| `-log.format` | Output format of log messages. One of: [logfmt, json] | `json` |
21-
| `-debug` | Log at debug level | `false` |
22-
| `-fips` | Use FIPS compliant AWS API | `false` |
23-
| `-cloudwatch-concurrency` | Maximum number of concurrent requests to CloudWatch API | `5` |
24-
| `-cloudwatch-concurrency.per-api-limit-enabled` | Enables a concurrency limiter, that has a specific limit per CloudWatch API call. | `false` |
25-
| `-cloudwatch-concurrency.list-metrics-limit` | Maximum number of concurrent requests to CloudWatch `ListMetrics` API. Only applicable if `per-api-limit-enabled` is `true`. | `5` |
26-
| `-cloudwatch-concurrency.get-metric-data-limit` | Maximum number of concurrent requests to CloudWatch `GetMetricsData` API. Only applicable if `per-api-limit-enabled` is `true`. | `5` |
27-
| `-cloudwatch-concurrency.get-metric-statistics-limit` | Maximum number of concurrent requests to CloudWatch `GetMetricStatistics` API. Only applicable if `per-api-limit-enabled` is `true`. | `5` |
28-
| `-tag-concurrency` | Maximum number of concurrent requests to Resource Tagging API | `5` |
29-
| `-scraping-interval` | Seconds to wait between scraping the AWS metrics | `300` |
30-
| `-metrics-per-query` | Number of metrics made in a single GetMetricsData request | `500` |
31-
| `-labels-snake-case` | Output labels on metrics in snake case instead of camel case | `false` |
32-
| `-profiling.enabled` | Enable the /debug/pprof endpoints for profiling | `false` |
16+
| Flag | Description | Default value |
17+
| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | ---------------- |
18+
| `-listen-address` | Network address to listen to | `127.0.0.1:5000` |
19+
| `-config.file` | Path to the configuration file | `config.yml` |
20+
| `-log.format` | Output format of log messages. One of: [logfmt, json] | `json` |
21+
| `-debug` | Log at debug level | `false` |
22+
| `-fips` | Use FIPS compliant AWS API | `false` |
23+
| `-cloudwatch-concurrency` | Maximum number of concurrent requests to CloudWatch API | `5` |
24+
| `-cloudwatch-concurrency.per-api-limit-enabled` | Enables a concurrency limiter, that has a specific limit per CloudWatch API call. | `false` |
25+
| `-cloudwatch-concurrency.list-metrics-limit` | Maximum number of concurrent requests to CloudWatch `ListMetrics` API. Only applicable if `per-api-limit-enabled` is `true`. | `5` |
26+
| `-cloudwatch-concurrency.get-metric-data-limit` | Maximum number of concurrent requests to CloudWatch `GetMetricsData` API. Only applicable if `per-api-limit-enabled` is `true`. | `5` |
27+
| `-cloudwatch-concurrency.get-metric-statistics-limit` | Maximum number of concurrent requests to CloudWatch `GetMetricStatistics` API. Only applicable if `per-api-limit-enabled` is `true`. | `5` |
28+
| `-tag-concurrency` | Maximum number of concurrent requests to Resource Tagging API | `5` |
29+
| `-scraping-interval` | Seconds to wait between scraping the AWS metrics | `300` |
30+
| `-metrics-per-query` | Number of metrics made in a single GetMetricsData request | `500` |
31+
| `-labels-snake-case` | Output labels on metrics in snake case instead of camel case | `false` |
32+
| `-profiling.enabled` | Enable the /debug/pprof endpoints for profiling | `false` |
3333

3434
## YAML configuration file
3535

@@ -91,8 +91,8 @@ type: <string>
9191
roles:
9292
[ - <role_config> ... ]
9393
94-
# List of Key/Value pairs to use for tag filtering (all must match).
95-
# The key is the AWS Tag key and is case-sensitive
94+
# List of Key/Value pairs to use for tag filtering (all must match).
95+
# The key is the AWS Tag key and is case-sensitive
9696
# The value will be treated as a regex
9797
searchTags:
9898
[ - <search_tags_config> ... ]
@@ -114,7 +114,7 @@ dimensionNameRequirements:
114114
# This is useful for reducing the number of metrics returned by CloudWatch, which can be very large for some services. See AWS Cloudwatch API docs for [ListMetrics](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_ListMetrics.html) for more details.
115115
[ recentlyActiveOnly: <boolean> ]
116116

117-
# Can be used to include contextual information (account_id, region, and customTags) on info metrics and cloudwatch metrics. This can be particularly
117+
# Can be used to include contextual information (account_id, region, and customTags) on info metrics and cloudwatch metrics. This can be particularly
118118
# useful when cloudwatch metrics might not be present or when using info metrics to understand where your resources exist
119119
[ includeContextOnInfoMetrics: <boolean> ]
120120

@@ -137,6 +137,11 @@ statistics:
137137
# Export the metric with the original CloudWatch timestamp (General Setting for all metrics in this job)
138138
[ addCloudwatchTimestamp: <boolean> ]
139139

140+
# Include any metrics in the past if they are present in the CloudWatch metric response. This is useful, for example, if a metric is setup with
141+
# period 60s and length 300s so all the 5 data points are exposed in the metrics endpoint and not just the last one
142+
# (General Setting for all metrics in this job)
143+
[ addHistoricalMetrics: <boolean> ]
144+
140145
# List of metric definitions
141146
metrics:
142147
[ - <metric_config> ... ]
@@ -152,18 +157,18 @@ discovery:
152157
kafka:
153158
- Name
154159
jobs:
155-
- type: kafka
156-
regions:
157-
- eu-west-1
158-
searchTags:
159-
- key: env
160-
value: dev
161-
metrics:
162-
- name: BytesOutPerSec
163-
statistics:
164-
- Average
165-
period: 600
166-
length: 600
160+
- type: kafka
161+
regions:
162+
- eu-west-1
163+
searchTags:
164+
- key: env
165+
value: dev
166+
metrics:
167+
- name: BytesOutPerSec
168+
statistics:
169+
- Average
170+
period: 600
171+
length: 600
167172
```
168173
169174
### `static_job_config`
@@ -178,23 +183,19 @@ name: <string>
178183
namespace: <string>
179184
180185
# List of AWS regions
181-
regions:
182-
[ - <string> ...]
186+
regions: [- <string> ...]
183187
184188
# List of IAM roles to assume (optional)
185-
roles:
186-
[ - <role_config> ... ]
189+
roles: [- <role_config> ...]
187190
188191
# Custom tags to be added as a list of Key/Value pairs
189-
customTags:
190-
[ - <custom_tags_config> ... ]
192+
customTags: [- <custom_tags_config> ...]
191193
192194
# CloudWatch metric dimensions as a list of Name/Value pairs
193-
dimensions: [ <dimensions_config> ]
195+
dimensions: [<dimensions_config>]
194196
195197
# List of metric definitions
196-
metrics:
197-
[ - <metric_config> ... ]
198+
metrics: [- <metric_config> ...]
198199
```
199200

200201
Example config file:
@@ -208,15 +209,15 @@ static:
208209
regions:
209210
- eu-west-1
210211
dimensions:
211-
- name: AutoScalingGroupName
212-
value: MyGroup
212+
- name: AutoScalingGroupName
213+
value: MyGroup
213214
customTags:
214215
- key: CustomTag
215216
value: CustomValue
216217
metrics:
217218
- name: GroupInServiceInstances
218219
statistics:
219-
- Minimum
220+
- Minimum
220221
period: 60
221222
length: 300
222223
```
@@ -276,6 +277,11 @@ statistics:
276277
# Export the metric with the original CloudWatch timestamp (General Setting for all metrics in this job)
277278
[ addCloudwatchTimestamp: <boolean> ]
278279

280+
# Include any metrics in the past if they are present in the CloudWatch metric response. This is useful, for example, if a metric is setup with
281+
# period 60s and length 300s so all the 5 data points are exposed in the metrics endpoint and not just the last one
282+
# (General Setting for all metrics in this job)
283+
[ addHistoricalMetrics: <boolean> ]
284+
279285
# List of metric definitions
280286
metrics:
281287
[ - <metric_config> ... ]
@@ -336,9 +342,10 @@ statistics:
336342
```
337343
338344
Notes:
345+
339346
- Available statistics: `Maximum`, `Minimum`, `Sum`, `SampleCount`, `Average`, `pXX` (e.g. `p90`).
340347

341-
- Watch out using `addCloudwatchTimestamp` for sparse metrics, e.g from S3, since Prometheus won't scrape metrics containing timestamps older than 2-3 hours.
348+
- Watch out using `addCloudwatchTimestamp` for sparse metrics, e.g from S3, since Prometheus won't scrape metrics containing timestamps older than 2-3 hours. Also the same applies when enabling `addHistoricalMetrics` in any metric
342349

343350
### `exported_tags_config`
344351

‎pkg/clients/cloudwatch/client.go‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,26 @@ type ConcurrencyLimiter interface {
5555
}
5656

5757
type MetricDataResult struct {
58-
ID string
59-
// A nil datapoint is a marker for no datapoint being found
58+
ID string
59+
Datapoints []*DatapointWithTimestamp
60+
}
61+
62+
type DatapointWithTimestamp struct {
6063
Datapoint *float64
6164
Timestamp time.Time
6265
}
6366

67+
func NewDataPoint(datapoint *float64, timestamp time.Time) *DatapointWithTimestamp {
68+
return &DatapointWithTimestamp{
69+
Timestamp: timestamp,
70+
Datapoint: datapoint,
71+
}
72+
}
73+
74+
func SingleDataPoint(datapoint *float64, timestamp time.Time) []*DatapointWithTimestamp {
75+
return []*DatapointWithTimestamp{NewDataPoint(datapoint, timestamp)}
76+
}
77+
6478
type limitedConcurrencyClient struct {
6579
client Client
6680
limiter ConcurrencyLimiter

‎pkg/clients/cloudwatch/v1/client.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -143,10 +143,14 @@ func (c client) GetMetricData(ctx context.Context, getMetricData []*model.Cloudw
143143
func toMetricDataResult(resp cloudwatch.GetMetricDataOutput) []cloudwatch_client.MetricDataResult {
144144
output := make([]cloudwatch_client.MetricDataResult, 0, len(resp.MetricDataResults))
145145
for _, metricDataResult := range resp.MetricDataResults {
146-
mappedResult := cloudwatch_client.MetricDataResult{ID: *metricDataResult.Id}
147-
if len(metricDataResult.Values) > 0 {
148-
mappedResult.Datapoint = metricDataResult.Values[0]
149-
mappedResult.Timestamp = *metricDataResult.Timestamps[0]
146+
mappedResult := cloudwatch_client.MetricDataResult{
147+
ID: *metricDataResult.Id,
148+
Datapoints: make([]*cloudwatch_client.DatapointWithTimestamp, 0, len(metricDataResult.Timestamps))}
149+
for i := 0; i < len(metricDataResult.Timestamps); i++ {
150+
mappedResult.Datapoints = append(mappedResult.Datapoints, &cloudwatch_client.DatapointWithTimestamp{
151+
Datapoint: metricDataResult.Values[i],
152+
Timestamp: *metricDataResult.Timestamps[i],
153+
})
150154
}
151155
output = append(output, mappedResult)
152156
}

‎pkg/clients/cloudwatch/v1/client_test.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,13 @@ func Test_toMetricDataResult(t *testing.T) {
6767
},
6868
},
6969
expectedMetricDataResults: []cloudwatch_client.MetricDataResult{
70-
{ID: "metric-1", Datapoint: aws.Float64(1.0), Timestamp: ts.Add(10 * time.Minute)},
71-
{ID: "metric-2", Datapoint: aws.Float64(2.0), Timestamp: ts},
70+
{ID: "metric-1", Datapoints: []*cloudwatch_client.DatapointWithTimestamp{
71+
cloudwatch_client.NewDataPoint(aws.Float64(1.0), ts.Add(10*time.Minute)),
72+
cloudwatch_client.NewDataPoint(aws.Float64(2.0), ts.Add(5*time.Minute)),
73+
cloudwatch_client.NewDataPoint(aws.Float64(3.0), ts),
74+
},
75+
},
76+
{ID: "metric-2", Datapoints: cloudwatch_client.SingleDataPoint(aws.Float64(2.0), ts)},
7277
},
7378
},
7479
{
@@ -88,8 +93,13 @@ func Test_toMetricDataResult(t *testing.T) {
8893
},
8994
},
9095
expectedMetricDataResults: []cloudwatch_client.MetricDataResult{
91-
{ID: "metric-1", Datapoint: aws.Float64(1.0), Timestamp: ts.Add(10 * time.Minute)},
92-
{ID: "metric-2", Datapoint: nil, Timestamp: time.Time{}},
96+
{ID: "metric-1", Datapoints: []*cloudwatch_client.DatapointWithTimestamp{
97+
cloudwatch_client.NewDataPoint(aws.Float64(1.0), ts.Add(10*time.Minute)),
98+
cloudwatch_client.NewDataPoint(aws.Float64(2.0), ts.Add(5*time.Minute)),
99+
cloudwatch_client.NewDataPoint(aws.Float64(3.0), ts)},
100+
},
101+
{ID: "metric-2",
102+
Datapoints: []*cloudwatch_client.DatapointWithTimestamp{}},
93103
},
94104
},
95105
}

‎pkg/clients/cloudwatch/v2/client.go‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -149,10 +149,15 @@ func (c client) GetMetricData(ctx context.Context, getMetricData []*model.Cloudw
149149
func toMetricDataResult(resp cloudwatch.GetMetricDataOutput) []cloudwatch_client.MetricDataResult {
150150
output := make([]cloudwatch_client.MetricDataResult, 0, len(resp.MetricDataResults))
151151
for _, metricDataResult := range resp.MetricDataResults {
152-
mappedResult := cloudwatch_client.MetricDataResult{ID: *metricDataResult.Id}
153-
if len(metricDataResult.Values) > 0 {
154-
mappedResult.Datapoint = &metricDataResult.Values[0]
155-
mappedResult.Timestamp = metricDataResult.Timestamps[0]
152+
mappedResult := cloudwatch_client.MetricDataResult{
153+
ID: *metricDataResult.Id,
154+
Datapoints: make([]*cloudwatch_client.DatapointWithTimestamp, 0, len(metricDataResult.Timestamps)),
155+
}
156+
for i := 0; i < len(metricDataResult.Timestamps); i++ {
157+
mappedResult.Datapoints = append(mappedResult.Datapoints, &cloudwatch_client.DatapointWithTimestamp{
158+
Datapoint: &metricDataResult.Values[i],
159+
Timestamp: metricDataResult.Timestamps[i],
160+
})
156161
}
157162
output = append(output, mappedResult)
158163
}

‎pkg/clients/cloudwatch/v2/client_test.go‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,13 @@ func Test_toMetricDataResult(t *testing.T) {
5252
},
5353
},
5454
expectedMetricDataResults: []cloudwatch_client.MetricDataResult{
55-
{ID: "metric-1", Datapoint: aws.Float64(1.0), Timestamp: ts.Add(10 * time.Minute)},
56-
{ID: "metric-2", Datapoint: aws.Float64(2.0), Timestamp: ts},
55+
{ID: "metric-1", Datapoints: []*cloudwatch_client.DatapointWithTimestamp{
56+
cloudwatch_client.NewDataPoint(aws.Float64(1.0), ts.Add(10*time.Minute)),
57+
cloudwatch_client.NewDataPoint(aws.Float64(2.0), ts.Add(5*time.Minute)),
58+
cloudwatch_client.NewDataPoint(aws.Float64(3.0), ts),
59+
},
60+
},
61+
{ID: "metric-2", Datapoints: cloudwatch_client.SingleDataPoint(aws.Float64(2.0), ts)},
5762
},
5863
},
5964
{
@@ -73,8 +78,12 @@ func Test_toMetricDataResult(t *testing.T) {
7378
},
7479
},
7580
expectedMetricDataResults: []cloudwatch_client.MetricDataResult{
76-
{ID: "metric-1", Datapoint: aws.Float64(1.0), Timestamp: ts.Add(10 * time.Minute)},
77-
{ID: "metric-2", Datapoint: nil, Timestamp: time.Time{}},
81+
{ID: "metric-1", Datapoints: []*cloudwatch_client.DatapointWithTimestamp{
82+
cloudwatch_client.NewDataPoint(aws.Float64(1.0), ts.Add(10*time.Minute)),
83+
cloudwatch_client.NewDataPoint(aws.Float64(2.0), ts.Add(5*time.Minute)),
84+
cloudwatch_client.NewDataPoint(aws.Float64(3.0), ts),
85+
}},
86+
{ID: "metric-2", Datapoints: []*cloudwatch_client.DatapointWithTimestamp{}},
7887
},
7988
},
8089
}

‎pkg/config/config.go‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ type JobLevelMetricFields struct {
5252
Delay int64 `yaml:"delay"`
5353
NilToZero *bool `yaml:"nilToZero"`
5454
AddCloudwatchTimestamp *bool `yaml:"addCloudwatchTimestamp"`
55+
AddHistoricalMetrics *bool `yaml:"addHistoricalMetrics"`
5556
}
5657

5758
type Job struct {
@@ -99,6 +100,7 @@ type Metric struct {
99100
Delay int64 `yaml:"delay"`
100101
NilToZero *bool `yaml:"nilToZero"`
101102
AddCloudwatchTimestamp *bool `yaml:"addCloudwatchTimestamp"`
103+
AddHistoricalMetrics *bool `yaml:"addHistoricalMetrics"`
102104
}
103105

104106
type Dimension struct {
@@ -154,7 +156,7 @@ func (c *ScrapeConf) Load(file string, logger *slog.Logger) (model.JobsConfig, e
154156

155157
func (c *ScrapeConf) Validate(logger *slog.Logger) (model.JobsConfig, error) {
156158
if c.Discovery.Jobs == nil && c.Static == nil && c.CustomNamespace == nil {
157-
return model.JobsConfig{}, fmt.Errorf("At least 1 Discovery job, 1 Static or one CustomNamespace must be defined")
159+
return model.JobsConfig{}, fmt.Errorf("at least 1 Discovery job, 1 Static or one CustomNamespace must be defined")
158160
}
159161

160162
if c.Discovery.Jobs != nil {
@@ -387,6 +389,15 @@ func (m *Metric) validateMetric(logger *slog.Logger, metricIdx int, parent strin
387389
}
388390
}
389391

392+
mAddHistoricalMetrics := m.AddHistoricalMetrics
393+
if mAddHistoricalMetrics == nil {
394+
if discovery != nil && discovery.AddHistoricalMetrics != nil {
395+
mAddHistoricalMetrics = discovery.AddHistoricalMetrics
396+
} else {
397+
mAddHistoricalMetrics = aws.Bool(false)
398+
}
399+
}
400+
390401
if mLength < mPeriod {
391402
return fmt.Errorf(
392403
"Metric [%s/%d] in %v: length(%d) is smaller than period(%d). This can cause that the data requested is not ready and generate data gaps",
@@ -398,6 +409,7 @@ func (m *Metric) validateMetric(logger *slog.Logger, metricIdx int, parent strin
398409
m.Delay = mDelay
399410
m.NilToZero = mNilToZero
400411
m.AddCloudwatchTimestamp = mAddCloudwatchTimestamp
412+
m.AddHistoricalMetrics = mAddHistoricalMetrics
401413
m.Statistics = mStatistics
402414

403415
return nil
@@ -519,6 +531,7 @@ func toModelMetricConfig(metrics []*Metric) []*model.MetricConfig {
519531
Delay: m.Delay,
520532
NilToZero: aws.BoolValue(m.NilToZero),
521533
AddCloudwatchTimestamp: aws.BoolValue(m.AddCloudwatchTimestamp),
534+
AddHistoricalMetrics: aws.BoolValue(m.AddHistoricalMetrics),
522535
})
523536
}
524537
return ret

‎pkg/job/custom.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ func getMetricDataForQueriesForCustomNamespace(
8686
MetricMigrationParams: model.MetricMigrationParams{
8787
NilToZero: metric.NilToZero,
8888
AddCloudwatchTimestamp: metric.AddCloudwatchTimestamp,
89+
AddHistoricalMetrics: metric.AddHistoricalMetrics,
8990
},
9091
Tags: nil,
9192
GetMetricDataResult: nil,

‎pkg/job/discovery.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,7 @@ func getFilteredMetricDatas(
179179
MetricMigrationParams: model.MetricMigrationParams{
180180
NilToZero: m.NilToZero,
181181
AddCloudwatchTimestamp: m.AddCloudwatchTimestamp,
182+
AddHistoricalMetrics: m.AddHistoricalMetrics,
182183
},
183184
Tags: metricTags,
184185
GetMetricDataResult: nil,

‎pkg/job/getmetricdata/processor.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -131,10 +131,15 @@ func mapResultsToBatch(logger *slog.Logger, results []cloudwatch.MetricDataResul
131131
}
132132
if batch[id].GetMetricDataResult == nil {
133133
cloudwatchData := batch[id]
134+
135+
mappedDataPoints := make([]*model.DatapointWithTimestamp, 0, len(entry.Datapoints))
136+
for i := 0; i < len(entry.Datapoints); i++ {
137+
mappedDataPoints = append(mappedDataPoints, model.NewDataPoint(entry.Datapoints[i].Datapoint, entry.Datapoints[i].Timestamp))
138+
}
139+
134140
cloudwatchData.GetMetricDataResult = &model.GetMetricDataResult{
135-
Statistic: cloudwatchData.GetMetricDataProcessingParams.Statistic,
136-
Datapoint: entry.Datapoint,
137-
Timestamp: entry.Timestamp,
141+
Statistic: cloudwatchData.GetMetricDataProcessingParams.Statistic,
142+
Datapoints: mappedDataPoints,
138143
}
139144

140145
// All GetMetricData processing is done clear the params

0 commit comments

Comments
 (0)