diff --git a/pkg/config/config.go b/pkg/config/config.go index ddccd6264..19b3131f9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -6,6 +6,7 @@ import ( "os" "github.com/aws/aws-sdk-go/aws" + "github.com/grafana/regexp" "gopkg.in/yaml.v2" "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/logging" @@ -42,15 +43,16 @@ type JobLevelMetricFields struct { } type Job struct { - Regions []string `yaml:"regions"` - Type string `yaml:"type"` - Roles []Role `yaml:"roles"` - SearchTags []Tag `yaml:"searchTags"` - CustomTags []Tag `yaml:"customTags"` - DimensionNameRequirements []string `yaml:"dimensionNameRequirements"` - Metrics []*Metric `yaml:"metrics"` - RoundingPeriod *int64 `yaml:"roundingPeriod"` - RecentlyActiveOnly bool `yaml:"recentlyActiveOnly"` + Regions []string `yaml:"regions"` + Type string `yaml:"type"` + Roles []Role `yaml:"roles"` + SearchTags []Tag `yaml:"searchTags"` + CustomTags []Tag `yaml:"customTags"` + DimensionNameRequirements []string `yaml:"dimensionNameRequirements"` + DimensionValueFilter []*Dimension `yaml:"dimensionValueFilter"` + Metrics []*Metric `yaml:"metrics"` + RoundingPeriod *int64 `yaml:"roundingPeriod"` + RecentlyActiveOnly bool `yaml:"recentlyActiveOnly"` JobLevelMetricFields `yaml:",inline"` } @@ -65,15 +67,16 @@ type Static struct { } type CustomNamespace struct { - Regions []string `yaml:"regions"` - Name string `yaml:"name"` - Namespace string `yaml:"namespace"` - RecentlyActiveOnly bool `yaml:"recentlyActiveOnly"` - Roles []Role `yaml:"roles"` - Metrics []*Metric `yaml:"metrics"` - CustomTags []Tag `yaml:"customTags"` - DimensionNameRequirements []string `yaml:"dimensionNameRequirements"` - RoundingPeriod *int64 `yaml:"roundingPeriod"` + Regions []string `yaml:"regions"` + Name string `yaml:"name"` + Namespace string `yaml:"namespace"` + RecentlyActiveOnly bool `yaml:"recentlyActiveOnly"` + Roles []Role `yaml:"roles"` + Metrics []*Metric `yaml:"metrics"` + CustomTags []Tag `yaml:"customTags"` + DimensionNameRequirements []string `yaml:"dimensionNameRequirements"` + DimensionValueFilter []*Dimension `yaml:"dimensionValueFilter"` + RoundingPeriod *int64 `yaml:"roundingPeriod"` JobLevelMetricFields `yaml:",inline"` } @@ -206,6 +209,12 @@ func (j *Job) validateDiscoveryJob(jobIdx int) error { return err } } + for dimensionIdx, dimension := range j.DimensionValueFilter { + err := dimension.validateDimensionValueRegexps(dimensionIdx, parent) + if err != nil { + return err + } + } return nil } @@ -239,7 +248,12 @@ func (j *CustomNamespace) validateCustomNamespaceJob(jobIdx int) error { return err } } - + for dimensionIdx, dimension := range j.DimensionValueFilter { + err := dimension.validateDimensionValueRegexps(dimensionIdx, parent) + if err != nil { + return err + } + } return nil } @@ -371,6 +385,7 @@ func (c *ScrapeConf) toModelConfig() model.JobsConfig { job.SearchTags = toModelTags(discoveryJob.SearchTags) job.CustomTags = toModelTags(discoveryJob.CustomTags) job.Metrics = toModelMetricConfig(discoveryJob.Metrics) + job.DimensionValueFilter = toModelDimensionValueFilterConfig(discoveryJob.DimensionValueFilter) job.ExportedTagsOnMetrics = []string{} if len(c.Discovery.ExportedTagsOnMetrics) > 0 { @@ -414,6 +429,7 @@ func (c *ScrapeConf) toModelConfig() model.JobsConfig { job.Roles = toModelRoles(customNamespaceJob.Roles) job.CustomTags = toModelTags(customNamespaceJob.CustomTags) job.Metrics = toModelMetricConfig(customNamespaceJob.Metrics) + job.DimensionValueFilter = toModelDimensionValueFilterConfig(customNamespaceJob.DimensionValueFilter) jobsCfg.CustomNamespaceJobs = append(jobsCfg.CustomNamespaceJobs, job) } @@ -493,3 +509,23 @@ func logConfigErrors(cfg []byte, logger logging.Logger) { logger.Warn(`Config file error(s) detected: Yace might not work as expected. Future versions of Yace might fail to run with an invalid config file.`) } } + +// converts string to regexp for model +func toModelDimensionValueFilterConfig(dimensionValueFilter []*Dimension) []*model.DimensionFilter { + ret := make([]*model.DimensionFilter, 0, len(dimensionValueFilter)) + for _, d := range dimensionValueFilter { + ret = append(ret, &model.DimensionFilter{ + Name: d.Name, + Value: regexp.MustCompile(d.Value), + }) + } + return ret +} + +func (d *Dimension) validateDimensionValueRegexps(dimensionIdx int, parent string) error { + _, err := regexp.Compile(d.Value) + if err != nil { + return fmt.Errorf("Dimension Value filter regexp %d, %q in %v: Name should not be empty", dimensionIdx, d.Value, parent) + } + return nil +} diff --git a/pkg/job/custom.go b/pkg/job/custom.go index 23bfdc177..0ec34096a 100644 --- a/pkg/job/custom.go +++ b/pkg/job/custom.go @@ -107,6 +107,9 @@ func getMetricDataForQueriesForCustomNamespace( if len(customNamespaceJob.DimensionNameRequirements) > 0 && !metricDimensionsMatchNames(cwMetric, customNamespaceJob.DimensionNameRequirements) { continue } + if len(customNamespaceJob.DimensionValueFilter) > 0 && metricDimensionsFilterValue(cwMetric, customNamespaceJob.DimensionValueFilter) { + continue + } for _, stats := range metric.Statistics { id := fmt.Sprintf("id_%d", rand.Int()) diff --git a/pkg/job/discovery.go b/pkg/job/discovery.go index 243be1719..ad6a02403 100644 --- a/pkg/job/discovery.go +++ b/pkg/job/discovery.go @@ -9,6 +9,8 @@ import ( "strings" "sync" + "github.com/grafana/regexp" + "golang.org/x/sync/errgroup" "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/clients/cloudwatch" @@ -192,7 +194,7 @@ func getMetricDataForQueries( assoc := maxdimassociator.NewAssociator(logger, svc.DimensionRegexps, resources) err := clientCloudwatch.ListMetrics(ctx, svc.Namespace, metric, discoveryJob.RecentlyActiveOnly, func(page []*model.Metric) { - data := getFilteredMetricDatas(logger, discoveryJob.Type, discoveryJob.ExportedTagsOnMetrics, page, discoveryJob.DimensionNameRequirements, metric, assoc) + data := getFilteredMetricDatas(logger, discoveryJob.Type, discoveryJob.ExportedTagsOnMetrics, page, discoveryJob.DimensionNameRequirements, discoveryJob.DimensionValueFilter, metric, assoc) mux.Lock() getMetricDatas = append(getMetricDatas, data...) @@ -215,6 +217,7 @@ func getFilteredMetricDatas( tagsOnMetrics []string, metricsList []*model.Metric, dimensionNameList []string, + dimensionValueFilterList []*model.DimensionFilter, m *model.MetricConfig, assoc resourceAssociator, ) []*model.CloudwatchData { @@ -223,6 +226,9 @@ func getFilteredMetricDatas( if len(dimensionNameList) > 0 && !metricDimensionsMatchNames(cwMetric, dimensionNameList) { continue } + if len(dimensionValueFilterList) > 0 && metricDimensionsFilterValue(cwMetric, dimensionValueFilterList) { + continue + } matchedResource, skip := assoc.AssociateMetricToResource(cwMetric) if skip { @@ -283,3 +289,25 @@ func metricDimensionsMatchNames(metric *model.Metric, dimensionNameRequirements } return true } + +// Modify this to look at values +func metricDimensionsFilterValue(metric *model.Metric, dimensionValueFilter []*model.DimensionFilter) bool { + for _, dimension := range metric.Dimensions { + for _, dimensionValueRegexp := range dimensionValueFilter { + if dimension.Name == dimensionValueRegexp.Name && regexpValue(dimension.Value, dimensionValueRegexp.Value) { + return true + } + } + } + return false +} + +// regex checker +func regexpValue(metricValue string, valueRegexp *regexp.Regexp) bool { + match := valueRegexp.FindStringSubmatch(metricValue) + if match == nil { + return false + } else { + return true + } +} diff --git a/pkg/job/discovery_test.go b/pkg/job/discovery_test.go index 052cd267e..460885f0f 100644 --- a/pkg/job/discovery_test.go +++ b/pkg/job/discovery_test.go @@ -25,6 +25,7 @@ func Test_getFilteredMetricDatas(t *testing.T) { tagsOnMetrics []string dimensionRegexps []*regexp.Regexp dimensionNameRequirements []string + dimensionValueFilter []*model.DimensionFilter resources []*model.TaggedResource metricsList []*model.Metric m *model.MetricConfig @@ -397,11 +398,172 @@ func Test_getFilteredMetricDatas(t *testing.T) { }, }, }, + { + "filter query dimension values for amq", + args{ + region: "us-east-1", + accountID: "123123123123", + namespace: "mq", + customTags: nil, + tagsOnMetrics: []string{ + "Value1", + "Value2", + }, + dimensionRegexps: config.SupportedServices.GetService("mq").DimensionRegexps, + dimensionValueFilter: []*model.DimensionFilter{ + { + Name: "Queue", + Value: regexp.MustCompile("^ActiveMQ\\.Statistics\\.Destination\\..+"), + }, + { + Name: "Queue", + Value: regexp.MustCompile("^.+\\.stats"), + }, + { + Name: "Queue", + Value: regexp.MustCompile("^Loadtesting.+"), + }, + { + Name: "Queue", + Value: regexp.MustCompile("^TEST.+"), + }, + }, + resources: []*model.TaggedResource{ + { + ARN: "arn:aws:mq:us-east-2:123456789012:broker:activemq-broker:b-000-111-222-333", + Tags: []model.Tag{ + { + Key: "Tag", + Value: "some-Tag", + }, + }, + Namespace: "mq", + Region: "us-east-1", + }, + }, + metricsList: []*model.Metric{ + { + MetricName: "QueueSize", + Dimensions: []*model.Dimension{ + { + Name: "Queue", + Value: "Fufillment", + }, + { + Name: "Broker", + Value: "activemq-broker-1", + }, + }, + Namespace: "AWS/AmazonMQ", + }, + { + MetricName: "QueueSize", + Dimensions: []*model.Dimension{ + { + Name: "Queue", + Value: "ActiveMQ.Statistics.Destination.test", + }, + { + Name: "Broker", + Value: "activemq-broker-1", + }, + }, + Namespace: "AWS/AmazonMQ", + }, + { + MetricName: "QueueSize", + Dimensions: []*model.Dimension{ + { + Name: "Queue", + Value: "test.stats", + }, + { + Name: "Broker", + Value: "activemq-broker-1", + }, + }, + Namespace: "AWS/AmazonMQ", + }, + { + MetricName: "QueueSize", + Dimensions: []*model.Dimension{ + { + Name: "Queue", + Value: "Loadtesting.wow", + }, + { + Name: "Broker", + Value: "activemq-broker-1", + }, + }, + Namespace: "AWS/AmazonMQ", + }, + { + MetricName: "QueueSize", + Dimensions: []*model.Dimension{ + { + Name: "Queue", + Value: "TEST.hmmm123", + }, + { + Name: "Broker", + Value: "activemq-broker-1", + }, + }, + Namespace: "AWS/AmazonMQ", + }, + }, + m: &model.MetricConfig{ + Name: "QueueSize", + Statistics: []string{ + "Average", + }, + Period: 60, + Length: 600, + Delay: 120, + NilToZero: aws.Bool(false), + AddCloudwatchTimestamp: aws.Bool(false), + }, + }, + []model.CloudwatchData{ + { + AddCloudwatchTimestamp: aws.Bool(false), + Dimensions: []*model.Dimension{ + { + Name: "Queue", + Value: "Fufillment", + }, + { + Name: "Broker", + Value: "activemq-broker-1", + }, + }, + ID: aws.String("arn:aws:mq:us-east-2:123456789012:broker:activemq-broker:b-000-111-222-333"), + Metric: aws.String("QueueSize"), + Namespace: aws.String("mq"), + NilToZero: aws.Bool(false), + Period: 60, + Statistics: []string{ + "Average", + }, + Tags: []model.Tag{ + { + Key: "Value1", + Value: "", + }, + { + Key: "Value2", + Value: "", + }, + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { assoc := maxdimassociator.NewAssociator(logging.NewNopLogger(), tt.args.dimensionRegexps, tt.args.resources) - metricDatas := getFilteredMetricDatas(logging.NewNopLogger(), tt.args.namespace, tt.args.tagsOnMetrics, tt.args.metricsList, tt.args.dimensionNameRequirements, tt.args.m, assoc) + metricDatas := getFilteredMetricDatas(logging.NewNopLogger(), tt.args.namespace, tt.args.tagsOnMetrics, tt.args.metricsList, tt.args.dimensionNameRequirements, tt.args.dimensionValueFilter, tt.args.m, assoc) if len(metricDatas) != len(tt.wantGetMetricsData) { t.Errorf("len(getFilteredMetricDatas()) = %v, want %v", len(metricDatas), len(tt.wantGetMetricsData)) } diff --git a/pkg/model/model.go b/pkg/model/model.go index c7415cb75..fb5d7d36e 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -26,6 +26,7 @@ type DiscoveryJob struct { SearchTags []Tag CustomTags []Tag DimensionNameRequirements []string + DimensionValueFilter []*DimensionFilter Metrics []*MetricConfig RoundingPeriod *int64 RecentlyActiveOnly bool @@ -52,6 +53,7 @@ type CustomNamespaceJob struct { Metrics []*MetricConfig CustomTags []Tag DimensionNameRequirements []string + DimensionValueFilter []*DimensionFilter RoundingPeriod *int64 JobLevelMetricFields } @@ -91,7 +93,10 @@ type Dimension struct { Name string Value string } - +type DimensionFilter struct { + Name string + Value *regexp.Regexp +} type Metric struct { // The dimensions for the metric. Dimensions []*Dimension