Merge pull request #2774 from byyML/master

ADD "disable_specific_metrics" which can remove specific metric by name or regex

Author: bobbypage

cmd/cadvisor.go

@@ -113,6 +113,8 @@ var (
 		container.ResctrlMetrics:                 struct{}{},
 		container.CPUSetMetrics:                  struct{}{},
 	}
+	// metrics to be ignored,the difference between ignoreMetrics and ignoreSpecificMetrics is the former filters a certain type of metrics and the later filters specific metrics.
+	ignoreSpecificMetrics DenyList = DenyList{}
 )
 
 type metricSetValue struct {
@@ -142,8 +144,27 @@ func (ml *metricSetValue) Set(value string) error {
 	return nil
 }
 
+type DenyList []string
+
+func (ms *DenyList) String() string {
+	return strings.Join(*ms, ",")
+}
+
+// Set converts a comma-separated string of metrics into a slice and appends it to the DenyList.
+func (ms *DenyList) Set(value string) error {
+	metrics := strings.Split(value, ",")
+	for _, metric := range metrics {
+		metric = strings.TrimSpace(metric)
+		if len(metric) != 0 {
+			*ms = append(*ms, metric)
+		}
+	}
+	return nil
+}
+
 func init() {
 	flag.Var(&ignoreMetrics, "disable_metrics", "comma-separated list of `metrics` to be disabled. Options are 'accelerator', 'cpu_topology','disk', 'diskIO', 'memory_numa', 'network', 'tcp', 'udp', 'percpu', 'sched', 'process', 'hugetlb', 'referenced_memory', 'resctrl', 'cpuset'.")
+	flag.Var(&ignoreSpecificMetrics, "disable_specific_metrics", "Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns.It differs from the 'disable_metrics' in that 'disable_specific_metrics' filters specific metrics, while 'disable_metrics' filters a certain type of metrics.Besides, 'disable_specific_metrics' will only disable metrics being exported by Prometheus ")
 
 	// Default logging verbosity to V(2)
 	flag.Set("v", "2")
@@ -160,6 +181,7 @@ func main() {
 	}
 
 	includedMetrics := toIncludedMetrics(ignoreMetrics.MetricSet)
+	denyList, err := metrics.NewDenyList(ignoreSpecificMetrics)
 
 	setMaxProcs()
 
@@ -172,7 +194,7 @@ func main() {
 
 	collectorHttpClient := createCollectorHttpClient(*collectorCert, *collectorKey)
 
-	resourceManager, err := manager.New(memoryStorage, sysFs, housekeepingConfig, includedMetrics, &collectorHttpClient, strings.Split(*rawCgroupPrefixWhiteList, ","), *perfEvents)
+	resourceManager, err := manager.New(memoryStorage, sysFs, housekeepingConfig, includedMetrics, &collectorHttpClient, strings.Split(*rawCgroupPrefixWhiteList, ","), *perfEvents, denyList)
 	if err != nil {
 		klog.Fatalf("Failed to create a manager: %s", err)
 	}
@@ -199,7 +221,7 @@ func main() {
 	}
 
 	// Register Prometheus collector to gather information about containers, Go runtime, processes, and machine
-	cadvisorhttp.RegisterPrometheusHandler(mux, resourceManager, *prometheusEndpoint, containerLabelFunc, includedMetrics)
+	cadvisorhttp.RegisterPrometheusHandler(mux, resourceManager, *prometheusEndpoint, containerLabelFunc, includedMetrics, denyList)
 
 	// Start the manager.
 	if err := resourceManager.Start(); err != nil {

cmd/internal/http/handlers.go

@@ -95,10 +95,10 @@ func RegisterHandlers(mux httpmux.Mux, containerManager manager.Manager, httpAut
 // RegisterPrometheusHandler creates a new PrometheusCollector and configures
 // the provided HTTP mux to handle the given Prometheus endpoint.
 func RegisterPrometheusHandler(mux httpmux.Mux, resourceManager manager.Manager, prometheusEndpoint string,
-	f metrics.ContainerLabelsFunc, includedMetrics container.MetricSet) {
+	f metrics.ContainerLabelsFunc, includedMetrics container.MetricSet, denyList *metrics.DenyList) {
 	goCollector := prometheus.NewGoCollector()
 	processCollector := prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})
-	machineCollector := metrics.NewPrometheusMachineCollector(resourceManager, includedMetrics)
+	machineCollector := metrics.NewPrometheusMachineCollector(resourceManager, includedMetrics, denyList)
 
 	mux.Handle(prometheusEndpoint, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		opts, err := api.GetRequestOptions(req)
@@ -111,7 +111,7 @@ func RegisterPrometheusHandler(mux httpmux.Mux, resourceManager manager.Manager,
 
 		r := prometheus.NewRegistry()
 		r.MustRegister(
-			metrics.NewPrometheusCollector(resourceManager, f, includedMetrics, clock.RealClock{}, opts),
+			metrics.NewPrometheusCollector(resourceManager, f, includedMetrics, clock.RealClock{}, opts, denyList),
 			machineCollector,
 			goCollector,
 			processCollector,

manager/manager.go

@@ -37,6 +37,7 @@ import (
 	info "github.com/google/cadvisor/info/v1"
 	"github.com/google/cadvisor/info/v2"
 	"github.com/google/cadvisor/machine"
+	"github.com/google/cadvisor/metrics"
 	"github.com/google/cadvisor/nvm"
 	"github.com/google/cadvisor/perf"
 	"github.com/google/cadvisor/resctrl"
@@ -146,7 +147,7 @@ type HouskeepingConfig = struct {
 }
 
 // New takes a memory storage and returns a new manager.
-func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, houskeepingConfig HouskeepingConfig, includedMetricsSet container.MetricSet, collectorHTTPClient *http.Client, rawContainerCgroupPathPrefixWhiteList []string, perfEventsFile string) (Manager, error) {
+func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, houskeepingConfig HouskeepingConfig, includedMetricsSet container.MetricSet, collectorHTTPClient *http.Client, rawContainerCgroupPathPrefixWhiteList []string, perfEventsFile string, denyList *metrics.DenyList) (Manager, error) {
 	if memoryCache == nil {
 		return nil, fmt.Errorf("manager requires memory storage")
 	}
@@ -198,6 +199,7 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, houskeepingConfig
 		maxHousekeepingInterval:               *houskeepingConfig.Interval,
 		allowDynamicHousekeeping:              *houskeepingConfig.AllowDynamic,
 		includedMetrics:                       includedMetricsSet,
+		DenyList:                              denyList,
 		containerWatchers:                     []watcher.ContainerWatcher{},
 		eventsChannel:                         eventsChannel,
 		collectorHTTPClient:                   collectorHTTPClient,
@@ -257,6 +259,7 @@ type manager struct {
 	maxHousekeepingInterval  time.Duration
 	allowDynamicHousekeeping bool
 	includedMetrics          container.MetricSet
+	DenyList                 *metrics.DenyList
 	containerWatchers        []watcher.ContainerWatcher
 	eventsChannel            chan watcher.ContainerEvent
 	collectorHTTPClient      *http.Client

metrics/deny_list.go

@@ -0,0 +1,53 @@
+// Copyright 2021 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import (
+	"regexp"
+)
+
+// DenyList encapsulates the logic needed to filter based on a string.
+type DenyList struct {
+	rList []*regexp.Regexp
+}
+
+// New constructs a new DenyList based on a white- and a
+// DenyList. Only one of them can be not empty.
+func NewDenyList(b []string) (*DenyList, error) {
+	l := &DenyList{}
+	var regs []*regexp.Regexp
+	for _, item := range b {
+		r, err := regexp.Compile(item)
+		if err != nil {
+			return nil, err
+		}
+		regs = append(regs, r)
+	}
+	l.rList = regs
+	return l, nil
+}
+
+// IsIncluded returns if the given item is included.
+func (l *DenyList) IsDenied(item string) bool {
+	var matched bool
+	for _, r := range l.rList {
+		matched = r.MatchString(item)
+		if matched {
+			break
+		}
+	}
+
+	return matched
+}

metrics/deny_list_test.go

@@ -0,0 +1,75 @@
+// Copyright 2021 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import (
+	"bufio"
+	"io"
+	"strings"
+
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+var (
+	ignoreSpecificMetrics = []string{"^machine_(memory|cpu).*", "^container_fs_*", "^container_cpu_*", "^container_blkio_.*"}
+)
+
+func TestNewDenyList(t *testing.T) {
+	denyList, err := NewDenyList(ignoreSpecificMetrics)
+	assert.Nil(t, err)
+
+	testDenyListIsDenied(t, denyList, "testdata/deny_metrics")
+	testDenyListAllowed(t, denyList, "testdata/allow_metrics")
+
+}
+
+func testDenyListIsDenied(t *testing.T, denyList *DenyList, metricsFile string) {
+	deniedMetrics, err := os.Open(metricsFile)
+	if err != nil {
+		t.Fatalf("unable to read input test file %s", metricsFile)
+	}
+	buf := bufio.NewReader(deniedMetrics)
+	for {
+		line, err := buf.ReadString('\n')
+		line = strings.TrimSpace(line)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+		}
+		assert.True(t, denyList.IsDenied(line))
+
+	}
+}
+
+func testDenyListAllowed(t *testing.T, denyList *DenyList, metricsFile string) {
+	allowedMetrics, err := os.Open(metricsFile)
+	if err != nil {
+		t.Fatalf("unable to read input test file %s", metricsFile)
+	}
+	buf := bufio.NewReader(allowedMetrics)
+	for {
+		line, err := buf.ReadString('\n')
+		line = strings.TrimSpace(line)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+		}
+		assert.False(t, denyList.IsDenied(line))
+	}
+}

metrics/prometheus.go

@@ -105,7 +105,7 @@ type PrometheusCollector struct {
 // ContainerLabelsFunc specifies which base labels will be attached to all
 // exported metrics. If left to nil, the DefaultContainerLabels function
 // will be used instead.
-func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetrics container.MetricSet, now clock.Clock, opts v2.RequestOptions) *PrometheusCollector {
+func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetrics container.MetricSet, now clock.Clock, opts v2.RequestOptions, denyList *DenyList) *PrometheusCollector {
 	if f == nil {
 		f = DefaultContainerLabels
 	}
@@ -1757,6 +1757,13 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri
 			},
 		}...)
 	}
+	var filtered []containerMetric
+	for _, val := range c.containerMetrics {
+		if !denyList.IsDenied(val.name) {
+			filtered = append(filtered, val)
+		}
+	}
+	c.containerMetrics = filtered
 	return c
 }
 

metrics/prometheus_machine.go

@@ -67,7 +67,7 @@ type PrometheusMachineCollector struct {
 }
 
 // NewPrometheusMachineCollector returns a new PrometheusCollector.
-func NewPrometheusMachineCollector(i infoProvider, includedMetrics container.MetricSet) *PrometheusMachineCollector {
+func NewPrometheusMachineCollector(i infoProvider, includedMetrics container.MetricSet, denyList *DenyList) *PrometheusMachineCollector {
 	c := &PrometheusMachineCollector{
 
 		infoProvider: i,
@@ -192,6 +192,13 @@ func NewPrometheusMachineCollector(i infoProvider, includedMetrics container.Met
 			},
 		}...)
 	}
+	var filtered []machineMetric
+	for _, val := range c.machineMetrics {
+		if !denyList.IsDenied(val.name) {
+			filtered = append(filtered, val)
+		}
+	}
+	c.machineMetrics = filtered
 	return c
 }
 

metrics/prometheus_machine_test.go

@@ -17,12 +17,14 @@ package metrics
 import (
 	"bytes"
 	"io/ioutil"
+	"os"
 	"reflect"
 	"testing"
 	"time"
 
 	"github.com/google/cadvisor/container"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/prometheus/common/expfmt"
 	"github.com/stretchr/testify/assert"
 )
@@ -31,7 +33,9 @@ const machineMetricsFile = "testdata/prometheus_machine_metrics"
 const machineMetricsFailureFile = "testdata/prometheus_machine_metrics_failure"
 
 func TestPrometheusMachineCollector(t *testing.T) {
-	collector := NewPrometheusMachineCollector(testSubcontainersInfoProvider{}, container.AllMetrics)
+	denyList, err := NewDenyList([]string{})
+	assert.Nil(t, err)
+	collector := NewPrometheusMachineCollector(testSubcontainersInfoProvider{}, container.AllMetrics, denyList)
 	registry := prometheus.NewRegistry()
 	registry.MustRegister(collector)
 
@@ -51,11 +55,13 @@ func TestPrometheusMachineCollector(t *testing.T) {
 }
 
 func TestPrometheusMachineCollectorWithFailure(t *testing.T) {
+	denyList, err := NewDenyList([]string{})
+	assert.Nil(t, err)
 	provider := &erroringSubcontainersInfoProvider{
 		successfulProvider: testSubcontainersInfoProvider{},
 		shouldFail:         true,
 	}
-	collector := NewPrometheusMachineCollector(provider, container.AllMetrics)
+	collector := NewPrometheusMachineCollector(provider, container.AllMetrics, denyList)
 	registry := prometheus.NewRegistry()
 	registry.MustRegister(collector)
 
@@ -73,6 +79,29 @@ func TestPrometheusMachineCollectorWithFailure(t *testing.T) {
 	assert.Equal(t, string(expectedMetrics), collectedMetrics)
 }
 
+func TestPrometheusMachineCollectorWithDenyList(t *testing.T) {
+	denyList, err := NewDenyList(ignoreSpecificMetrics)
+	assert.Nil(t, err)
+	cMachine := NewPrometheusMachineCollector(testSubcontainersInfoProvider{}, container.AllMetrics, denyList)
+	registry := prometheus.NewRegistry()
+	registry.MustRegister(cMachine)
+
+	testPrometheusMachineCollectorWithIsDenied(t, registry, "testdata/prometheus_machine_metrics_denylist")
+
+}
+
+func testPrometheusMachineCollectorWithIsDenied(t *testing.T, gatherer prometheus.Gatherer, metricsFile string) {
+	wantMetrics, err := os.Open(metricsFile)
+	if err != nil {
+		t.Fatalf("unable to read input test file %s", metricsFile)
+	}
+
+	err = testutil.GatherAndCompare(gatherer, wantMetrics)
+	if err != nil {
+		t.Fatalf("Metric comparison failed: %s", err)
+	}
+}
+
 func TestGetMemoryByType(t *testing.T) {
 	machineInfo, err := testSubcontainersInfoProvider{}.GetMachineInfo()
 	assert.Nil(t, err)