Merge pull request #2796 from kwisniewski98/cgroup_memory_migrate

Add cgroup_memory_migrate metric

Author: bobbypage

cmd/cadvisor.go

@@ -91,6 +91,7 @@ var (
 		container.ReferencedMemoryMetrics:        struct{}{},
 		container.CPUTopologyMetrics:             struct{}{},
 		container.ResctrlMetrics:                 struct{}{},
+		container.CPUSetMetrics:                  struct{}{},
 	}}
 
 	// List of metrics that can be ignored.
@@ -110,6 +111,7 @@ var (
 		container.ReferencedMemoryMetrics:        struct{}{},
 		container.CPUTopologyMetrics:             struct{}{},
 		container.ResctrlMetrics:                 struct{}{},
+		container.CPUSetMetrics:                  struct{}{},
 	}
 )
 
@@ -141,7 +143,7 @@ func (ml *metricSetValue) Set(value string) error {
 }
 
 func init() {
-	flag.Var(&ignoreMetrics, "disable_metrics", "comma-separated list of `metrics` to be disabled. Options are 'accelerator', 'cpu_topology','disk', 'diskIO', 'memory_numa', 'network', 'tcp', 'udp', 'percpu', 'sched', 'process', 'hugetlb', 'referenced_memory', 'resctrl'.")
+	flag.Var(&ignoreMetrics, "disable_metrics", "comma-separated list of `metrics` to be disabled. Options are 'accelerator', 'cpu_topology','disk', 'diskIO', 'memory_numa', 'network', 'tcp', 'udp', 'percpu', 'sched', 'process', 'hugetlb', 'referenced_memory', 'resctrl', 'cpuset'.")
 
 	// Default logging verbosity to V(2)
 	flag.Set("v", "2")

cmd/cadvisor_test.go

@@ -108,6 +108,7 @@ func TestToIncludedMetrics(t *testing.T) {
 			container.ReferencedMemoryMetrics:        struct{}{},
 			container.CPUTopologyMetrics:             struct{}{},
 			container.ResctrlMetrics:                 struct{}{},
+			container.CPUSetMetrics:                  struct{}{},
 		},
 		container.AllMetrics,
 		{},

container/factory.go

@@ -63,6 +63,7 @@ const (
 	ReferencedMemoryMetrics        MetricKind = "referenced_memory"
 	CPUTopologyMetrics             MetricKind = "cpu_topology"
 	ResctrlMetrics                 MetricKind = "resctrl"
+	CPUSetMetrics                  MetricKind = "cpuset"
 )
 
 // AllMetrics represents all kinds of metrics that cAdvisor supported.
@@ -87,6 +88,7 @@ var AllMetrics = MetricSet{
 	ReferencedMemoryMetrics:        struct{}{},
 	CPUTopologyMetrics:             struct{}{},
 	ResctrlMetrics:                 struct{}{},
+	CPUSetMetrics:                  struct{}{},
 }
 
 func (mk MetricKind) String() string {

container/libcontainer/handler.go

@@ -835,6 +835,10 @@ func setMemoryStats(s *cgroups.Stats, ret *info.ContainerStats) {
 	ret.Memory.WorkingSet = workingSet
 }
 
+func setCPUSetStats(s *cgroups.Stats, ret *info.ContainerStats) {
+	ret.CpuSet.MemoryMigrate = s.CPUSetStats.MemoryMigrate
+}
+
 func getNumaStats(memoryStats map[uint8]uint64) map[uint8]uint64 {
 	stats := make(map[uint8]uint64, len(memoryStats))
 	for node, usage := range memoryStats {
@@ -912,6 +916,9 @@ func newContainerStats(libcontainerStats *libcontainer.Stats, includedMetrics co
 		if includedMetrics.Has(container.HugetlbUsageMetrics) {
 			setHugepageStats(s, ret)
 		}
+		if includedMetrics.Has(container.CPUSetMetrics) {
+			setCPUSetStats(s, ret)
+		}
 	}
 	if len(libcontainerStats.Interfaces) > 0 {
 		setNetworkStats(libcontainerStats, ret)

docs/runtime_options.md

@@ -125,7 +125,7 @@ cAdvisor stores the latest historical data in memory. How long of a history it s
 --application_metrics_count_limit=100: Max number of application metrics to store (per container) (default 100)
 --collector_cert="": Collector's certificate, exposed to endpoints for certificate based authentication.
 --collector_key="": Key for the collector's certificate
---disable_metrics=tcp,advtcp,udp,sched,process,hugetlb: comma-separated list of metrics to be disabled. Options are 'disk', 'network', 'tcp', 'advtcp', 'udp', 'sched', 'process', 'hugetlb'. Note: tcp and udp are disabled by default due to high CPU usage. (default tcp,advtcp,udp,sched,process,hugetlb)
+--disable_metrics=tcp,advtcp,udp,sched,process,hugetlb: comma-separated list of metrics to be disabled. Options are 'disk', 'network', 'tcp', 'advtcp', 'udp', 'sched', 'process', 'hugetlb', 'cpuset'. Note: tcp and udp are disabled by default due to high CPU usage. (default tcp,advtcp,udp,sched,process,hugetlb,cpuset)
 --prometheus_endpoint="/metrics": Endpoint to expose Prometheus metrics on (default "/metrics")
 --disable_root_cgroup_stats=false: Disable collecting root Cgroup stats
 ```

docs/storage/prometheus.md

@@ -65,6 +65,7 @@ Metric name | Type | Description | Unit (where applicable) | -disable_metrics pa
 `container_memory_rss` | Gauge | Size of RSS | bytes | |
 `container_memory_swap` | Gauge | Container swap usage | bytes | |
 `container_memory_mapped_file` | Gauge | Size of memory mapped files | bytes | |
+`container_memory_migrate` | Gauge | Memory migrate status | | cpuset |
 `container_memory_usage_bytes` | Gauge | Current memory usage, including all memory regardless of when it was accessed | bytes | |
 `container_memory_working_set_bytes` | Gauge | Current working set | bytes | |
 `container_network_receive_bytes_total` | Counter | Cumulative count of bytes received | bytes | network |

info/v1/container.go

@@ -399,6 +399,10 @@ type MemoryStats struct {
 	HierarchicalData MemoryStatsMemoryData `json:"hierarchical_data,omitempty"`
 }
 
+type CPUSetStats struct {
+	MemoryMigrate uint64 `json:"memory_migrate"`
+}
+
 type MemoryNumaStats struct {
 	File        map[uint8]uint64 `json:"file,omitempty"`
 	Anon        map[uint8]uint64 `json:"anon,omitempty"`
@@ -957,6 +961,8 @@ type ContainerStats struct {
 
 	// Resource Control (resctrl) statistics
 	Resctrl ResctrlStats `json:"resctrl,omitempty"`
+
+	CpuSet CPUSetStats `json:"cpuset,omitempty"`
 }
 
 func timeEq(t1, t2 time.Time, tolerance time.Duration) bool {

metrics/prometheus.go

@@ -455,6 +455,16 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri
 			},
 		}...)
 	}
+	if includedMetrics.Has(container.CPUSetMetrics) {
+		c.containerMetrics = append(c.containerMetrics, containerMetric{
+			name:      "container_memory_migrate",
+			help:      "Memory migrate status.",
+			valueType: prometheus.GaugeValue,
+			getValues: func(s *info.ContainerStats) metricValues {
+				return metricValues{{value: float64(s.CpuSet.MemoryMigrate), timestamp: s.Timestamp}}
+			},
+		})
+	}
 	if includedMetrics.Has(container.MemoryNumaMetrics) {
 		c.containerMetrics = append(c.containerMetrics, []containerMetric{
 			{

metrics/prometheus_fake.go

@@ -723,6 +723,7 @@ func (p testSubcontainersInfoProvider) GetRequestedContainersInfo(string, v2.Req
 							},
 						},
 					},
+					CpuSet: info.CPUSetStats{MemoryMigrate: 1},
 				},
 			},
 		},

metrics/testdata/prometheus_metrics

@@ -166,6 +166,9 @@ container_memory_mapped_file{container_env_foo_env="prod",container_label_foo_la
 # HELP container_memory_max_usage_bytes Maximum memory usage recorded in bytes
 # TYPE container_memory_max_usage_bytes gauge
 container_memory_max_usage_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 8 1395066363000
+# HELP container_memory_migrate Memory migrate status.
+# TYPE container_memory_migrate gauge
+container_memory_migrate{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1 1395066363000
 # HELP container_memory_numa_pages Number of used pages per NUMA node
 # TYPE container_memory_numa_pages gauge
 container_memory_numa_pages{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",node="0",scope="container",type="anon",zone_name="hello"} 10000 1395066363000