Introduce disk/percent_used metric, fixes #822

The Stackdriver exports the `/guest/disk/percent_used` metric to the
`custom.googleapis.com` namespace as the reserved one
`compute.googleapis.com` cannot be used at this stage.

This was tested within GCP Container-Optimized OS with the following:
```
/mnt/disks/scratch/node-problem-detector --enable-k8s-exporter=false --config.system-stats-monitor=/etc/node_problem_detector/system-stats-monitor.json --config.system-log-monitor=/etc/node_problem_detector/kernel-monitor.json --config.custom-plugin-monitor=/etc/node_problem_detector/boot-disk-size-consistency-monitor.json --exporter.stackdriver=/etc/node_problem_detector/stackdriver-exporter.json
```
The `/mnt/disks/scratch/` directory was mounted specifically to get execution permissions:
```
sudo mount -t tmpfs tmpfs /mnt/disks/scratch/
```

Author: AndreMiras

config/system-stats-monitor.json

@@ -44,6 +44,9 @@
       "disk/bytes_used": {
         "displayName": "disk/bytes_used"
       },
+      "disk/percent_used": {
+        "displayName": "disk/percent_used"
+      },
       "disk/io_time": {
         "displayName": "disk/io_time"
       },

pkg/exporters/stackdriver/stackdriver_exporter.go

@@ -54,6 +54,7 @@ var NPDMetricToSDMetric = map[metrics.MetricID]string{
 	metrics.CPULoad15m:              "compute.googleapis.com/guest/cpu/load_15m",
 	metrics.DiskAvgQueueLenID:       "compute.googleapis.com/guest/disk/queue_length",
 	metrics.DiskBytesUsedID:         "compute.googleapis.com/guest/disk/bytes_used",
+	metrics.DiskPercentUsedID:       "custom.googleapis.com/guest/disk/percent_used",
 	metrics.DiskIOTimeID:            "compute.googleapis.com/guest/disk/io_time",
 	metrics.DiskMergedOpsCountID:    "compute.googleapis.com/guest/disk/merged_operation_count",
 	metrics.DiskOpsBytesID:          "compute.googleapis.com/guest/disk/operation_bytes_count",

pkg/systemstatsmonitor/README.md

@@ -48,6 +48,7 @@ Below metrics are collected from `disk` component:
 * `disk_operation_bytes_count`: # of Bytes used for reads/writes on this device
 * `disk_operation_time`: [# of milliseconds spent reading/writing][iostat doc]
 * `disk_bytes_used`: Disk usage in Bytes. The usage state is reported under the `state` metric label (e.g. `used`, `free`). Summing values of all states yields the disk size.
+* `disk_percent_used`: Disk utilization percentage. The usage state is reported under the `state` metric label (e.g. `used`, `free`). The utilization is between 0.0 and 100.0.
 FSType and MountOptions are also reported as additional information.
 
 The name of the disk block device is reported in the `device_name` metric label (e.g. `sda`).

pkg/systemstatsmonitor/disk_collector.go

@@ -38,6 +38,7 @@ type diskCollector struct {
 	mOpsBytes       *metrics.Int64Metric
 	mOpsTime        *metrics.Int64Metric
 	mBytesUsed      *metrics.Int64Metric
+	mPercentUsed    *metrics.Float64Metric
 
 	config *ssmtypes.DiskStatsConfig
 
@@ -149,6 +150,16 @@ func NewDiskCollectorOrDie(diskConfig *ssmtypes.DiskStatsConfig) *diskCollector
 	if err != nil {
 		klog.Fatalf("Error initializing metric for %q: %v", metrics.DiskBytesUsedID, err)
 	}
+	dc.mPercentUsed, err = metrics.NewFloat64Metric(
+		metrics.DiskPercentUsedID,
+		diskConfig.MetricsConfigs[string(metrics.DiskPercentUsedID)].DisplayName,
+		"Disk utilization percentage",
+		"%",
+		metrics.LastValue,
+		[]string{deviceNameLabel, fsTypeLabel, mountOptionLabel, stateLabel})
+	if err != nil {
+		klog.Fatalf("Error initializing metric for %q: %v", metrics.DiskPercentUsedID, err)
+	}
 
 	dc.lastIOTime = make(map[string]uint64)
 	dc.lastWeightedIO = make(map[string]uint64)
@@ -291,6 +302,8 @@ func (dc *diskCollector) collect() {
 		opttypes := strings.Join(partition.Opts, ",")
 		dc.mBytesUsed.Record(map[string]string{deviceNameLabel: deviceName, fsTypeLabel: fstype, mountOptionLabel: opttypes, stateLabel: "free"}, int64(usageStat.Free))
 		dc.mBytesUsed.Record(map[string]string{deviceNameLabel: deviceName, fsTypeLabel: fstype, mountOptionLabel: opttypes, stateLabel: "used"}, int64(usageStat.Used))
+		dc.mPercentUsed.Record(map[string]string{deviceNameLabel: deviceName, fsTypeLabel: fstype, mountOptionLabel: opttypes, stateLabel: "free"}, float64(100 - usageStat.UsedPercent))
+		dc.mPercentUsed.Record(map[string]string{deviceNameLabel: deviceName, fsTypeLabel: fstype, mountOptionLabel: opttypes, stateLabel: "used"}, float64(usageStat.UsedPercent))
 	}
 
 }

pkg/util/metrics/metric.go

@@ -35,6 +35,7 @@ const (
 	DiskOpsBytesID          MetricID = "disk/operation_bytes_count"
 	DiskOpsTimeID           MetricID = "disk/operation_time"
 	DiskBytesUsedID         MetricID = "disk/bytes_used"
+	DiskPercentUsedID       MetricID = "disk/percent_used"
 	HostUptimeID            MetricID = "host/uptime"
 	MemoryBytesUsedID       MetricID = "memory/bytes_used"
 	MemoryAnonymousUsedID   MetricID = "memory/anonymous_used"

test/e2e/metriconly/metrics_test.go

@@ -85,6 +85,7 @@ var _ = ginkgo.Describe("NPD should export Prometheus metrics.", func() {
 			assertMetricExist(gotMetrics, "disk_operation_bytes_count", map[string]string{}, false)
 			assertMetricExist(gotMetrics, "disk_operation_time", map[string]string{}, false)
 			assertMetricExist(gotMetrics, "disk_bytes_used", map[string]string{}, false)
+			assertMetricExist(gotMetrics, "disk_percent_used", map[string]string{}, false)
 			assertMetricExist(gotMetrics, "disk_io_time", map[string]string{}, false)
 			assertMetricExist(gotMetrics, "disk_weighted_io", map[string]string{}, false)
 			assertMetricExist(gotMetrics, "memory_bytes_used", map[string]string{}, false)