fs2: add iocost statistics

When the iocost controller is enabled it will emit statistics about its
usage in io.stat. Export this when available. This is a no-op when
iocost is not enabled.

There is a comment that we only expose data which directly maps to
cgroups v1. We're already breaking this by adding PSI metrics, so reword
this to just indicate that we currently don't recognize the other
fields.

Signed-off-by: Morten Hein Tiljeset <[email protected]>

Author: tiljeset

fs2/io.go

@@ -165,11 +165,22 @@ func statIo(dirPath string, stats *cgroups.Stats) error {
 			case "wios":
 				op = "Write"
 				targetTable = &parsedStats.IoServicedRecursive
+
+			case "cost.usage":
+				op = "Count"
+				targetTable = &parsedStats.IoCostUsage
+			case "cost.wait":
+				op = "Count"
+				targetTable = &parsedStats.IoCostWait
+			case "cost.indebt":
+				op = "Count"
+				targetTable = &parsedStats.IoCostIndebt
+			case "cost.indelay":
+				op = "Count"
+				targetTable = &parsedStats.IoCostIndelay
+
 			default:
-				// Skip over entries we cannot map to cgroupv1 stats for now.
-				// In the future we should expand the stats struct to include
-				// them.
-				logrus.Debugf("cgroupv2 io stats: skipping over unmappable %s entry", item)
+				logrus.Debugf("cgroupv2 io stats: unknown entry %s", item)
 				continue
 			}
 

fs2/io_test.go

@@ -14,6 +14,10 @@ const exampleIoStatData = `254:1 rbytes=6901432320 wbytes=14245535744 rios=26327
 254:0 rbytes=2702336 wbytes=0 rios=97 wios=0 dbytes=0 dios=0
 259:0 rbytes=6911345664 wbytes=14245536256 rios=264538 wios=244914 dbytes=530485248 dios=2`
 
+const exampleIoCostDebugData = `251:0 rbytes=2285568 wbytes=688128 rios=107 wios=168 dbytes=0 dios=0
+252:0 rbytes=2037743988736 wbytes=1036567117824 rios=169193849 wios=41541021 dbytes=1012840136704 dios=199909
+259:0 rbytes=4085926524416 wbytes=1036680064512 rios=185034771 wios=40358485 dbytes=1013982564352 dios=199959 cost.vrate=100.00 cost.usage=1532009788 cost.wait=1477289869 cost.indebt=0 cost.indelay=0`
+
 var exampleIoStatsParsed = cgroups.BlkioStats{
 	IoServiceBytesRecursive: []cgroups.BlkioStatEntry{
 		{Major: 254, Minor: 1, Value: 6901432320, Op: "Read"},
@@ -33,6 +37,37 @@ var exampleIoStatsParsed = cgroups.BlkioStats{
 	},
 }
 
+var exampleIoCostDebugParsed = cgroups.BlkioStats{
+	IoServiceBytesRecursive: []cgroups.BlkioStatEntry{
+		{Major: 251, Minor: 0, Value: 2285568, Op: "Read"},
+		{Major: 251, Minor: 0, Value: 688128, Op: "Write"},
+		{Major: 252, Minor: 0, Value: 2037743988736, Op: "Read"},
+		{Major: 252, Minor: 0, Value: 1036567117824, Op: "Write"},
+		{Major: 259, Minor: 0, Value: 4085926524416, Op: "Read"},
+		{Major: 259, Minor: 0, Value: 1036680064512, Op: "Write"},
+	},
+	IoServicedRecursive: []cgroups.BlkioStatEntry{
+		{Major: 251, Minor: 0, Value: 107, Op: "Read"},
+		{Major: 251, Minor: 0, Value: 168, Op: "Write"},
+		{Major: 252, Minor: 0, Value: 169193849, Op: "Read"},
+		{Major: 252, Minor: 0, Value: 41541021, Op: "Write"},
+		{Major: 259, Minor: 0, Value: 185034771, Op: "Read"},
+		{Major: 259, Minor: 0, Value: 40358485, Op: "Write"},
+	},
+	IoCostUsage: []cgroups.BlkioStatEntry{
+		{Major: 259, Minor: 0, Value: 1532009788, Op: "Count"},
+	},
+	IoCostWait: []cgroups.BlkioStatEntry{
+		{Major: 259, Minor: 0, Value: 1477289869, Op: "Count"},
+	},
+	IoCostIndebt: []cgroups.BlkioStatEntry{
+		{Major: 259, Minor: 0, Value: 0, Op: "Count"},
+	},
+	IoCostIndelay: []cgroups.BlkioStatEntry{
+		{Major: 259, Minor: 0, Value: 0, Op: "Count"},
+	},
+}
+
 func lessBlkioStatEntry(a, b cgroups.BlkioStatEntry) bool {
 	if a.Major != b.Major {
 		return a.Major < b.Major
@@ -56,26 +91,49 @@ func sortBlkioStats(stats *cgroups.BlkioStats) {
 }
 
 func TestStatIo(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected cgroups.BlkioStats
+	}{
+		{
+			name:     "default io.stat case",
+			input:    exampleIoStatData,
+			expected: exampleIoStatsParsed,
+		},
+		{
+			name:     "io.stat with iocost debug data",
+			input:    exampleIoCostDebugData,
+			expected: exampleIoCostDebugParsed,
+		},
+	}
+
 	// We're using a fake cgroupfs.
 	cgroups.TestMode = true
 
-	fakeCgroupDir := t.TempDir()
-	statPath := filepath.Join(fakeCgroupDir, "io.stat")
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
 
-	if err := os.WriteFile(statPath, []byte(exampleIoStatData), 0o644); err != nil {
-		t.Fatal(err)
-	}
+			fakeCgroupDir := t.TempDir()
+			statPath := filepath.Join(fakeCgroupDir, "io.stat")
 
-	var gotStats cgroups.Stats
-	if err := statIo(fakeCgroupDir, &gotStats); err != nil {
-		t.Error(err)
-	}
+			if err := os.WriteFile(statPath, []byte(tt.input), 0o644); err != nil {
+				t.Fatal(err)
+			}
+
+			var gotStats cgroups.Stats
+			if err := statIo(fakeCgroupDir, &gotStats); err != nil {
+				t.Error(err)
+			}
 
-	// Sort the output since statIo uses a map internally.
-	sortBlkioStats(&gotStats.BlkioStats)
-	sortBlkioStats(&exampleIoStatsParsed)
+			// Sort the output since statIo uses a map internally.
+			sortBlkioStats(&gotStats.BlkioStats)
+			sortBlkioStats(&tt.expected)
 
-	if !reflect.DeepEqual(gotStats.BlkioStats, exampleIoStatsParsed) {
-		t.Errorf("parsed cgroupv2 io.stat doesn't match expected result: \ngot %#v\nexpected %#v\n", gotStats.BlkioStats, exampleIoStatsParsed)
+			if !reflect.DeepEqual(gotStats.BlkioStats, tt.expected) {
+				t.Errorf("parsed cgroupv2 io.stat doesn't match expected result: \ngot %#v\nexpected %#v\n", gotStats.BlkioStats, tt.expected)
+			}
+		})
 	}
 }

stats.go

@@ -159,6 +159,10 @@ type BlkioStats struct {
 	IoTimeRecursive         []BlkioStatEntry `json:"io_time_recursive,omitempty"`
 	SectorsRecursive        []BlkioStatEntry `json:"sectors_recursive,omitempty"`
 	PSI                     *PSIStats        `json:"psi,omitempty"`
+	IoCostUsage             []BlkioStatEntry `json:"io_cost_usage,omitempty"`
+	IoCostWait              []BlkioStatEntry `json:"io_cost_wait,omitempty"`
+	IoCostIndebt            []BlkioStatEntry `json:"io_cost_indebt,omitempty"`
+	IoCostIndelay           []BlkioStatEntry `json:"io_cost_indelay,omitempty"`
 }
 
 type HugetlbStats struct {