Merge #146966

146966: pkg/storage/disk: Add support for monitoring disk stats for unknown dev r=sean- a=sean-

Previously, crdb couldn't monitor stores running on unknown devices.  Now the code attempts to resolve the underlying device based on the fstype.  Any failures fallback to the current behavior.

Epic: none
Release note: None
Fixes: #146336

Co-authored-by: Sean Chittenden <[email protected]>

Author: craig[bot]

pkg/storage/disk/BUILD.bazel

@@ -49,6 +49,7 @@ go_test(
         "linux_parse_test.go",
         "monitor_test.go",
         "monitor_tracer_test.go",
+        "platform_linux_test.go",
     ],
     data = glob(["testdata/**"]),
     embed = [":disk"],

pkg/storage/disk/monitor.go

@@ -321,5 +321,5 @@ func getDeviceIDFromPath(fs vfs.FS, path string) (DeviceID, error) {
 	if err != nil {
 		return DeviceID{}, errors.Wrapf(err, "fstat(%s)", path)
 	}
-	return deviceIDFromFileInfo(finfo), nil
+	return deviceIDFromFileInfo(finfo, path), nil
 }

pkg/storage/disk/platform_darwin.go

@@ -26,7 +26,7 @@ func newStatsCollector(fs vfs.FS) (*darwinCollector, error) {
 	return &darwinCollector{}, nil
 }
 
-func deviceIDFromFileInfo(finfo fs.FileInfo) DeviceID {
+func deviceIDFromFileInfo(finfo fs.FileInfo, path string) DeviceID {
 	statInfo := finfo.Sys().(*sysutil.StatT)
 	id := DeviceID{
 		major: unix.Major(uint64(statInfo.Dev)),

pkg/storage/disk/platform_default.go

@@ -24,6 +24,6 @@ func newStatsCollector(fs vfs.FS) (*defaultCollector, error) {
 	return &defaultCollector{}, nil
 }
 
-func deviceIDFromFileInfo(fs.FileInfo) DeviceID {
+func deviceIDFromFileInfo(fs.FileInfo, string) DeviceID {
 	return DeviceID{}
 }

pkg/storage/disk/platform_linux.go

@@ -8,10 +8,20 @@
 package disk
 
 import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
 	"io"
 	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
 	"time"
 
+	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/sysutil"
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/vfs"
@@ -63,11 +73,219 @@ func newStatsCollector(fs vfs.FS) (*linuxStatsCollector, error) {
 	}, nil
 }
 
-func deviceIDFromFileInfo(finfo fs.FileInfo) DeviceID {
+func deviceIDFromFileInfo(finfo fs.FileInfo, path string) DeviceID {
+	ctx := context.TODO()
 	statInfo := finfo.Sys().(*sysutil.StatT)
+	major := unix.Major(statInfo.Dev)
+	minor := unix.Minor(statInfo.Dev)
+
+	// Per /usr/include/linux/major.h and Documentation/admin-guide/devices.rst:
+	switch major {
+	case 0: // UNNAMED_MAJOR
+
+		// Perform additional lookups for unknown device types
+		var statfs sysutil.StatfsT
+		err := sysutil.Statfs(path, &statfs)
+		if err != nil {
+			maybeWarnf(ctx, "unable to statfs(2) path %q (%d:%d): %v", path, major, minor, err)
+			return DeviceID{major, minor}
+		}
+
+		switch statfs.Type {
+		case 0x2fc12fc1: // ZFS_SUPER_MAGIC from include/sys/fs/zfs.h
+			major, minor, err = deviceIDForZFS(path)
+			if err != nil {
+				maybeWarnf(ctx, "zfs: unable to find device ID for %q: %v", path, err)
+			} else {
+				maybeInfof(ctx, "zfs: mapping %q to diskstats device %d:%d", path, major, minor)
+			}
+
+			id := DeviceID{
+				major: major,
+				minor: minor,
+			}
+			return id
+
+		case 0x58465342: // XFS_SUPER_MAGIC	from linux/magic.h "XFSB"
+			maybeWarnf(ctx, "xfs: unable to find device ID for %q: %v", path, err)
+
+		default:
+			maybeWarnf(ctx, "unsupported file system type %x for path (%d:%d) %q", statfs.Type, major, minor, path)
+		}
+
+	case 259: // BLOCK_EXT_MAJOR=259
+
+		// NOTE: Major device 259 is the happy path for ext4 and xfs filesystems: no
+		// additional handling is required.
+
+		maybeInfof(ctx, "mapping %q to diskstats device %d:%d", path, major, minor)
+
+	default:
+		maybeWarnf(ctx, "unsupported device type %d:%d for store at %q", major, minor, path)
+	}
+
 	id := DeviceID{
-		major: unix.Major(statInfo.Dev),
-		minor: unix.Minor(statInfo.Dev),
+		major: major,
+		minor: minor,
 	}
 	return id
 }
+
+type _ZPoolName string
+
+func deviceIDForZFS(path string) (uint32, uint32, error) {
+	zpoolName, err := zfsGetPoolName(path)
+	if err != nil {
+		return 0, 0, errors.Newf("unable to find the zpool for %q: %v", path, err) // nolint:errwrap
+	}
+
+	// If there are multiple devices for a zpool, an error is returned along with
+	// a device name.  Continue resolving the device's major:minor numbers,
+	// despite the multiple drives.
+	devName, err := zpoolGetDevice(zpoolName)
+	if err != nil && devName == "" {
+		return 0, 0, errors.Newf("unable to find the device for pool %q: %v", zpoolName, err) // nolint:errwrap
+	}
+
+	major, minor, err := getDeviceID(devName)
+	if err != nil {
+		return 0, 0, errors.Newf("unable to find the device numbers for device %q: %v", devName, err) // nolint:errwrap
+	}
+
+	return major, minor, nil
+}
+
+func zfsGetPoolName(path string) (_ZPoolName, error) {
+	out, err := exec.Command("df", "--no-sync", "--output=source,fstype", path).Output()
+	if err != nil {
+		return "", errors.Newf("unable to exec df(1): %v", err) // nolint:errwrap
+	}
+
+	return zfsParseDF(out)
+}
+
+func zfsParseDF(df []byte) (_ZPoolName, error) {
+	lines := strings.Split(strings.TrimSpace(string(df)), "\n")
+	if len(lines) < 2 {
+		return "", fmt.Errorf("unexpected df(1) output: %q", df)
+	}
+
+	fields := strings.Fields(lines[1])
+	if len(fields) != 2 {
+		return "", fmt.Errorf("unexpected df(1) fields (expected 2, got %d): %q", len(fields), lines[1])
+	}
+
+	if fields[1] != "zfs" {
+		return "", fmt.Errorf("unexpected df(1) fields (expected 2, got %d): %q", len(fields), lines[1])
+	}
+
+	// Need to accept inputs formed like "data1" and "data1/crdb-logs"
+	poolName := strings.Split(fields[0], "/")[0]
+
+	return _ZPoolName(poolName), nil
+}
+
+func zpoolGetDevice(poolName _ZPoolName) (string, error) {
+	ctx := context.TODO()
+
+	out, err := exec.Command("zpool", "status", "-pPL", string(poolName)).Output()
+	if err != nil {
+		return "", errors.Newf("unable to find the devices attached to pool %q: %v", poolName, err) // nolint:errwrap
+	}
+
+	return zpoolParseStatus(ctx, poolName, out)
+}
+
+func zpoolParseStatus(ctx context.Context, poolName _ZPoolName, output []byte) (string, error) {
+	scanner := bufio.NewScanner(bytes.NewReader(output))
+	var devName string
+	var devCount int
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		fields := strings.Fields(line)
+		if len(fields) >= 2 && fields[1] == "ONLINE" && strings.HasPrefix(fields[0], "/dev/") {
+			devCount++
+			if devName == "" {
+				devName = stripDevicePartition(fields[0])
+			} else {
+				maybeWarnf(ctx, "unsupported configuration: multiple devices (i.e. %q, %q) detected for zpool %q", devName, fields[0], string(poolName))
+			}
+		}
+	}
+
+	switch {
+	case devCount == 1:
+		return devName, nil
+	case devCount > 1:
+		return devName, errors.Newf("unsupported configuration: %d devices detected for zpool %q", devCount, string(poolName))
+	default:
+		return "", fmt.Errorf("no device found for zpool %q", poolName)
+	}
+}
+
+var (
+	nvmePartitionRegex = regexp.MustCompile(`^(nvme\d+n\d+)(p\d+)?$`)
+	scsiPartitionRegex = regexp.MustCompile(`^(ram|loop|fd|(h|s|v|xv)d[a-z])(\d+)?$`)
+)
+
+// stripDevicePartition removes partition suffix from a device path.
+func stripDevicePartition(devicePath string) string {
+	base := filepath.Base(devicePath)
+
+	nvmeMatches := nvmePartitionRegex.FindStringSubmatch(base)
+	if len(nvmeMatches) == 3 {
+		return nvmeMatches[1]
+	}
+
+	scsiMatches := scsiPartitionRegex.FindStringSubmatch(base)
+	if len(scsiMatches) >= 3 {
+		return scsiMatches[1]
+	}
+
+	// If no match, return original device path
+	return devicePath
+}
+
+// getDeviceID takes a block device name (e.g., nvme5n1) and returns its major and minor numbers.
+func getDeviceID(devPath string) (uint32, uint32, error) {
+	devName := filepath.Base(devPath)
+	devFilePath := fmt.Sprintf("/sys/block/%s/dev", devName)
+	data, err := os.ReadFile(devFilePath)
+	if err != nil {
+		return 0, 0, errors.Newf("unable to read %q: %v", devFilePath, err) // nolint:errwrap
+	}
+
+	return parseDeviceID(devFilePath, data)
+}
+
+func parseDeviceID(devFilePath string, data []byte) (uint32, uint32, error) {
+	devStr := strings.TrimSpace(string(data))
+	parts := strings.Split(devStr, ":")
+	if len(parts) != 2 {
+		return 0, 0, fmt.Errorf("unexpected device string format in %q: %s", devFilePath, devStr)
+	}
+
+	var maj, min uint32
+	_, err := fmt.Sscanf(devStr, "%d:%d", &maj, &min)
+	if err != nil {
+		return 0, 0, errors.Newf("failed parsing device numbers: %v", err) // nolint:errwrap
+	}
+
+	return maj, min, nil
+}
+
+// maybeWarnf is a convenience function to prevent panicing during bootstrap
+// from using logging before it is setup.
+func maybeWarnf(ctx context.Context, format string, args ...interface{}) {
+	if active, _ := log.IsActive(); active {
+		log.Ops.WarningfDepth(ctx, 1, format, args...)
+	}
+}
+
+// maybeInfof is a convenience function to prevent panicing during bootstrap
+// from using logging before it is setup.
+func maybeInfof(ctx context.Context, format string, args ...interface{}) {
+	if active, _ := log.IsActive(); active {
+		log.Ops.InfofDepth(ctx, 1, format, args...)
+	}
+}