btrfs: add io read stats per device to devinfo

This adds read stats per device to devinfo to evaluate the effects of
different read policies better.

This adds a new file /sys/fs/btrfs/BTRFS-UUID/devinfo/ID/read_stats.

Signed-off-by: Kai Krakow <kai@kaishome.de>

Author: kakra

fs/btrfs/Kconfig

@@ -129,6 +129,18 @@ config BTRFS_ALLOCATOR_HINTS
 
 	  If unsure, say N.
 
+config BTRFS_PER_DEVICE_IO_STATS
+	bool "Btrfs per io devices stats"
+	depends on BTRFS_FS
+	default n
+	help
+	  Enable collecting io read stats per devices to evaluate the effects
+	  of different read policies better.
+
+	  This adds a new file /sys/fs/btrfs/BTRFS-UUID/devinfo/ID/read_stats.
+
+	  If unsure, say N.
+
 config BTRFS_EXPERIMENTAL
 	bool "Btrfs experimental features"
 	depends on BTRFS_FS

fs/btrfs/sysfs.c

@@ -26,6 +26,10 @@
 #include "fs.h"
 #include "accessors.h"
 
+#ifdef CONFIG_BTRFS_PER_DEVICE_IO_STATS
+#include <linux/part_stat.h>
+#endif
+
 /*
  * Structure name                       Path
  * --------------------------------------------------------------------------
@@ -2214,12 +2218,36 @@ static ssize_t btrfs_devinfo_type_store(struct kobject *kobj,
 BTRFS_ATTR_RW(devid, type, btrfs_devinfo_type_show, btrfs_devinfo_type_store);
 #endif
 
+#ifdef CONFIG_BTRFS_PER_DEVICE_IO_STATS
+static ssize_t btrfs_devinfo_read_stats_show(struct kobject *kobj,
+                                                       struct kobj_attribute *a, char *buf)
+{
+	struct btrfs_device *device = container_of(kobj, struct btrfs_device,
+						   devid_kobj);
+	u64 read_wait = part_stat_read(device->bdev, nsecs[READ]);
+	unsigned long read_ios = part_stat_read(device->bdev, ios[READ]);
+
+	u64 avg_wait = 0;
+	if (read_wait && read_ios && read_wait >= read_ios)
+		avg_wait = div_u64(read_wait, read_ios);
+
+	return scnprintf(buf, PAGE_SIZE, "ios %lu wait %llu avg %llu age %llu ignored %llu\n",
+	                 read_ios, read_wait, avg_wait,
+	                 (u64)atomic64_read(&device->last_io_age),
+	                 (u64)atomic64_read(&device->stripe_ignored));
+}
+BTRFS_ATTR(devid, read_stats, btrfs_devinfo_read_stats_show);
+#endif
+
 /*
  * Information about one device.
  *
  * Path: /sys/fs/btrfs/<uuid>/devinfo/<devid>/
  */
 static struct attribute *devid_attrs[] = {
+#ifdef CONFIG_BTRFS_PER_DEVICE_IO_STATS
+	BTRFS_ATTR_PTR(devid, read_stats),
+#endif
 	BTRFS_ATTR_PTR(devid, error_stats),
 	BTRFS_ATTR_PTR(devid, fsid),
 	BTRFS_ATTR_PTR(devid, in_fs_metadata),

fs/btrfs/volumes.c

@@ -6139,6 +6139,15 @@ static int find_live_mirror(struct btrfs_fs_info *fs_info,
 	else
 		num_stripes = map->num_stripes;
 
+#ifdef CONFIG_BTRFS_PER_DEVICE_IO_STATS
+	/* age each possible stripe by 1 IO */
+	for (int i = first; i < first + num_stripes; i++) {
+		struct btrfs_device *device = map->stripes[i].dev;
+		atomic64_inc(&device->last_io_age);
+		atomic64_inc(&device->stripe_ignored);
+	}
+#endif
+
 	switch (policy) {
 	default:
 		/* Shouldn't happen, just warn and use pid instead of failing */
@@ -6174,14 +6183,29 @@ static int find_live_mirror(struct btrfs_fs_info *fs_info,
 	for (tolerance = 0; tolerance < 2; tolerance++) {
 		if (map->stripes[preferred_mirror].dev->bdev &&
 		    (tolerance || map->stripes[preferred_mirror].dev != srcdev))
-			return preferred_mirror;
+			goto out;
 		for (i = first; i < first + num_stripes; i++) {
 			if (map->stripes[i].dev->bdev &&
-			    (tolerance || map->stripes[i].dev != srcdev))
-				return i;
+			    (tolerance || map->stripes[i].dev != srcdev)) {
+				preferred_mirror = i;
+				goto out;
+			}
 		}
 	}
 
+out:
+#ifdef CONFIG_BTRFS_PER_DEVICE_IO_STATS
+	do {
+		struct btrfs_device *preferred_device = map->stripes[preferred_mirror].dev;
+
+		/* reset age of selected stripe */
+		atomic64_set(&preferred_device->last_io_age, 0);
+
+		/* do not count ignores for the selected stripe */
+		atomic64_dec(&preferred_device->stripe_ignored);
+	} while (0);
+#endif
+
 	/* we couldn't find one that doesn't fail.  Just return something
 	 * and the io error handling code will clean up eventually
 	 */

fs/btrfs/volumes.h

@@ -201,6 +201,14 @@ struct btrfs_device {
 
 	/* Bandwidth limit for scrub, in bytes */
 	u64 scrub_speed_max;
+
+#ifdef CONFIG_BTRFS_PER_DEVICE_IO_STATS
+	/* store an age of last read access */
+	atomic64_t last_io_age;
+
+	/* store how often a stripe has been ignored as a read candidate */
+	atomic64_t stripe_ignored;
+#endif
 };
 
 /*