Merge PR ceph#64434 into main

* refs/pull/64434/head:

Reviewed-by: Juan Miguel Olmo <[email protected]>
Reviewed-by: Venky Shankar <[email protected]>

Author: vshankar

doc/cephfs/mds-config-ref.rst

@@ -1,3 +1,4 @@
+.. _MDS Config Reference:
 ======================
  MDS Config Reference
 ======================
@@ -65,3 +66,4 @@
 .. confval:: mds_min_caps_per_client
 .. confval:: mds_symlink_recovery
 .. confval:: mds_extraordinary_events_dump_interval
+.. confval:: subv_metrics_window_interval

doc/cephfs/metrics.rst

@@ -66,6 +66,40 @@ CephFS exports client metrics as :ref:`Labeled Perf Counters`, which could be us
      - Gauge
      - Number of bytes written in input/output operations generated by all processes
 
+Subvolume Metrics
+-----------------
+
+CephFS exports subvolume metrics as :ref:`Labeled Perf Counters`, which could be used to monitor the subvolume performance. CephFS exports the below subvolume metrics.
+Subvolume metrics are aggregated within sliding window of 30 seconds (default value, configurable via the ``subv_metrics_window_interval`` parameter, see :ref:`MDS config reference`).
+In large Ceph clusters with tens of thousands of subvolumes, this parameter also helps clean up stale metrics.
+When a subvolume’s sliding window becomes empty, it's metrics are removed and not reported as “zero” values, reducing memory usage and computational overhead.
+
+.. list-table:: Subvolume Metrics
+   :widths: 25 25 75
+   :header-rows: 1
+
+   * - Name
+     - Type
+     - Description
+   * - ``avg_read_iops``
+     - Gauge
+     - Average read IOPS (input/output operations per second) over the sliding window.
+   * - ``avg_read_tp_Bps``
+     - Gauge
+     - Average read throughput in bytes per second.
+   * - ``avg_read_lat_msec``
+     - Gauge
+     - Average read latency in milliseconds.
+   * - ``avg_write_iops``
+     - Gauge
+     - Average write IOPS over the sliding window.
+   * - ``avg_write_tp_Bps``
+     - Gauge
+     - Average write throughput in bytes per second.
+   * - ``avg_write_lat_msec``
+     - Gauge
+     - Average write latency in milliseconds.
+
 Getting Metrics
 ===============
 
@@ -130,3 +164,21 @@ The metrics could be scraped from the MDS admin socket as well as using the tell
             }
         }
     ]
+
+The subvolume metrics are dumped as a part of the same command. The ``mds_subvolume_metrics`` section in the output of ``counter dump`` command displays the metrics for each client as shown below::
+
+    "mds_subvolume_metrics": [
+        {
+            "labels": {
+                "fs_name": "a",
+                "subvolume_path": "/volumes/_nogroup/test_subvolume"
+            },
+            "counters": {
+                "avg_read_iops": 0,
+                "avg_read_tp_Bps": 11,
+                "avg_read_lat_msec": 0,
+                "avg_write_iops": 1564,
+                "avg_write_tp_Bps": 6408316,
+                "avg_write_lat_msec": 338
+            }
+        }

qa/suites/fs/volumes/mount

@@ -0,0 +1 @@
+../.qa/

qa/suites/fs/volumes/mount/.qa

@@ -0,0 +1 @@
+.qa/cephfs/mount/fuse.yaml

qa/suites/fs/volumes/mount/fuse.yaml

@@ -6,4 +6,5 @@ tasks:
         - tasks.cephfs.test_volumes.TestVolumeCreate
         - tasks.cephfs.test_volumes.TestSubvolumeGroups
         - tasks.cephfs.test_volumes.TestSubvolumes
-        - tasks.cephfs.test_subvolume
+        - tasks.cephfs.test_subvolume.TestSubvolume
+        - tasks.cephfs.test_subvolume.TestSubvolumeReplicated

qa/suites/fs/volumes/tasks/volumes/test/basic.yaml

@@ -0,0 +1,12 @@
+overrides:
+  install:
+    extra_system_packages:
+      rpm:
+        - fio
+      deb:
+        - fio
+tasks:
+  - cephfs_test_runner:
+      fail_on_skip: false
+      modules:
+        - tasks.cephfs.test_subvolume.TestSubvolumeMetrics

qa/suites/fs/volumes/tasks/volumes/test/metrics.yaml

@@ -1713,4 +1713,7 @@ def validate_subvol_options(self):
             path_to_mount = subvol_paths[mount_subvol_num]
             self.cephfs_mntpt = path_to_mount
 
+    def get_mount_point(self):
+        return self.hostfs_mntpt
+
 CephFSMount = CephFSMountBase

qa/tasks/cephfs/mount.py

@@ -1,8 +1,10 @@
 import logging
 from time import sleep
+import os
 
 from tasks.cephfs.cephfs_test_case import CephFSTestCase
 from teuthology.exceptions import CommandFailedError
+from teuthology.contextutil import safe_while
 
 log = logging.getLogger(__name__)
 
@@ -16,6 +18,7 @@ def setUp(self):
         self.setup_test()
 
     def tearDown(self):
+        #pass
         # clean up
         self.cleanup_test()
         super().tearDown()
@@ -185,7 +188,7 @@ def test_subvolume_create_snapshot_inside_new_subvolume_parent(self):
         self.mount_a.run_shell(['mkdir', 'group/subvol2/dir/.snap/s2'])
 
         # override subdir subvolume with parent subvolume
-        self.mount_a.run_shell(['setfattr', '-n', 'ceph.dir.subvolume',
+        (['setfattr', '-n', 'ceph.dir.subvolume',
                                 '-v', '1', 'group/subvol2/dir'])
         self.mount_a.run_shell(['setfattr', '-n', 'ceph.dir.subvolume',
                                 '-v', '1', 'group/subvol2'])
@@ -197,6 +200,7 @@ def test_subvolume_create_snapshot_inside_new_subvolume_parent(self):
         # clean up
         self.mount_a.run_shell(['rmdir', 'group/subvol2/dir/.snap/s2'])
 
+
     def test_subvolume_vxattr_removal_without_setting(self):
         """
         To verify that the ceph.dir.subvolume vxattr removal without setting doesn't cause mds crash
@@ -209,7 +213,6 @@ def test_subvolume_vxattr_removal_without_setting(self):
         # cleanup
         self.mount_a.run_shell(['rm', '-rf', 'group/subvol3'])
 
-
 class TestSubvolumeReplicated(CephFSTestCase):
     CLIENTS_REQUIRED = 1
     MDSS_REQUIRED = 2
@@ -245,3 +248,108 @@ def test_subvolume_replicated(self):
         ino0 = self.fs.read_cache("/dir1/dir2", depth=0, rank=0)[0]
         self.assertFalse(ino0['is_auth'])
         self.assertTrue(ino0['is_subvolume'])
+
+class TestSubvolumeMetrics(CephFSTestCase):
+    CLIENTS_REQUIRED = 1
+    MDSS_REQUIRED = 1
+
+    def get_subvolume_metrics(self, mds_rank=0):
+        """
+        Helper to fetch current subvolume metrics from MDS counters using rank_tell.
+        """
+        mds_info = self.fs.get_rank(rank=mds_rank)
+        mds_name = mds_info['name']
+        counters = self.fs.mds_tell(["counter", "dump"], mds_id=mds_name)
+        return counters.get("mds_subvolume_metrics")
+
+    def test_subvolume_metrics_lifecycle(self):
+        """
+        Verify that subvolume metrics are initially absent, appear after IO,
+        and disappear after the aggregation window expires.
+        """
+        subvol_name = "metrics_subv"
+        subv_path = "/volumes/_nogroup/metrics_subv"
+
+        # no metrics initially
+        subvol_metrics = self.get_subvolume_metrics()
+        self.assertFalse(subvol_metrics, "Subvolume metrics should not be present before I/O")
+
+        # create subvolume
+        self.fs.run_ceph_cmd('fs', 'subvolume', 'create', 'cephfs', subvol_name)
+
+        # generate some I/O
+        mount_point = self.mount_a.get_mount_point()
+        suvolume_fs_path = self.fs.get_ceph_cmd_stdout('fs', 'subvolume', 'getpath', 'cephfs', subvol_name).strip()
+        suvolume_fs_path = os.path.join(mount_point, suvolume_fs_path.strip('/'))
+
+        # do some writes
+        filename = os.path.join(suvolume_fs_path, "file0")
+        self.mount_a.run_shell_payload("sudo fio "
+                                       "--name test -rw=write "
+                                       "--bs=4k --numjobs=1 --time_based "
+                                       "--runtime=20s --verify=0 --size=1G "
+                                       f"--filename={filename}", wait=True)
+
+        subvol_metrics = None
+        with safe_while(sleep=1, tries=30, action='wait for subvolume write counters') as proceed:
+            while proceed():
+                # verify that metrics are available
+                subvol_metrics = self.get_subvolume_metrics()
+                if subvol_metrics:
+                    break
+
+        log.debug(f'verifying for write: subvol_metrics={subvol_metrics}')
+
+        # Extract first metric entry
+        metric = subvol_metrics[0]
+        counters = metric["counters"]
+        labels = metric["labels"]
+
+        # Label checks
+        self.assertEqual(labels["fs_name"], "cephfs", "Unexpected fs_name in subvolume metrics")
+        self.assertEqual(labels["subvolume_path"], subv_path, "Unexpected subvolume_path in subvolume metrics")
+
+        # Counter presence and value checks
+        self.assertIn("avg_read_iops", counters)
+        self.assertIn("avg_read_tp_Bps", counters)
+        self.assertIn("avg_read_lat_msec", counters)
+        self.assertIn("avg_write_iops", counters)
+        self.assertIn("avg_write_tp_Bps", counters)
+        self.assertIn("avg_write_lat_msec", counters)
+
+        # check write metrics
+        self.assertGreater(counters["avg_write_iops"], 0, "Expected avg_write_iops to be > 0")
+        self.assertGreater(counters["avg_write_tp_Bps"], 0, "Expected avg_write_tp_Bps to be > 0")
+        self.assertGreaterEqual(counters["avg_write_lat_msec"], 0, "Expected avg_write_lat_msec to be > 0")
+
+        # do some reads
+        self.mount_a.run_shell_payload("sudo fio "
+                                       "--name test -rw=read "
+                                       "--bs=4k --numjobs=1 --time_based "
+                                       "--runtime=20s --verify=0 --size=1G "
+                                       f"--filename={filename}", wait=True)
+
+        subvol_metrics = None
+        with safe_while(sleep=1, tries=30, action='wait for subvolume read counters') as proceed:
+            while proceed():
+                # verify that metrics are available
+                subvol_metrics = self.get_subvolume_metrics()
+                if subvol_metrics:
+                    break
+
+        log.debug(f'verifying for read: subvol_metrics={subvol_metrics}')
+
+        metric = subvol_metrics[0]
+        counters = metric["counters"]
+
+        # Assert expected values (example: write I/O occurred, read did not)
+        self.assertGreater(counters["avg_read_iops"], 0, "Expected avg_read_iops to be >= 0")
+        self.assertGreater(counters["avg_read_tp_Bps"], 0, "Expected avg_read_tp_Bps to be >= 0")
+        self.assertGreaterEqual(counters["avg_read_lat_msec"], 0, "Expected avg_read_lat_msec to be >= 0")
+
+        # wait for metrics to expire after inactivity
+        sleep(60)
+
+        # verify that metrics are not present anymore
+        subvolume_metrics = self.get_subvolume_metrics()
+        self.assertFalse(subvolume_metrics, "Subvolume metrics should be gone after inactivity window")