Merge PR ceph#61562 into main

* refs/pull/61562/head:
	qa: remove redundant and broken test
	mds: skip scrubbing damaged dirfrag
	tools/cephfs/DataScan: test equality of link including frag
	tools/cephfs/DataScan: skip linkages that have been removed
	tools/cephfs/DataScan: do not error out when failing to read a dentry
	tools/cephfs/DataScan: create all ancestors during scan_inodes
	tools/cephfs/DataScan: cleanup debug prints
	qa: remove old MovedDir test
	qa: add data scan tests for ancestry rebuild
	qa: make the directory non-empty to force migration
	qa: avoid unnecessary mds restart

Reviewed-by: Venky Shankar <[email protected]>

Author: batrick

qa/tasks/cephfs/filesystem.py

@@ -61,6 +61,14 @@ def __init__(self, object_name):
     def __str__(self):
         return "Object not found: '{0}'".format(self._object_name)
 
+class FSDamaged(Exception):
+    def __init__(self, ident, ranks):
+        self.ident = ident
+        self.ranks = ranks
+
+    def __str__(self):
+        return f"File system {self.ident} has damaged ranks {self.ranks}"
+
 class FSMissing(Exception):
     def __init__(self, ident):
         self.ident = ident
@@ -1092,10 +1100,17 @@ def are_daemons_healthy(self, status=None, skip_max_mds_check=False):
             mds.check_status()
 
         active_count = 0
-        mds_map = self.get_mds_map(status=status)
 
+        if status is None:
+            status = self.status()
+
+        mds_map = self.get_mds_map(status=status)
         log.debug("are_daemons_healthy: mds map: {0}".format(mds_map))
 
+        damaged = self.get_damaged(status=status)
+        if damaged:
+            raise FSDamaged(self.id, damaged)
+
         for mds_id, mds_status in mds_map['info'].items():
             if mds_status['state'] not in ["up:active", "up:standby", "up:standby-replay"]:
                 log.warning("Unhealthy mds state {0}:{1}".format(mds_id, mds_status['state']))

qa/tasks/cephfs/test_data_scan.py

@@ -15,6 +15,8 @@
 from textwrap import dedent
 
 from teuthology.exceptions import CommandFailedError
+from teuthology import contextutil
+from tasks.cephfs.filesystem import FSDamaged
 from tasks.cephfs.cephfs_test_case import CephFSTestCase, for_teuthology
 
 log = logging.getLogger(__name__)
@@ -84,6 +86,18 @@ def damage(self):
         pool = self._filesystem.get_metadata_pool_name()
         self._filesystem.rados(["purge", pool, '--yes-i-really-really-mean-it'])
 
+    def is_damaged(self):
+        sleep = 2
+        timeout = 120
+        with contextutil.safe_while(sleep=sleep, tries=timeout/sleep) as proceed:
+            while proceed():
+                try:
+                    self._filesystem.wait_for_daemons()
+                except FSDamaged as e:
+                    if 0 in e.ranks:
+                        return True
+        return False
+
     def flush(self):
         """
         Called after client unmount, after write: flush whatever you want
@@ -150,6 +164,90 @@ def validate(self):
         self.assert_equal(target, "symdir/onemegs")
         return self._errors
 
+class NestedDirWorkload(Workload):
+    """
+    Nested directories, one is lost.
+    """
+
+    def write(self):
+        self._mount.run_shell_payload("mkdir -p dir_x/dir_xx/dir_xxx/")
+        self._mount.run_shell_payload("dd if=/dev/urandom of=dir_x/dir_xx/dir_xxx/file_y conv=fsync bs=1 count=1")
+        self._initial_state = self._filesystem.read_cache("dir_x/dir_xx", depth=0)
+
+    def damage(self):
+        dirfrag_obj = "{0:x}.00000000".format(self._initial_state[0]['ino'])
+        self._filesystem.radosm(["rm", dirfrag_obj])
+
+    def is_damaged(self):
+        # workload runner expects MDS to be offline
+        self._filesystem.fail()
+        return True
+
+    def validate(self):
+        self._mount.run_shell_payload("find dir_x -execdir stat {} +")
+        self._mount.run_shell_payload("stat dir_x/dir_xx/dir_xxx/file_y")
+        return self._errors
+
+class NestedDirWorkloadRename(Workload):
+    """
+    Nested directories, one is lost. With renames.
+    """
+
+    def write(self):
+        self._mount.run_shell_payload("mkdir -p dir_x/dir_xx/dir_xxx/; mkdir -p dir_y")
+        self._mount.run_shell_payload("dd if=/dev/urandom of=dir_x/dir_xx/dir_xxx/file_y conv=fsync bs=1 count=1")
+        self._initial_state = self._filesystem.read_cache("dir_x/dir_xx", depth=0)
+        self._filesystem.flush()
+        self._mount.run_shell_payload("mv dir_x/dir_xx dir_y/dir_yy; sync dir_y")
+
+    def damage(self):
+        dirfrag_obj = "{0:x}.00000000".format(self._initial_state[0]['ino'])
+        self._filesystem.radosm(["rm", dirfrag_obj])
+
+    def is_damaged(self):
+        # workload runner expects MDS to be offline
+        self._filesystem.fail()
+        return True
+
+    def validate(self):
+        self._mount.run_shell_payload("find . -execdir stat {} +")
+        self._mount.run_shell_payload("stat dir_y/dir_yy/dir_xxx/file_y")
+        return self._errors
+
+class NestedDoubleDirWorkloadRename(Workload):
+    """
+    Nested directories, two lost with backtraces to rebuild. With renames.
+    """
+
+    def write(self):
+        self._mount.run_shell_payload("mkdir -p dir_x/dir_xx/dir_xxx/; mkdir -p dir_y")
+        self._mount.run_shell_payload("dd if=/dev/urandom of=dir_x/dir_xx/dir_xxx/file_y conv=fsync bs=1 count=1")
+        self._initial_state = []
+        self._initial_state.append(self._filesystem.read_cache("dir_x/dir_xx", depth=0))
+        self._initial_state.append(self._filesystem.read_cache("dir_y", depth=0))
+        self._filesystem.flush()
+        self._mount.run_shell_payload("""
+        mv dir_x/dir_xx dir_y/dir_yy
+        sync dir_y
+        dd if=/dev/urandom of=dir_y/dir_yy/dir_xxx/file_z conv=fsync bs=1 count=1
+        """)
+
+    def damage(self):
+        for o in self._initial_state:
+            dirfrag_obj = "{0:x}.00000000".format(o[0]['ino'])
+            self._filesystem.radosm(["rm", dirfrag_obj])
+
+    def is_damaged(self):
+        # workload runner expects MDS to be offline
+        self._filesystem.fail()
+        return True
+
+    def validate(self):
+        self._mount.run_shell_payload("find . -execdir stat {} +")
+        # during recovery: we may get dir_x/dir_xx or dir_y/dir_yy; depending on rados pg iteration order
+        self._mount.run_shell_payload("stat dir_y/dir_yy/dir_xxx/file_y || stat dir_x/dir_xx/dir_xxx/file_y")
+        return self._errors
+
 
 class MovedFile(Workload):
     def write(self):
@@ -319,37 +417,6 @@ def validate(self):
         return self._errors
 
 
-class MovedDir(Workload):
-    def write(self):
-        # Create a nested dir that we will then move.  Two files with two different
-        # backtraces referring to the moved dir, claiming two different locations for
-        # it.  We will see that only one backtrace wins and the dir ends up with
-        # single linkage.
-        self._mount.run_shell(["mkdir", "-p", "grandmother/parent"])
-        self._mount.write_n_mb("grandmother/parent/orig_pos_file", 1)
-        self._filesystem.mds_asok(["flush", "journal"])
-        self._mount.run_shell(["mkdir", "grandfather"])
-        self._mount.run_shell(["mv", "grandmother/parent", "grandfather"])
-        self._mount.write_n_mb("grandfather/parent/new_pos_file", 2)
-        self._filesystem.mds_asok(["flush", "journal"])
-
-        self._initial_state = (
-            self._mount.stat("grandfather/parent/orig_pos_file"),
-            self._mount.stat("grandfather/parent/new_pos_file")
-        )
-
-    def validate(self):
-        root_files = self._mount.ls()
-        self.assert_equal(len(root_files), 1)
-        self.assert_equal(root_files[0] in ["grandfather", "grandmother"], True)
-        winner = root_files[0]
-        st_opf = self._mount.stat(f"{winner}/parent/orig_pos_file", sudo=True)
-        st_npf = self._mount.stat(f"{winner}/parent/new_pos_file", sudo=True)
-
-        self.assert_equal(st_opf['st_size'], self._initial_state[0]['st_size'])
-        self.assert_equal(st_npf['st_size'], self._initial_state[1]['st_size'])
-
-
 class MissingZerothObject(Workload):
     def write(self):
         self._mount.run_shell(["mkdir", "subdir"])
@@ -391,10 +458,6 @@ def validate(self):
 class TestDataScan(CephFSTestCase):
     MDSS_REQUIRED = 2
 
-    def is_marked_damaged(self, rank):
-        mds_map = self.fs.get_mds_map()
-        return rank in mds_map['damaged']
-
     def _rebuild_metadata(self, workload, workers=1, unmount=True):
         """
         That when all objects in metadata pool are removed, we can rebuild a metadata pool
@@ -416,28 +479,20 @@ def _rebuild_metadata(self, workload, workers=1, unmount=True):
 
         # After recovery, we need the MDS to not be strict about stats (in production these options
         # are off by default, but in QA we need to explicitly disable them)
-        self.fs.set_ceph_conf('mds', 'mds verify scatter', False)
-        self.fs.set_ceph_conf('mds', 'mds debug scatterstat', False)
+        self.config_set('mds', 'mds verify scatter', False)
+        self.config_set('mds', 'mds debug scatterstat', False)
 
         # Apply any data damage the workload wants
         workload.damage()
 
         # Reset the MDS map in case multiple ranks were in play: recovery procedure
         # only understands how to rebuild metadata under rank 0
         self.fs.reset()
+        self.assertEqual(self.fs.get_var('max_mds'), 1)
 
         self.fs.set_joinable() # redundant with reset
 
-        def get_state(mds_id):
-            info = self.mds_cluster.get_mds_info(mds_id)
-            return info['state'] if info is not None else None
-
-        self.wait_until_true(lambda: self.is_marked_damaged(0), 60)
-        for mds_id in self.fs.mds_ids:
-            self.wait_until_equal(
-                    lambda: get_state(mds_id),
-                    "up:standby",
-                    timeout=60)
+        self.assertTrue(workload.is_damaged())
 
         self.fs.table_tool([self.fs.name + ":0", "reset", "session"])
         self.fs.table_tool([self.fs.name + ":0", "reset", "snap"])
@@ -450,7 +505,7 @@ def get_state(mds_id):
                 self.fs.journal_tool(["journal", "reset", "--yes-i-really-really-mean-it"], 0)
 
         self.fs.journal_tool(["journal", "reset", "--force", "--yes-i-really-really-mean-it"], 0)
-        self.fs.data_scan(["init"])
+        self.fs.data_scan(["init", "--force-init"])
         self.fs.data_scan(["scan_extents"], worker_count=workers)
         self.fs.data_scan(["scan_inodes"], worker_count=workers)
         self.fs.data_scan(["scan_links"])
@@ -461,7 +516,7 @@ def get_state(mds_id):
         self.run_ceph_cmd('mds', 'repaired', '0')
 
         # Start the MDS
-        self.fs.mds_restart()
+        self.fs.set_joinable() # necessary for some tests without damage
         self.fs.wait_for_daemons()
         log.info(str(self.mds_cluster.status()))
 
@@ -490,6 +545,15 @@ def test_rebuild_simple(self):
     def test_rebuild_symlink(self):
         self._rebuild_metadata(SymlinkWorkload(self.fs, self.mount_a))
 
+    def test_rebuild_nested(self):
+        self._rebuild_metadata(NestedDirWorkload(self.fs, self.mount_a))
+
+    def test_rebuild_nested_rename(self):
+        self._rebuild_metadata(NestedDirWorkloadRename(self.fs, self.mount_a))
+
+    def test_rebuild_nested_double_rename(self):
+        self._rebuild_metadata(NestedDoubleDirWorkloadRename(self.fs, self.mount_a))
+
     def test_rebuild_moved_file(self):
         self._rebuild_metadata(MovedFile(self.fs, self.mount_a))
 
@@ -499,9 +563,6 @@ def test_rebuild_backtraceless(self):
     def test_rebuild_backtraceless_with_lf_dir_removed(self):
         self._rebuild_metadata(BacktracelessFileRemoveLostAndFoundDirectory(self.fs, self.mount_a), unmount=False)
 
-    def test_rebuild_moved_dir(self):
-        self._rebuild_metadata(MovedDir(self.fs, self.mount_a))
-
     def test_rebuild_missing_zeroth(self):
         self._rebuild_metadata(MissingZerothObject(self.fs, self.mount_a))
 
@@ -723,10 +784,10 @@ def test_rebuild_inotable(self):
         mds0_id = active_mds_names[0]
         mds1_id = active_mds_names[1]
 
-        self.mount_a.run_shell(["mkdir", "dir1"])
+        self.mount_a.run_shell_payload("mkdir -p dir1/dir2")
         dir_ino = self.mount_a.path_to_ino("dir1")
         self.mount_a.setfattr("dir1", "ceph.dir.pin", "1")
-        # wait for subtree migration
+        self._wait_subtrees([('/dir1', 1)], rank=1)
 
         file_ino = 0;
         while True:

qa/tasks/cephfs/test_forward_scrub.py

@@ -390,87 +390,6 @@ def test_health_status_after_dentry_repair(self):
         # Clean up the omap object
         self.fs.radosm(["setomapval", dirfrag_obj, "file_to_be_damaged_head", junk])
 
-    def test_health_status_after_dirfrag_repair(self):
-        """
-        Test that the damage health status is cleared
-        after the damaged dirfrag is repaired
-        """
-        self.mount_a.run_shell(["mkdir", "dir"])
-        self.mount_a.run_shell(["touch", "dir/file"])
-        self.mount_a.run_shell(["mkdir", "testdir"])
-        self.mount_a.run_shell(["ln", "dir/file", "testdir/hardlink"])
-
-        dir_ino = self.mount_a.path_to_ino("dir")
-
-        # Ensure everything is written to backing store
-        self.mount_a.umount_wait()
-        self.fs.mds_asok(["flush", "journal"])
-
-        # Drop everything from the MDS cache
-        self.fs.fail()
-
-        self.fs.radosm(["rm", "{0:x}.00000000".format(dir_ino)])
-
-        self.fs.journal_tool(['journal', 'reset', '--yes-i-really-really-mean-it'], 0)
-        self.fs.set_joinable()
-        self.fs.wait_for_daemons()
-        self.mount_a.mount_wait()
-
-        # Check that touching the hardlink gives EIO
-        ran = self.mount_a.run_shell(["stat", "testdir/hardlink"], wait=False)
-        try:
-            ran.wait()
-        except CommandFailedError:
-            self.assertTrue("Input/output error" in ran.stderr.getvalue())
-
-        out_json = self.fs.run_scrub(["start", "/dir", "recursive"])
-        self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
-
-        # Check that an entry is created in the damage table
-        damage = json.loads(
-            self.fs.mon_manager.raw_cluster_cmd(
-                'tell', 'mds.{0}'.format(self.fs.get_active_names()[0]),
-                "damage", "ls", '--format=json-pretty'))
-        self.assertEqual(len(damage), 3)
-        damage_types = set()
-        for i in range(0, 3):
-            damage_types.add(damage[i]['damage_type'])
-        self.assertIn("dir_frag", damage_types)
-        self.wait_until_true(lambda: self._is_MDS_damage(), timeout=100)
-
-        out_json = self.fs.run_scrub(["start", "/dir", "recursive,repair"])
-        self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
-
-        # Check that the entry is cleared from the damage table
-        damage = json.loads(
-            self.fs.mon_manager.raw_cluster_cmd(
-                'tell', 'mds.{0}'.format(self.fs.get_active_names()[0]),
-                "damage", "ls", '--format=json-pretty'))
-        self.assertEqual(len(damage), 1)
-        self.assertNotEqual(damage[0]['damage_type'], "dir_frag")
-
-        self.mount_a.umount_wait()
-        self.fs.mds_asok(["flush", "journal"])
-        self.fs.fail()
-
-        # Run cephfs-data-scan
-        self.fs.data_scan(["scan_extents", self.fs.get_data_pool_name()])
-        self.fs.data_scan(["scan_inodes", self.fs.get_data_pool_name()])
-        self.fs.data_scan(["scan_links"])
-
-        self.fs.set_joinable()
-        self.fs.wait_for_daemons()
-        self.mount_a.mount_wait()
-
-        out_json = self.fs.run_scrub(["start", "/dir", "recursive,repair"])
-        self.assertEqual(self.fs.wait_until_scrub_complete(tag=out_json["scrub_tag"]), True)
-        damage = json.loads(
-            self.fs.mon_manager.raw_cluster_cmd(
-                'tell', 'mds.{0}'.format(self.fs.get_active_names()[0]),
-                "damage", "ls", '--format=json-pretty'))
-        self.assertEqual(len(damage), 0)
-        self.wait_until_true(lambda: not self._is_MDS_damage(), timeout=100)
-
     def test_health_status_after_backtrace_repair(self):
         """
         Test that the damage health status is cleared