Merge PR ceph#56536 into main

* refs/pull/56536/head:
	mds: do not dispatch aborted internal requests
	qa: add test for killing quiesce_inode with outstanding remote authpin requests
	mds: add lockup debugging command

Reviewed-by: Leonid Usov <[email protected]>

Author: batrick

qa/tasks/cephfs/test_quiesce.py

@@ -638,6 +638,69 @@ def test_quiesce_path_splitauth(self):
         op = self.fs.rank_tell(["quiesce", "path", self.subvolume, '--wait'], rank=0)['op']
         self.assertEqual(op['result'], -1) # EPERM
 
+    def test_quiesce_authpin_wait(self):
+        """
+        That a quiesce_inode op with outstanding remote authpin requests can be killed.
+        """
+
+        self.config_set('mds', 'mds_heartbeat_grace', '60')
+        self._configure_subvolume()
+        self.mount_a.setfattr(".", "ceph.dir.pin.distributed", "1")
+        self._client_background_workload()
+        self._wait_distributed_subtrees(2*2, rank="all", path=self.mntpnt)
+        status = self.fs.status()
+
+        p = self.mount_a.run_shell_payload("ls", stdout=StringIO())
+        dirs = p.stdout.getvalue().strip().split()
+
+        # make rank 0 unresponsive to auth pin requests
+        p = self.run_ceph_cmd("tell", f"mds.{self.fs.id}:1", "lockup", "30000", wait=False)
+
+        qops = []
+        for d in dirs:
+            path = os.path.join(self.mntpnt, d)
+            op = self.fs.rank_tell("quiesce", "path", path, rank=0)['op']
+            reqid = self.reqid_tostr(op['reqid'])
+            log.info(f"created {reqid}")
+            qops.append(reqid)
+
+        def find_quiesce(blocked_on_remote_auth_pin):
+            # verify no quiesce ops
+            ops = self.fs.get_ops(locks=False, rank=0, path="/tmp/mds.0-ops", status=status)['ops']
+            for op in ops:
+                type_data = op['type_data']
+                flag_point = type_data['flag_point']
+                op_type = type_data['op_type']
+                if op_type == 'client_request' or op_type == 'peer_request':
+                    continue
+                if type_data['op_name'] == "quiesce_inode":
+                    if blocked_on_remote_auth_pin:
+                        if flag_point == "requesting remote authpins":
+                            return True
+                    else:
+                        return True
+            return False
+
+        with safe_while(sleep=1, tries=30, action='wait for quiesce op with outstanding remote authpin requests') as proceed:
+            while proceed():
+                if find_quiesce(True):
+                    break
+
+        # okay, now kill all quiesce ops
+        for reqid in qops:
+            self.fs.kill_op(reqid, rank=0)
+
+        # verify some quiesce_inode ops still exist because authpin acks have not been received
+        if not find_quiesce(True):
+            self.fail("did not find quiesce_inode op blocked on remote authpins!")
+
+        # wait for sleep to complete
+        p.wait()
+
+        with safe_while(sleep=1, tries=30, action='wait for quiesce kill') as proceed:
+            while proceed():
+                if not find_quiesce(False):
+                    break
 
     def test_quiesce_path_multirank(self):
         """

src/mds/MDCache.cc

@@ -9818,6 +9818,11 @@ void MDCache::dispatch_request(const MDRequestRef& mdr)
   } else if (mdr->peer_request) {
     mds->server->dispatch_peer_request(mdr);
   } else {
+    if (mdr->aborted) {
+      mdr->aborted = false;
+      request_kill(mdr);
+      return;
+    }
     switch (mdr->internal_op) {
     case CEPH_MDS_OP_QUIESCE_PATH:
       dispatch_quiesce_path(mdr);

src/mds/MDSDaemon.cc

@@ -149,6 +149,13 @@ void MDSDaemon::asok_command(
   if (command == "status") {
     dump_status(f);
     r = 0;
+  } else if (command == "lockup") {
+    int64_t millisecs;
+    cmd_getval(cmdmap, "millisecs", millisecs);
+    derr << "(lockup) sleeping with mds_lock for " << millisecs << dendl;
+    std::lock_guard l(mds_lock);
+    std::this_thread::sleep_for(std::chrono::milliseconds(millisecs));
+    r = 0;
   } else if (command == "exit") {
     outbl.append("Exiting...\n");
     r = 0;
@@ -256,6 +263,11 @@ void MDSDaemon::set_up_admin_socket()
   r = admin_socket->register_command("status", asok_hook,
 				     "high-level status of MDS");
   ceph_assert(r == 0);
+  r = admin_socket->register_command("lockup "
+				     "name=millisecs,type=CephInt,req=true,range=0"
+                                     , asok_hook
+				     , "sleep with mds_lock held (dev)");
+  ceph_assert(r == 0);
   r = admin_socket->register_command("dump_ops_in_flight", asok_hook,
 				     "show the ops currently in flight");
   ceph_assert(r == 0);