Merge pull request ceph#60684 from salieri11/wip-igolikov-asok-56442

mds: add MDS asok command for dumping stray directories

Author: kotreshhr

doc/cephfs/scrub.rst

@@ -162,3 +162,30 @@ Evaluate strays using recursive scrub
   at root, run scrub with flags ``scrub_mdsdir`` and ``recursive``::
 
     ceph tell mds.<fsname>:0 scrub start / recursive,scrub_mdsdir
+
+Dump stray folder content
+=====================================
+
+- In order to dump stray folder content on a specific MDS, use the following command::
+
+    ceph tell mds.<fsname>:0 dump stray
+    {
+    "strays": [
+        {
+            "ino": "0x100000001f7",
+            "stray_prior_path": "/dir/dir1",
+            "client_caps": [
+                {
+                    "client_id": 4156,
+                    "pending": "pAsLsXsFscr",
+                    "issued": "pAsLsXsFscr",
+                    "wanted": "-",
+                    "last_sent": 3
+                }
+            ],
+            "loner": -1,
+            "want_loner": -1,
+            "mds_caps_wanted": [],
+            "is_subvolume": false
+        }
+    ]}

qa/tasks/cephfs/test_strays.py

@@ -1037,3 +1037,32 @@ def test_replicated_delete_speed(self):
 
         duration = (end - begin).total_seconds()
         self.assertLess(duration, (file_count * tick_period) * 0.25)
+    
+    def test_asok_dump_stray_command(self):
+        """
+        Test MDS asok dump stray command
+        """
+
+        LOW_LIMIT = 50
+        # need to create more folder to force fragmentation, creating more then needed
+        # to be on the safe side.
+        # we want to test the case when dumping stray folder must wait for the next dirfrag to be fetched
+        NUM_DIRS = LOW_LIMIT * 20
+        TOP_DIR = "topdir"
+        self.config_set("mds", "mds_bal_split_size", str(LOW_LIMIT))
+        self.assertEqual(self.config_get("mds", "mds_bal_split_size"), str(LOW_LIMIT), "LOW_LIMIT was not set on mds!")
+              
+        # create 2 level tree with enough folders to force the stray folder be fragmented
+        # total of NUM_DIRS subdirs will be created
+        self.mount_a.run_shell(f"mkdir -p {TOP_DIR}/subdir{{1..{NUM_DIRS}}}")  
+        # create snapshot
+        self.mount_a.run_shell(f"mkdir {TOP_DIR}/.snap/snap1")
+
+        # delete 2nd level dirs to generate strays
+        # don't wait, we want to dump stray dir while delete runs, to make it more interesting
+        self.mount_a.run_shell(f"rm -rf {TOP_DIR}/*", wait=False)
+
+        # wait for all deleted folders to become strays
+        self.wait_until_equal(
+        lambda: len(self.fs.rank_tell(["dump", "stray"])),
+        expect_val=NUM_DIRS, timeout=60, period=1)

src/mds/MDCache.cc

@@ -10374,13 +10374,21 @@ void MDCache::notify_global_snaprealm_update(int snap_op)
 
 struct C_MDC_RetryScanStray : public MDCacheContext {
   dirfrag_t next;
-  C_MDC_RetryScanStray(MDCache *c,  dirfrag_t n) : MDCacheContext(c), next(n) { }
+  std::unique_ptr<MDCache::C_MDS_DumpStrayDirCtx> cmd_ctx;
+  C_MDC_RetryScanStray(MDCache *c,  dirfrag_t n, std::unique_ptr<MDCache::C_MDS_DumpStrayDirCtx> ctx) :
+   MDCacheContext(c), next(n), cmd_ctx(std::move(ctx)) {}
   void finish(int r) override {
-    mdcache->scan_stray_dir(next);
+    mdcache->scan_stray_dir(next, std::move(cmd_ctx));
   }
 };
 
-void MDCache::scan_stray_dir(dirfrag_t next)
+/*
+ * If the cmd_ctx is not nullptr, the caller is asok command handler,
+ * which will block until the on_finish will be called.
+ * The cmd_ctx holds the formatter to dump stray dir content while scanning.
+ * The function can return EAGAIN, to make possible waiting semantics clear.
+*/
+int MDCache::scan_stray_dir(dirfrag_t next, std::unique_ptr<MDCache::C_MDS_DumpStrayDirCtx> cmd_ctx)
 {
   dout(10) << "scan_stray_dir " << next << dendl;
 
@@ -10399,13 +10407,13 @@ void MDCache::scan_stray_dir(dirfrag_t next)
 	continue;
 
       if (!dir->can_auth_pin()) {
-	dir->add_waiter(CDir::WAIT_UNFREEZE, new C_MDC_RetryScanStray(this, dir->dirfrag()));
-	return;
+	dir->add_waiter(CDir::WAIT_UNFREEZE, new C_MDC_RetryScanStray(this, dir->dirfrag(), std::move(cmd_ctx)));
+	return -EAGAIN;
       }
 
       if (!dir->is_complete()) {
-	dir->fetch(new C_MDC_RetryScanStray(this, dir->dirfrag()));
-	return;
+	dir->fetch(new C_MDC_RetryScanStray(this, dir->dirfrag(), std::move(cmd_ctx)));
+	return -EAGAIN;
       }
 
       for (auto &p : dir->items) {
@@ -10414,14 +10422,32 @@ void MDCache::scan_stray_dir(dirfrag_t next)
 	CDentry::linkage_t *dnl = dn->get_projected_linkage();
 	if (dnl->is_primary()) {
 	  CInode *in = dnl->get_inode();
+    // only if we came from asok cmd handler
+    if (cmd_ctx) {
+      cmd_ctx->begin_dump();
+      cmd_ctx->get_formatter()->open_object_section("stray_inode");
+      cmd_ctx->get_formatter()->dump_int("ino: ", in->ino());
+      cmd_ctx->get_formatter()->dump_string("stray_prior_path: ", in->get_inode()->stray_prior_path);
+      in->dump(cmd_ctx->get_formatter(), CInode::DUMP_CAPS);
+      cmd_ctx->get_formatter()->close_section();
+    }
 	  if (in->get_inode()->nlink == 0)
 	    in->state_set(CInode::STATE_ORPHAN);
-	  maybe_eval_stray(in);
+    // no need to evaluate stray when dumping the dir content
+    if (!cmd_ctx) {
+	    maybe_eval_stray(in);
+    }
 	}
       }
     }
     next.frag = frag_t();
   }
+  // only if we came from asok cmd handler
+  if (cmd_ctx) {
+    cmd_ctx->end_dump();
+    cmd_ctx->finish(0);
+  }
+  return 0;
 }
 
 void MDCache::fetch_backtrace(inodeno_t ino, int64_t pool, bufferlist& bl, Context *fin)
@@ -10432,9 +10458,10 @@ void MDCache::fetch_backtrace(inodeno_t ino, int64_t pool, bufferlist& bl, Conte
     mds->logger->inc(l_mds_openino_backtrace_fetch);
 }
 
-
-
-
+int MDCache::stray_status(std::unique_ptr<C_MDS_DumpStrayDirCtx> ctx)
+{
+  return scan_stray_dir(dirfrag_t(), std::move(ctx));
+}
 
 // ========================================================================================
 // DISCOVER

src/mds/MDCache.h

@@ -642,6 +642,40 @@ class MDCache {
     bool ap_freeze = false;
   };
 
+  /**
+   * Helper wrapper, provides both context object with finish function
+   * and placeholder for formatter. Better alternative to passing formatter as another argument
+   * to the MDSCache function
+   */
+  class C_MDS_DumpStrayDirCtx : public MDSInternalContext {
+  public:
+  void finish(int r) override {
+    ceph_assert(on_finish);
+    MDSContext::finish(r);
+    on_finish(r);
+  }
+  Formatter* get_formatter() const {
+    ceph_assert(dump_formatter);
+    return dump_formatter;
+  }
+  void begin_dump() {
+    if(!started) {
+      started = true;
+      get_formatter()->open_array_section("strays");
+    }
+  }
+  void end_dump() {
+    get_formatter()->close_section();
+  }
+  C_MDS_DumpStrayDirCtx(MDCache *c, Formatter* f, std::function<void(int)>&& ext_on_finish) : 
+   MDSInternalContext(c->mds), cache(c), dump_formatter(f), on_finish(std::move(ext_on_finish)) {}
+  private:
+  MDCache *cache;
+  Formatter* dump_formatter;
+  std::function<void(int)> on_finish;
+  bool started = false;
+  };
+
   MDRequestRef lock_path(LockPathConfig config, std::function<void(MDRequestRef const& mdr)> on_locked = {});
 
   void clean_open_file_lists();
@@ -1073,6 +1107,7 @@ class MDCache {
   void dump_tree(CInode *in, const int cur_depth, const int max_depth, Formatter *f);
 
   void cache_status(Formatter *f);
+  int stray_status(std::unique_ptr<C_MDS_DumpStrayDirCtx> ctx);
 
   void dump_resolve_status(Formatter *f) const;
   void dump_rejoin_status(Formatter *f) const;
@@ -1283,7 +1318,7 @@ class MDCache {
   void handle_open_ino(const cref_t<MMDSOpenIno> &m, int err=0);
   void handle_open_ino_reply(const cref_t<MMDSOpenInoReply> &m);
 
-  void scan_stray_dir(dirfrag_t next=dirfrag_t());
+  int scan_stray_dir(dirfrag_t next=dirfrag_t(), std::unique_ptr<C_MDS_DumpStrayDirCtx> ctx = nullptr);
   // -- replicas --
   void handle_discover(const cref_t<MDiscover> &dis);
   void handle_discover_reply(const cref_t<MDiscoverReply> &m);

src/mds/MDSContext.cc

@@ -21,13 +21,12 @@
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_mds
 
-void MDSContext::complete(int r) {
+void MDSContext::finish(int r) {
   MDSRank *mds = get_mds();
   ceph_assert(mds != nullptr);
   ceph_assert(ceph_mutex_is_locked_by_me(mds->mds_lock));
-  dout(10) << "MDSContext::complete: " << typeid(*this).name() << dendl;
+  dout(10) << "MDSContext::finish: " << typeid(*this).name() << dendl;
   mds->heartbeat_reset();
-  return Context::complete(r);
 }
 
 void MDSInternalContextWrapper::finish(int r)

src/mds/MDSContext.h

@@ -44,7 +44,7 @@ template<template<typename> class A>
   using que_alloc = std::deque<MDSContext*, A<MDSContext*>>;
   using que = que_alloc<std::allocator>;
 
-  void complete(int r) override;
+  void finish(int r) override;
   virtual MDSRank *get_mds() = 0;
 };
 

src/mds/MDSDaemon.cc

@@ -544,6 +544,11 @@ void MDSDaemon::set_up_admin_socket()
     asok_hook,
     "run cpu profiling on daemon");
   ceph_assert(r == 0);
+  r = admin_socket->register_command(
+    "dump stray",
+    asok_hook,
+    "dump stray folder content");
+  ceph_assert(r == 0);
 }
 
 void MDSDaemon::clean_up_admin_socket()

src/mds/MDSRank.cc

@@ -3070,6 +3070,23 @@ void MDSRankDispatcher::handle_asok_command(
   } else if (command == "quiesce db") {
     command_quiesce_db(cmdmap, on_finish);
     return;
+  } else if (command == "dump stray") {
+    dout(10) << "dump_stray start" <<  dendl;
+    // the context is a wrapper for formatter to be used while scanning stray dir
+    auto context = std::make_unique<MDCache::C_MDS_DumpStrayDirCtx>(mdcache, f,
+     [this,on_finish](int r) {
+      // completion callback, will be called when scan is done
+      dout(10) << "dump_stray done" <<  dendl;
+      bufferlist bl;
+      on_finish(r, "", bl);
+    });
+    std::lock_guard l(mds_lock);
+    r = mdcache->stray_status(std::move(context));
+    // since the scanning op can be async, we want to know it, for better semantics
+    if (r == -EAGAIN) {
+     dout(10) << "dump_stray wait" << dendl;
+    }
+    return;
   } else {
     r = -ENOSYS;
   }
@@ -3506,7 +3523,7 @@ void MDSRank::command_quiesce_path(Formatter* f, const cmdmap_t& cmdmap, asok_fi
 
   // This is a little ugly, apologies.
   // We should still be under the mds lock for this test to be valid.
-  // MDCache will delete the quiesce_ctx if it manages to complete syncrhonously,
+  // MDCache will delete the quiesce_ctx if it manages  
   // so we are testing the `mdr->internal_op_finish` to see if that has happend
   if (!await && mdr && mdr->internal_op_finish) {
     ceph_assert(mdr->internal_op_finish == quiesce_ctx);