mds: Mark the scrub passed if dirfrag is dirty

The in-memory and on-disk stats might not match on a
directory inode if any of the local or remote dirfrag
is dirty.  So don't fail the scrub and mark it as
passed if the local or remote dirfrag is dirty.

Signed-off-by: Kotresh HR <[email protected]>
Fixes: https://tracker.ceph.com/issues/65020

Author: kotreshhr

src/mds/CInode.cc

@@ -5086,6 +5086,7 @@ void CInode::validate_disk_state(CInode::validated_data *results,
       results->raw_stats.memory_value.rstat = in->get_inode()->rstat;
       frag_info_t& dir_info = results->raw_stats.ondisk_value.dirstat;
       nest_info_t& nest_info = results->raw_stats.ondisk_value.rstat;
+      bool has_local_dirty_dirfrag = false;
 
       if (rval != 0) {
         results->raw_stats.error_str << "Failed to read dirfrags off disk";
@@ -5103,6 +5104,8 @@ void CInode::validate_disk_state(CInode::validated_data *results,
 	  ceph_assert(dir->get_version() > 0);
 	  nest_info.add(dir->get_fnode()->accounted_rstat);
 	  dir_info.add(dir->get_fnode()->accounted_fragstat);
+          if (dir->is_auth() && dir->is_dirty())
+              has_local_dirty_dirfrag = true;
         }
       }
       nest_info.rsubdirs++; // it gets one to account for self
@@ -5128,7 +5131,17 @@ void CInode::validate_disk_state(CInode::validated_data *results,
                       "please rerun scrub when system is stable; "
                       "assuming passed for now;" << dendl;
           results->raw_stats.passed = true;
-        }
+        } else if (has_local_dirty_dirfrag || in->has_dirty_remote_dirfrag_scrubbed()) {
+          MDCache *mdcache = in->mdcache; // for dout()
+          auto ino = [this]() { return in->ino(); }; // for dout()
+          dout(20) << (has_local_dirty_dirfrag ? "local": "remote") << " dirfrag : "
+	              "raw stats most likely wont match since it's a directory "
+	              "inode and a dirfrag is dirty; please rerun scrub when "
+		      "system is stable; assuming passed for now;" << dendl;
+          results->raw_stats.passed = true;
+	  if (in->has_dirty_remote_dirfrag_scrubbed())
+	    in->clear_dirty_remote_dirfrag_scrubbed();
+	}
 	goto next;
       }
 

src/mds/CInode.h

@@ -467,6 +467,16 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
     return scrub_infop->queued_frags;
   }
 
+  bool has_dirty_remote_dirfrag_scrubbed() {
+    return dirty_remote_dirfrag_scrubbed;
+  }
+  void mark_dirty_remote_dirfrag_scrubbed() {
+    dirty_remote_dirfrag_scrubbed = true;
+  }
+  void clear_dirty_remote_dirfrag_scrubbed() {
+    dirty_remote_dirfrag_scrubbed = false;
+  }
+
   bool is_multiversion() const {
     return snaprealm ||  // other snaprealms will link to me
       get_inode()->is_dir() ||  // links to me in other snaps
@@ -1255,6 +1265,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
 
   int stickydir_ref = 0;
   std::unique_ptr<scrub_info_t> scrub_infop;
+  bool dirty_remote_dirfrag_scrubbed = false;
   /** @} Scrubbing and fsck */
 };
 

src/mds/ScrubStack.cc

@@ -1001,6 +1001,7 @@ void ScrubStack::handle_scrub(const cref_t<MMDSScrub> &m)
       ceph_assert(diri);
 
       std::vector<CDir*> dfs;
+      bool has_dirty_dirfrag = false;
       MDSGatherBuilder gather(g_ceph_context);
       frag_vec_t frags;
       diri->dirfragtree.get_leaves(frags);
@@ -1024,6 +1025,10 @@ void ScrubStack::handle_scrub(const cref_t<MMDSScrub> &m)
 	    dir->add_waiter(CDir::WAIT_UNFREEZE, gather.new_sub());
 	    continue;
 	  }
+          if (dir->is_dirty()) {
+            dout(10) << __func__ << " found dirty remote dirfrag " << *dir <<  dendl;
+            has_dirty_dirfrag = true;
+	  }
 	  dfs.push_back(dir);
 	}
       }
@@ -1056,7 +1061,8 @@ void ScrubStack::handle_scrub(const cref_t<MMDSScrub> &m)
       }
 
       auto r = make_message<MMDSScrub>(MMDSScrub::OP_QUEUEDIR_ACK, m->get_ino(),
-				       std::move(queued), m->get_tag());
+				       std::move(queued), m->get_tag(),
+				       inodeno_t(), false, false, false, false, has_dirty_dirfrag);
       mdcache->mds->send_message_mds(r, from);
     }
     break;
@@ -1078,6 +1084,8 @@ void ScrubStack::handle_scrub(const cref_t<MMDSScrub> &m)
 
 	    const auto& header = diri->get_scrub_header();
 	    header->set_epoch_last_forwarded(scrub_epoch);
+	    if (m->is_remote_dirfrag_dirty())
+	      diri->mark_dirty_remote_dirfrag_scrubbed();
 	    remove_from_waiting(diri);
 	  }
 	}

src/messages/MMDSScrub.h

@@ -51,6 +51,7 @@ class MMDSScrub : public MMDSOp {
     if (is_force()) out << " force";
     if (is_recursive()) out << " recursive";
     if (is_repair()) out << " repair";
+    if (is_remote_dirfrag_dirty()) out << " dirty_remote_dirfrag";
     out << ")";
   }
   void encode_payload(uint64_t features) override {
@@ -99,23 +100,28 @@ class MMDSScrub : public MMDSOp {
   bool is_repair() const {
     return flags & FLAG_REPAIR;
   }
+  bool is_remote_dirfrag_dirty() const {
+    return flags & FLAG_DIRTY_REMOTE_DIRFRAG;
+  }
 
 protected:
-  static constexpr int HEAD_VERSION = 1;
-  static constexpr int COMPAT_VERSION = 1;
+  static constexpr int HEAD_VERSION = 2;
+  static constexpr int COMPAT_VERSION = 2;
 
   MMDSScrub() : MMDSOp(MSG_MDS_SCRUB, HEAD_VERSION, COMPAT_VERSION) {}
   MMDSScrub(int o)
     : MMDSOp(MSG_MDS_SCRUB, HEAD_VERSION, COMPAT_VERSION), op(o) {}
   MMDSScrub(int o, inodeno_t i, fragset_t&& _frags, std::string_view _tag,
 	    inodeno_t _origin=inodeno_t(), bool internal_tag=false,
-	    bool force=false, bool recursive=false, bool repair=false)
+	    bool force=false, bool recursive=false, bool repair=false,
+	    bool remote_dirfrag_dirty=false)
     : MMDSOp(MSG_MDS_SCRUB, HEAD_VERSION, COMPAT_VERSION),
     op(o), ino(i), frags(std::move(_frags)), tag(_tag), origin(_origin) {
     if (internal_tag) flags |= FLAG_INTERNAL_TAG;
     if (force) flags |= FLAG_FORCE;
     if (recursive) flags |= FLAG_RECURSIVE;
     if (repair) flags |= FLAG_REPAIR;
+    if (remote_dirfrag_dirty) flags |= FLAG_DIRTY_REMOTE_DIRFRAG;
   }
 
   ~MMDSScrub() override {}
@@ -129,6 +135,7 @@ class MMDSScrub : public MMDSOp {
   static constexpr unsigned FLAG_FORCE		= 1<<1;
   static constexpr unsigned FLAG_RECURSIVE	= 1<<2;
   static constexpr unsigned FLAG_REPAIR		= 1<<3;
+  static constexpr unsigned FLAG_DIRTY_REMOTE_DIRFRAG	= 1<<4;
 
   int32_t op;
   inodeno_t ino;