Merge PR ceph#53503 into main

* refs/pull/53503/head:
	qa: add tests for `mds last-seen` command
	doc/cephfs: add documentation for `mds last-seen`
	PendingReleaseNotes: add note on last-seen command
	mon/MDSMonitor: add command to lookup when mds was last seen
	mon/MDSMonitor: set birth time on FSMap during encode
	pybind/mgr/dashboard: show context diff for openapi check

Reviewed-by: Venky Shankar <[email protected]>

Author: batrick

PendingReleaseNotes

@@ -39,6 +39,8 @@
   a large buildup of session metadata resulting in the MDS going read-only due to
   the RADOS operation exceeding the size threshold. `mds_session_metadata_threshold`
   config controls the maximum size that a (encoded) session metadata can grow.
+* CephFS: A new "mds last-seen" command is available for querying the last time
+  an MDS was in the FSMap, subject to a pruning threshold.
 * CephFS: For clusters with multiple CephFS file systems, all the snap-schedule
   commands now expect the '--fs' argument.
 * CephFS: The period specifier ``m`` now implies minutes and the period specifier

doc/cephfs/administration.rst

@@ -280,6 +280,17 @@ Mark the file system rank as repaired. Unlike the name suggests, this command
 does not change a MDS; it manipulates the file system rank which has been
 marked damaged.
 
+::
+
+    ceph mds last-seen <name>
+
+Learn the when the MDS named ``name`` was last in the FSMap. The JSON output
+includes the epoch the MDS was last seen. Historically information is limited by
+the following ``mon`` configuration:
+
+
+.. confval:: mon_fsmap_prune_threshold
+
 
 Required Client Features
 ------------------------

qa/tasks/cephfs/test_admin.py

@@ -4,6 +4,7 @@
 import uuid
 from io import StringIO
 from os.path import join as os_path_join
+import re
 from time import sleep
 
 from teuthology.exceptions import CommandFailedError
@@ -196,6 +197,108 @@ def wait_till_health_warn(self, health_warn, active_mds_id, sleep=3,
                     return
 
 
+class TestMdsLastSeen(CephFSTestCase):
+    """
+    Tests for `mds last-seen` command.
+    """
+
+    MDSS_REQUIRED = 2
+
+    def test_in_text(self):
+        """
+        That `mds last-seen` returns 0 for an MDS currently in the map.
+        """
+
+        status = self.fs.status()
+        r0 = self.fs.get_rank(0, status=status)
+        s = self.get_ceph_cmd_stdout("mds", "last-seen", r0['name'])
+        seconds = int(re.match(r"^(\d+)s$", s).group(1))
+        self.assertEqual(seconds, 0)
+
+    def test_in_json(self):
+        """
+        That `mds last-seen` returns 0 for an MDS currently in the map.
+        """
+
+        status = self.fs.status()
+        r0 = self.fs.get_rank(0, status=status)
+        s = self.get_ceph_cmd_stdout("--format=json", "mds", "last-seen", r0['name'])
+        J = json.loads(s)
+        seconds = int(re.match(r"^(\d+)s$", J['last-seen']).group(1))
+        self.assertEqual(seconds, 0)
+
+    def test_unknown(self):
+        """
+        That `mds last-seen` returns ENOENT for an mds not in recent maps.
+        """
+
+        try:
+            self.get_ceph_cmd_stdout("--format=json", "mds", "last-seen", 'foo')
+        except CommandFailedError as e:
+            self.assertEqual(e.exitstatus, errno.ENOENT)
+        else:
+            self.fail("non-existent mds should fail ENOENT")
+
+    def test_standby(self):
+        """
+        That `mds last-seen` returns 0 for a standby.
+        """
+
+        status = self.fs.status()
+        for info in status.get_standbys():
+            s = self.get_ceph_cmd_stdout("--format=json", "mds", "last-seen", info['name'])
+            J = json.loads(s)
+            seconds = int(re.match(r"^(\d+)s$", J['last-seen']).group(1))
+            self.assertEqual(seconds, 0)
+
+    def test_stopped(self):
+        """
+        That `mds last-seen` returns >0 for mds that is stopped.
+        """
+
+        status = self.fs.status()
+        r0 = self.fs.get_rank(0, status=status)
+        self.fs.mds_stop(mds_id=r0['name'])
+        self.fs.rank_fail()
+        sleep(2)
+        with safe_while(sleep=1, tries=self.fs.beacon_timeout, action='wait for last-seen >0') as proceed:
+            while proceed():
+                s = self.get_ceph_cmd_stdout("--format=json", "mds", "last-seen", r0['name'])
+                J = json.loads(s)
+                seconds = int(re.match(r"^(\d+)s$", J['last-seen']).group(1))
+                if seconds == 0:
+                    continue
+                self.assertGreater(seconds, 1)
+                break
+
+    def test_gc(self):
+        """
+        That historical mds information is eventually garbage collected.
+        """
+
+        prune_time = 20
+        sleep_time = 2
+        self.config_set('mon', 'mon_fsmap_prune_threshold', prune_time)
+        status = self.fs.status()
+        r0 = self.fs.get_rank(0, status=status)
+        self.fs.mds_stop(mds_id=r0['name'])
+        self.fs.rank_fail()
+        last = 0
+        for i in range(prune_time):
+            sleep(sleep_time) # we will sleep twice prune_time
+            try:
+                s = self.get_ceph_cmd_stdout("--format=json", "mds", "last-seen", r0['name'])
+                J = json.loads(s)
+                seconds = int(re.match(r"^(\d+)s$", J['last-seen']).group(1))
+                self.assertGreater(seconds, last)
+                log.debug("last_seen: %ds", seconds)
+                last = seconds
+            except CommandFailedError as e:
+                self.assertEqual(e.exitstatus, errno.ENOENT)
+                self.assertGreaterEqual(last + sleep_time + 1, prune_time) # rounding error add 1
+                return
+        self.fail("map was no garbage collected as expected")
+
 @classhook('_add_valid_tell')
 class TestValidTell(TestAdminCommands):
     @classmethod

qa/tasks/mgr/dashboard/test_health.py

@@ -186,6 +186,7 @@ def test_full_health(self):
                 })
             }),
             'fs_map': JObj({
+                'btime': str,
                 'compat': JObj({
                     'compat': JObj({}, allow_unknown=True, unknown_schema=str),
                     'incompat': JObj(

src/common/options/mon.yaml.in

@@ -778,6 +778,18 @@ options:
   services:
   - mon
   with_legacy: true
+- name: mon_fsmap_prune_threshold
+  type: secs
+  level: advanced
+  desc: prune fsmap older than this threshold in seconds
+  fmt_desc: The monitors keep historical fsmaps in memory to optimize asking
+    when an MDS daemon was last seen in the FSMap. This option controls
+    how far back in time the monitors will look.
+  default: 300
+  flags:
+  - runtime
+  services:
+  - mon
 - name: mds_beacon_mon_down_grace
   type: secs
   level: advanced

src/mds/FSMap.cc

@@ -135,6 +135,7 @@ void Filesystem::dump(Formatter *f) const
 void FSMap::dump(Formatter *f) const
 {
   f->dump_int("epoch", epoch);
+  f->dump_string("btime", fmt::format("{}", btime));
   // Use 'default' naming to match 'set-default' CLI
   f->dump_int("default_fscid", legacy_client_fscid);
 
@@ -168,6 +169,7 @@ void FSMap::dump(Formatter *f) const
 FSMap &FSMap::operator=(const FSMap &rhs)
 {
   epoch = rhs.epoch;
+  btime = rhs.btime;
   next_filesystem_id = rhs.next_filesystem_id;
   legacy_client_fscid = rhs.legacy_client_fscid;
   default_compat = rhs.default_compat;
@@ -206,6 +208,7 @@ void FSMap::generate_test_instances(std::list<FSMap*>& ls)
 void FSMap::print(ostream& out) const
 {
   out << "e" << epoch << std::endl;
+  out << "btime " << fmt::format("{}", btime) << std::endl;
   out << "enable_multiple, ever_enabled_multiple: " << enable_multiple << ","
       << ever_enabled_multiple << std::endl;
   out << "default compat: " << default_compat << std::endl;
@@ -296,6 +299,7 @@ void FSMap::print_summary(Formatter *f, ostream *out) const
 {
   if (f) {
     f->dump_unsigned("epoch", get_epoch());
+    f->dump_string("btime", fmt::format("{}", btime));
     for (const auto& [fscid, fs] : filesystems) {
       f->dump_unsigned("id", fscid);
       f->dump_unsigned("up", fs.mds_map.up.size());
@@ -643,6 +647,7 @@ void FSMap::encode(bufferlist& bl, uint64_t features) const
   encode(standby_daemons, bl, features);
   encode(standby_epochs, bl);
   encode(ever_enabled_multiple, bl);
+  encode(btime, bl);
   ENCODE_FINISH(bl);
 }
 
@@ -674,6 +679,9 @@ void FSMap::decode(bufferlist::const_iterator& p)
   if (struct_v >= 7) {
     decode(ever_enabled_multiple, p);
   }
+  if (struct_v >= 8) {
+    decode(btime, p);
+  }
   DECODE_FINISH(p);
 }
 

src/mds/FSMap.h

@@ -26,6 +26,7 @@
 #include <errno.h>
 
 #include "include/types.h"
+#include "common/ceph_time.h"
 #include "common/Clock.h"
 #include "mds/MDSMap.h"
 
@@ -268,19 +269,21 @@ WRITE_CLASS_ENCODER_FEATURES(Filesystem)
 
 class FSMap {
 public:
+  using real_clock = ceph::real_clock;
   using mds_info_t = MDSMap::mds_info_t;
   using fsmap = typename std::map<fs_cluster_id_t, Filesystem>;
   using const_iterator = typename fsmap::const_iterator;
   using iterator = typename fsmap::iterator;
 
-  static const version_t STRUCT_VERSION = 7;
+  static const version_t STRUCT_VERSION = 8;
   static const version_t STRUCT_VERSION_TRIM_TO = 7;
 
   FSMap() : default_compat(MDSMap::get_compat_set_default()) {}
 
   FSMap(const FSMap &rhs)
     :
       epoch(rhs.epoch),
+      btime(rhs.btime),
       next_filesystem_id(rhs.next_filesystem_id),
       legacy_client_fscid(rhs.legacy_client_fscid),
       default_compat(rhs.default_compat),
@@ -584,6 +587,13 @@ class FSMap {
   epoch_t get_epoch() const { return epoch; }
   void inc_epoch() { epoch++; }
 
+  void set_btime() {
+    btime = real_clock::now();
+  }
+  auto get_btime() const {
+    return btime;
+  }
+
   version_t get_struct_version() const { return struct_version; }
   bool is_struct_old() const {
     return struct_version < STRUCT_VERSION_TRIM_TO;
@@ -676,6 +686,8 @@ class FSMap {
   }
 
   epoch_t epoch = 0;
+  ceph::real_time btime = real_clock::zero();
+
   uint64_t next_filesystem_id = FS_CLUSTER_ID_ANONYMOUS + 1;
   fs_cluster_id_t legacy_client_fscid = FS_CLUSTER_ID_NONE;
   CompatSet default_compat;

src/mon/MDSMonitor.cc

@@ -16,6 +16,7 @@
 #include <sstream>
 #include <queue>
 #include <ranges>
+#include <boost/range/adaptors.hpp>
 #include <boost/utility.hpp>
 
 #include "MDSMonitor.h"
@@ -238,6 +239,7 @@ void MDSMonitor::encode_pending(MonitorDBStore::TransactionRef t)
   if (!g_conf()->mon_mds_skip_sanity) {
     pending.sanity(true);
   }
+  pending.set_btime();
 
   // apply to paxos
   ceph_assert(get_last_committed() + 1 == pending.get_epoch());
@@ -1024,6 +1026,52 @@ bool MDSMonitor::preprocess_command(MonOpRequestRef op)
       ds << fsmap;
     }
     r = 0;
+  } else if (prefix == "mds last-seen") {
+    std::string id;
+    cmd_getval(cmdmap, "id", id);
+
+    dout(10) << "last seen check for " << id << dendl;
+
+    auto& history = get_fsmap_history();
+    auto now = real_clock::now();
+    bool found = false;
+    /* Special case:
+     * If the mons consider the MDS "in" the latest FSMap, then the mds
+     * is always "last seen" **now** (for the purposes of this API).  We
+     * don't look at past beacons because that is only managed by the
+     * leader and the logic is fudged in places in the event of suspected
+     * network partitions.
+     */
+    std::chrono::seconds since = std::chrono::seconds(0);
+
+    for (auto& [epoch, fsmaph] : boost::adaptors::reverse(history)) {
+      dout(25) << "looking at epoch " << epoch << dendl;
+      auto* info = fsmaph.find_by_name(id);
+      if (info) {
+        dout(10) << "found: " << *info << dendl;
+        found = true;
+        if (f) {
+          f->open_object_section("mds last-seen");
+          f->dump_object("info", *info);
+          f->dump_string("last-seen", fmt::format("{}", since));
+          f->dump_int("epoch", epoch);
+          f->close_section();
+          f->flush(ds);
+        } else {
+          ds << fmt::format("{}", since);
+        }
+        break;
+      }
+      /* If the MDS appears in the next epoch, then it went away as of this epoch's btime.
+       */
+      since = std::chrono::duration_cast<std::chrono::seconds>(now - fsmaph.get_btime());
+    }
+    if (found) {
+      r = 0;
+    } else {
+      ss << "mds " << id << " not found in recent FSMaps";
+      r = -ENOENT;
+    }
   } else if (prefix == "mds ok-to-stop") {
     vector<string> ids;
     if (!cmd_getval(cmdmap, "ids", ids)) {
@@ -2379,6 +2427,39 @@ bool MDSMonitor::maybe_promote_standby(FSMap &fsmap, const Filesystem& fs)
 
 void MDSMonitor::tick()
 {
+  {
+    auto _history_prune_time = g_conf().get_val<std::chrono::seconds>("mon_fsmap_prune_threshold");
+    set_fsmap_history_threshold(_history_prune_time);
+    dout(20) << _history_prune_time << dendl;
+    prune_fsmap_history();
+    auto& history = get_fsmap_history();
+    auto now = real_clock::now();
+    if (auto it = history.begin(); it != history.end()) {
+      auto start = it->second.get_epoch();
+      dout(20) << "oldest epoch in history is " << start << dendl;
+      for (;;) {
+        --start;
+        bufferlist bl;
+        FSMap fsmaph;
+        int err = get_version(start, bl);
+        if (err == -ENOENT) {
+          break;
+        }
+	ceph_assert(err == 0);
+	ceph_assert(bl.length());
+	fsmaph.decode(bl);
+        auto btime = fsmaph.get_btime();
+        auto since = std::chrono::duration_cast<std::chrono::milliseconds>(now - btime);
+        dout(20) << "loaded epoch " << fsmaph.get_epoch() << " which is " << since << " old" << dendl;
+        if (since <= _history_prune_time) {
+          put_fsmap_history(fsmaph);
+        } else {
+          break;
+        }
+      }
+    }
+  }
+
   if (!is_active() || !is_leader()) return;
 
   auto &pending = get_pending_fsmap_writeable();

src/mon/MonCommands.h

@@ -298,6 +298,9 @@ COMMAND("versions",
 
 #define FS_NAME_GOODCHARS "[A-Za-z0-9-_.]"
 COMMAND_WITH_FLAG("mds stat", "show MDS status", "mds", "r", FLAG(HIDDEN))
+COMMAND("mds last-seen name=id,type=CephString,req=true",
+	"fetch metadata for mds <id>",
+	"mds", "r")
 COMMAND("fs dump "
 	"name=epoch,type=CephInt,req=false,range=0",
 	"dump all CephFS status, optionally from epoch", "mds", "r")