Merge pull request ceph#55103 from kamoltat/wip-ksirivad-fix-63861

src/mon/OSDMonitor.cc: [Stretch Mode] WRN non-existent CRUSH location assigned to MON

Reviewed-by: Ronen Friedman <[email protected]>
Reviewed-by: Greg Farnum <[email protected]>
Reviewed-by: Anthony D'Atri <[email protected]>

Author: yuriw

doc/rados/operations/health-checks.rst

@@ -1645,6 +1645,19 @@ We encourage you to fix this by making the weights even on both dividing buckets
 This can be done by making sure the combined weight of the OSDs on each dividing
 bucket are the same.
 
+NONEXISTENT_MON_CRUSH_LOC_STRETCH_MODE
+______________________________________
+
+The CRUSH location specified for the monitor must belong to one of the dividing
+buckets when stretch mode is enabled. With the ``tiebreaker`` monitor being the
+only exception.
+
+This warning suggests that one or more monitors have a CRUSH location that does
+not belong to any of the dividing buckets in stretch mode.
+
+We encourage you to fix this by making sure the CRUSH location of the monitor
+belongs to one of the dividing buckets.
+
 NVMeoF Gateway
 --------------
 

src/mon/HealthMonitor.cc

@@ -25,6 +25,7 @@
 
 #include "mon/Monitor.h"
 #include "mon/HealthMonitor.h"
+#include "mon/OSDMonitor.h"
 
 #include "messages/MMonHealthChecks.h"
 
@@ -740,6 +741,8 @@ bool HealthMonitor::check_leader_health()
   if (g_conf().get_val<bool>("mon_warn_on_msgr2_not_enabled")) {
     check_if_msgr2_enabled(&next);
   }
+  // STRETCH MODE
+  check_mon_crush_loc_stretch_mode(&next);
 
   if (next != leader_checks) {
     changed = true;
@@ -885,3 +888,33 @@ void HealthMonitor::check_if_msgr2_enabled(health_check_map_t *checks)
     }
   }
 }
+
+void HealthMonitor::check_mon_crush_loc_stretch_mode(health_check_map_t *checks)
+{
+  // Check if the CRUSH location exists for all MONs
+  if (!mon.monmap->stretch_mode_enabled){
+    return;
+  }
+  list<string> details;
+  for (auto& i : mon.monmap->mon_info) {
+    // Skip the tiebreaker monitor
+    if (i.second.name == mon.monmap->tiebreaker_mon) {
+      continue;
+    }
+    for (auto& pair : i.second.crush_loc){
+      if (!mon.osdmon()->osdmap.crush->name_exists(pair.second)) {
+        ostringstream ds;
+        ds << "CRUSH location " << pair.second << " does not exist";
+        details.push_back(ds.str());
+      }
+    }
+  }
+  // WARN in ceph -s if the CRUSH location does not exist
+  if (!details.empty()) {
+    ostringstream ss;
+    ss << details.size() << " monitor(s) have nonexistent CRUSH location";
+    auto &d = checks->add("NONEXISTENT_MON_CRUSH_LOC_STRETCH_MODE", HEALTH_WARN, ss.str(),
+                details.size());
+    d.detail.swap(details);
+  }
+}

src/mon/HealthMonitor.h

@@ -66,6 +66,7 @@ class HealthMonitor : public PaxosService
   void check_for_older_version(health_check_map_t *checks);
   void check_for_mon_down(health_check_map_t *checks);
   void check_for_clock_skew(health_check_map_t *checks);
+  void check_mon_crush_loc_stretch_mode(health_check_map_t *checks);
   void check_if_msgr2_enabled(health_check_map_t *checks);
   bool check_leader_health();
   bool check_member_health();

src/mon/OSDMonitor.cc

@@ -15270,7 +15270,10 @@ bool OSDMonitor::check_for_dead_crush_zones(const map<string,set<string>>& dead_
   bool really_down = false;
   for (auto dbi : dead_buckets) {
     const string& bucket_name = dbi.first;
-    ceph_assert(osdmap.crush->name_exists(bucket_name));
+    if (!osdmap.crush->name_exists(bucket_name)) {
+      dout(10) << "CRUSH bucket " << bucket_name << " does not exist" << dendl;
+      continue;
+    }
     int bucket_id = osdmap.crush->get_item_id(bucket_name);
     dout(20) << "Checking " << bucket_name << " id " << bucket_id
 	     << " to see if OSDs are also down" << dendl;