Merge pull request ceph#64045 from shraddhaag/wip-71743-tentacle

tentacle: mon: add config option to toggle availability score feature

Author: shraddhaag

PendingReleaseNotes

@@ -151,7 +151,8 @@
   users to view the availability score for each pool in a cluster. A pool is considered 
   unavailable if any PG in the pool is not in active state or if there are unfound 
   objects. Otherwise the pool is considered available. The score is updated every 
-  5 seconds. This feature is in tech preview. 
+  5 seconds. The feature is on by default. A new config option `enable_availability_tracking`
+  can be used to turn off the feature if required. This feature is in tech preview. 
   Related trackers:
    - https://tracker.ceph.com/issues/67777
 

doc/rados/configuration/mon-config-ref.rst

@@ -627,6 +627,7 @@ Miscellaneous
 .. confval:: mon_osd_cache_size_min
 .. confval:: mon_memory_target
 .. confval:: mon_memory_autotune
+.. confval:: enable_availability_tracking
 
 .. _Paxos: https://en.wikipedia.org/wiki/Paxos_(computer_science)
 .. _Monitor Keyrings: ../../../dev/mon-bootstrap#secret-keys

doc/rados/operations/monitoring.rst

@@ -784,4 +784,14 @@ the ratio of MTBF to the total time.
 The score is updated every five seconds. This interval is currently 
 not configurable. Any intermittent changes to the pools that 
 occur between this duration but are reset before we recheck the pool 
-status will not be captured by this feature. 
+status will not be captured by this feature. 
+
+This feature is on by default. To turn the feature off, e.g. - for an expected 
+downtime, the ``enable_availability_tracking`` config option can be set to ``false``. 
+
+.. prompt:: bash $
+
+   ceph config set mon enable_availability_tracking false
+
+While the feature is turned off, the last calculated score will be preserved. The 
+score will again start updating once the feature is turned on again. 

qa/standalone/mon/availability.sh

@@ -57,8 +57,30 @@ function TEST_availablity_score() {
     AVAILABILITY_STATUS=$(ceph osd pool availability-status | grep -w "foo")
     SCORE=$(echo "$AVAILABILITY_STATUS" | awk '{print $7}')
     IS_AVAILABLE=$(echo "$AVAILABILITY_STATUS" | awk '{print $8}')
+    UPTIME_DURATION=$(echo "$AVAILABILITY_STATUS" | awk '{print $2}')
+    UPTIME_SECONDS=$(( ${UPTIME_DURATION%[sm]} * (${UPTIME_DURATION: -1} == "m" ? 60 : 1) ))
     if [ $IS_AVAILABLE -ne 1 ]; then
       echo "Failed: Pool is not available in availabilty status"
+      return 1
+    fi
+
+    # unset config option enable_availability_tracking to disable feature 
+    ceph config set mon enable_availability_tracking false 
+    AVAILABILITY_STATUS=$(ceph osd pool availability-status | grep -w "foo")
+    if [ "$AVAILABILITY_STATUS" != "" ]; then
+      echo "Failed: feature not disabled successfully."
+      return 1
+    fi
+    sleep 120
+
+    # enable feature and check is score updated when it was off
+    ceph config set mon enable_availability_tracking true 
+    AVAILABILITY_STATUS=$(ceph osd pool availability-status | grep -w "foo")
+    UPTIME_DURATION=$(echo "$AVAILABILITY_STATUS" | awk '{print $2}')
+    NEW_UPTIME_SECONDS=$(( ${UPTIME_DURATION%[sm]} * (${UPTIME_DURATION: -1} == "m" ? 60 : 1) ))
+    if [ "$NEW_UPTIME_SECONDS" -gt $((UPTIME_SECONDS + 120)) ]; then
+      echo "Failed: score is updated even when feature is disabled"
+      return 1
     fi
 
     # write some objects

src/common/options/mon.yaml.in

@@ -1395,3 +1395,14 @@ options:
   default: 2
   services:
   - mon
+- name: enable_availability_tracking
+  type: bool
+  level: advanced
+  desc: Calculate and store availablity score for each pool in the 
+    cluster at regular intervals
+  default: true
+  services :
+  - mon
+  flags:
+  - runtime
+ 

src/mon/MgrStatMonitor.cc

@@ -52,9 +52,44 @@ static ostream& _prefix(std::ostream *_dout, Monitor &mon) {
 MgrStatMonitor::MgrStatMonitor(Monitor &mn, Paxos &p, const string& service_name)
   : PaxosService(mn, p, service_name)
 {
+    g_conf().add_observer(this);
 }
 
-MgrStatMonitor::~MgrStatMonitor() = default;
+MgrStatMonitor::~MgrStatMonitor() 
+{
+  g_conf().remove_observer(this);
+}
+
+std::vector<std::string> MgrStatMonitor::get_tracked_keys() const noexcept
+{
+  return {
+    "enable_availability_tracking",
+  };
+}
+
+void MgrStatMonitor::handle_conf_change(
+  const ConfigProxy& conf,
+  const std::set<std::string>& changed)
+{
+  if (changed.count("enable_availability_tracking")) {
+    std::scoped_lock l(lock);
+    bool oldval = enable_availability_tracking;
+    bool newval = g_conf().get_val<bool>("enable_availability_tracking");
+    dout(10) << __func__ << " enable_availability_tracking config option is changed from " 
+             << oldval << " to " << newval
+             << dendl;
+
+    // if fetaure is toggled from off to on, 
+    // store the new value of last_uptime and last_downtime 
+    // (to be updated in calc_pool_availability) 
+    if (newval > oldval) {
+      reset_availability_last_uptime_downtime_val = ceph_clock_now();
+      dout(10) << __func__ << " reset_availability_last_uptime_downtime_val " 
+               <<  reset_availability_last_uptime_downtime_val << dendl; 
+    }
+    enable_availability_tracking = newval;
+  }
+}
 
 void MgrStatMonitor::create_initial()
 {
@@ -69,6 +104,29 @@ void MgrStatMonitor::create_initial()
 void MgrStatMonitor::calc_pool_availability()
 {
   dout(20) << __func__ << dendl;
+  std::scoped_lock l(lock);
+
+  // if feature is disabled by user, do not update the uptime 
+  // and downtime, exit early
+  if (!enable_availability_tracking) {
+    dout(20) << __func__ << " tracking availability score is disabled" << dendl;
+    return;
+  }
+
+  // if reset_availability_last_uptime_downtime_val is not utime_t(1, 2), 
+  // update last_uptime and last_downtime for all pools to the 
+  // recorded values
+  if (reset_availability_last_uptime_downtime_val.has_value()) {
+    for (const auto& i : pool_availability) {
+      const auto& poolid = i.first;
+      pool_availability[poolid].last_downtime = reset_availability_last_uptime_downtime_val.value();
+      pool_availability[poolid].last_uptime = reset_availability_last_uptime_downtime_val.value();
+    }
+    dout(20) << __func__ << " reset last_uptime and last_downtime to " 
+             << reset_availability_last_uptime_downtime_val << dendl;
+    reset_availability_last_uptime_downtime_val.reset();
+  }
+
   auto pool_avail_end = pool_availability.end();
   for (const auto& i : digest.pool_pg_unavailable_map) {
     const auto& poolid = i.first;

src/mon/MgrStatMonitor.h

@@ -8,7 +8,8 @@
 #include "mon/PGMap.h"
 #include "mgr/ServiceMap.h"
 
-class MgrStatMonitor : public PaxosService {
+ class MgrStatMonitor : public PaxosService, 
+                        public md_config_obs_t {
   // live version
   version_t version = 0;
   PGMapDigest digest;
@@ -27,6 +28,8 @@ class MgrStatMonitor : public PaxosService {
   MgrStatMonitor(Monitor &mn, Paxos &p, const std::string& service_name);
   ~MgrStatMonitor() override;
 
+  ceph::mutex lock = ceph::make_mutex("MgrStatMonitor::lock");
+
   void init() override {}
   void on_shutdown() override {}
 
@@ -52,7 +55,9 @@ class MgrStatMonitor : public PaxosService {
   bool preprocess_statfs(MonOpRequestRef op);
 
   void calc_pool_availability();
-
+  bool enable_availability_tracking = g_conf().get_val<bool>("enable_availability_tracking"); ///< tracking availability score feature 
+  std::optional<utime_t> reset_availability_last_uptime_downtime_val;
+  
   void check_sub(Subscription *sub);
   void check_subs();
   void send_digests();
@@ -114,4 +119,9 @@ class MgrStatMonitor : public PaxosService {
 		       bool verbose) const {
     digest.dump_pool_stats_full(osdm, ss, f, verbose);
   }
+
+  // config observer 
+  std::vector<std::string> get_tracked_keys() const noexcept override;
+  void handle_conf_change(const ConfigProxy& conf,
+                          const std::set <std::string> &changed) override;
 };

src/mon/OSDMonitor.cc

@@ -14410,6 +14410,11 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
 						   get_last_committed() + 1));
     return true;
   } else if (prefix == "osd pool availability-status") {
+    if (!g_conf().get_val<bool>("enable_availability_tracking")) {
+      ss << "availability tracking is disabled; you can enable it by setting the config option enable_availability_tracking";
+      err = -EOPNOTSUPP;
+      goto reply_no_propose;
+    }
     TextTable tbl;
     tbl.define_column("POOL", TextTable::LEFT, TextTable::LEFT);
     tbl.define_column("UPTIME", TextTable::LEFT, TextTable::RIGHT);