Merge pull request ceph#63552 from shraddhaag/wip-shraddhaag-availability-3

mon: add command osd pool clear-availability-status

Author: SrinivasaBharath

PendingReleaseNotes

@@ -147,12 +147,14 @@
   `s3:GetObjectRetention` are also considered when fetching the source object.
   Replication of tags is controlled by the `s3:GetObject(Version)Tagging` permission.
 
-* RADOS: A new command, `ceph osd pool availability-status`, has been added that allows
+* RADOS: A new command, ``ceph osd pool availability-status``, has been added that allows
   users to view the availability score for each pool in a cluster. A pool is considered 
   unavailable if any PG in the pool is not in active state or if there are unfound 
   objects. Otherwise the pool is considered available. The score is updated every 
-  5 seconds. The feature is on by default. A new config option `enable_availability_tracking`
-  can be used to turn off the feature if required. This feature is in tech preview. 
+  5 seconds. The feature is on by default. A new config option ``enable_availability_tracking``
+  can be used to turn off the feature if required. Another command is added to clear the 
+  availability status for a specific pool, ``ceph osd pool clear-availability-status <pool-name>``. 
+  This feature is in tech preview. 
   Related trackers:
    - https://tracker.ceph.com/issues/67777
 

doc/rados/operations/monitoring.rst

@@ -794,4 +794,13 @@ downtime, the ``enable_availability_tracking`` config option can be set to ``fal
    ceph config set mon enable_availability_tracking false
 
 While the feature is turned off, the last calculated score will be preserved. The 
-score will again start updating once the feature is turned on again. 
+score will again start updating once the feature is turned on again. 
+
+It's also possible to clear the data availability score for a specific 
+pool if needed with a command of the following form:
+
+.. prompt:: bash $
+
+   ceph osd pool clear-availability-status <pool-name>
+
+Note: Clearing a score is not allowed if the feature itself is disabled. 

qa/standalone/mon/availability.sh

@@ -73,6 +73,13 @@ function TEST_availablity_score() {
     fi
     sleep 120
 
+    # try clearing availability score: should fail as feature is disabled 
+    CLEAR_SCORE_RESPONSE=$(ceph osd pool clear-availability-status foo)
+    if [ "$CLEAR_SCORE_RESPONSE" != "" ]; then
+      echo "Failed: score clear attempted when feature is disabled"
+      return 1
+    fi
+
     # enable feature and check is score updated when it was off
     ceph config set mon enable_availability_tracking true 
     AVAILABILITY_STATUS=$(ceph osd pool availability-status | grep -w "foo")
@@ -125,7 +132,19 @@ function TEST_availablity_score() {
       echo "Failed: Availability score for the pool did not drop"
       return 1
     fi
+    UPTIME_DURATION=$(echo "$AVAILABILITY_STATUS" | awk '{print $2}')
+    UPTIME_SECONDS=$(( ${UPTIME_DURATION%[sm]} * (${UPTIME_DURATION: -1} == "m" ? 60 : 1) ))
 
+    # reset availability score for pool foo 
+    ceph osd pool clear-availability-status foo
+    AVAILABILITY_STATUS=$(ceph osd pool availability-status | grep -w "foo")
+    NEW_UPTIME_DURATION=$(echo "$AVAILABILITY_STATUS" | awk '{print $2}')
+    NEW_UPTIME_SECONDS=$(( ${UPTIME_DURATION%[sm]} * (${UPTIME_DURATION: -1} == "m" ? 60 : 1) ))
+    if [ "$NEW_UPTIME_SECONDS" -gt "$UPTIME_SECONDS" ]; then
+      echo "Failed: Availability score for the pool did not drop after clearing"
+      return 1
+    fi
+  
     echo "TEST PASSED"
     return 0
 }

src/mon/MgrStatMonitor.cc

@@ -101,6 +101,20 @@ void MgrStatMonitor::create_initial()
   encode(service_map, pending_service_map_bl, CEPH_FEATURES_ALL);
 }
 
+void MgrStatMonitor::clear_pool_availability(int64_t poolid)
+{
+  dout(20) << __func__ << dendl;
+  std::scoped_lock l(lock);
+  auto pool_itr = pending_pool_availability.find(poolid);
+  if (pool_itr != pending_pool_availability.end()) {
+    pool_itr->second = PoolAvailability();
+  } else {
+    dout(1) << "failed to clear a non-existing pool: " << poolid << dendl;
+    return; 
+  };
+  dout(20) << __func__ << " cleared availability score for pool: " << poolid << dendl;
+}
+
 void MgrStatMonitor::calc_pool_availability()
 {
   dout(20) << __func__ << dendl;
@@ -398,6 +412,7 @@ bool MgrStatMonitor::prepare_report(MonOpRequestRef op)
   dout(20) << "pool_availability:\n";
   JSONFormatter jf(true);
   jf.open_object_section("pool_availability");
+  std::scoped_lock l(lock);
   for (auto& i : pending_pool_availability) {
     jf.dump_object(std::to_string(i.first), i.second);
   }

src/mon/MgrStatMonitor.h

@@ -58,6 +58,8 @@
   bool enable_availability_tracking = g_conf().get_val<bool>("enable_availability_tracking"); ///< tracking availability score feature 
   std::optional<utime_t> reset_availability_last_uptime_downtime_val;
 
+  void clear_pool_availability(int64_t poolid);
+
   void check_sub(Subscription *sub);
   void check_subs();
   void send_digests();

src/mon/MonCommands.h

@@ -1246,6 +1246,10 @@ COMMAND("osd pool stretch unset "
 COMMAND("osd pool availability-status", \
         "obtain availability stats from all pools", \
         "osd", "r")
+COMMAND("osd pool clear-availability-status "
+		"name=pool,type=CephPoolname ",
+        "clear a pool's existing availability stats", 
+        "osd", "r")
 COMMAND("osd utilization",
 	"get basic pg distribution stats",
 	"osd", "r")

src/mon/OSDMonitor.cc

@@ -14406,6 +14406,33 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
     wait_for_commit(op, new Monitor::C_Command(mon, op, 0, rs,
 						   get_last_committed() + 1));
     return true;
+  } else if (prefix == "osd pool clear-availability-status") {
+    if (!g_conf().get_val<bool>("enable_availability_tracking")) {
+      ss << "Availability tracking is disabled. Availability status can not be cleared "
+      << "while the feature is disabled. Enable it by setting the config "
+      << "option enable_availability_tracking to ``true`` then try again.";
+      err = -EOPNOTSUPP;
+      goto reply_no_propose;
+    }
+
+    string pool_name;
+    cmd_getval(cmdmap, "pool", pool_name);
+    int64_t pool_id = osdmap.lookup_pg_pool_name(pool_name);
+    // check if pool exists
+    if (pool_id < 0) {
+      ss << "unrecognized pool '" << pool_name << "'";
+      err = -ENOENT;
+      goto reply_no_propose;
+    }
+    std::map<uint64_t, PoolAvailability> pool_availability = mon.mgrstatmon()->get_pool_availability();
+    // check if pool exists in pool_availability
+    if (pool_availability.find(pool_id) == pool_availability.end()){
+      ss << "unrecognized pool '" << pool_name << "'";
+      err = -ENOENT;
+      goto reply_no_propose;
+    }
+    // clear existing calculations
+    mon.mgrstatmon()->clear_pool_availability(pool_id);
   } else if (prefix == "osd pool availability-status") {
     if (!g_conf().get_val<bool>("enable_availability_tracking")) {
       ss << "availability tracking is disabled; you can enable it by setting the config option enable_availability_tracking";