Merge pull request ceph#65172 from shraddhaag/wip-shraddhaag-availability-frequency-config

mon: add config option to change availability score update interval

Author: SrinivasaBharath

PendingReleaseNotes

@@ -179,6 +179,14 @@
   five seconds and the feature is enabled by default. A new config option ``enable_availability_tracking``
   can be used to turn off the feature if required. Another command is added to clear the 
   availability status for a specific pool, ``ceph osd pool clear-availability-status <pool-name>``. 
+  users to view the availability score for each pool in a cluster. A pool is considered 
+  unavailable if any PG in the pool is not in active state or if there are unfound 
+  objects. Otherwise the pool is considered available. The score is updated every 
+  one second by default. This interval can be changed using the new config option
+  ``pool_availability_update_interval.``. The feature is on by default. A new config option 
+  ``enable_availability_tracking`` can be used to turn off the feature if required. 
+  Another command is added to clear the availability status for a specific pool, 
+  ``ceph osd pool clear-availability-status <pool-name>``. 
   This feature is in tech preview. 
   Related trackers:
    - https://tracker.ceph.com/issues/67777

doc/rados/configuration/mon-config-ref.rst

@@ -628,6 +628,7 @@ Miscellaneous
 .. confval:: mon_memory_target
 .. confval:: mon_memory_autotune
 .. confval:: enable_availability_tracking
+.. confval:: pool_availability_update_interval
 
 NVMe-oF Monitor Client
 ======================

doc/rados/operations/monitoring.rst

@@ -781,10 +781,19 @@ the Mean Time Between Failures (MTBF) and Mean Time To Recover (MTTR)
 for each pool. The availability score is then calculated by finding 
 the ratio of MTBF to the total time.  
 
-The score is updated every five seconds. This interval is currently 
-not configurable. Any intermittent changes to the pools that 
-occur between this duration but are reset before we recheck the pool 
-status will not be captured by this feature. 
+The score is updated every one second. Transient changes to pools that 
+occur and are reverted between successive updates will not be captured. 
+It is possible to configure this interval with a command of the following 
+form: 
+
+.. prompt:: bash $
+
+   ceph config set mon pool_availability_update_interval 2
+
+This will set the update interval to two seconds. Please note that 
+it is not possible to set this interval less than the config value set
+for ``paxos_propose_interval``. 
+
 
 This feature is on by default. To turn the feature off, e.g. - for an expected 
 downtime, the ``enable_availability_tracking`` config option can be set to ``false``. 

src/common/options/mon.yaml.in

@@ -1420,3 +1420,13 @@ options:
   default: 30
   services:
   - mon
+- name: pool_availability_update_interval
+  type: float
+  level: advanced
+  desc: Update data availability score at this interval. By default the interval 
+    is same as paxos_propose_interval configuration.  
+  default: 1
+  services :
+  - mon
+  flags:
+  - runtime

src/mon/MgrStatMonitor.cc

@@ -65,6 +65,7 @@ std::vector<std::string> MgrStatMonitor::get_tracked_keys() const noexcept
 {
   return {
     "enable_availability_tracking",
+    "pool_availability_update_interval",
   };
 }
 
@@ -94,6 +95,16 @@ void MgrStatMonitor::handle_conf_change(
     }
     enable_availability_tracking = newval;
   }
+
+  if (changed.count("pool_availability_update_interval")) {
+      std::scoped_lock l(lock);
+      dout(10) << __func__ << " pool_availability_update_interval config changed from " 
+             << pool_availability_update_interval << " to " 
+             << g_conf().get_val<double>("pool_availability_update_interval")
+             << dendl;
+      
+      pool_availability_update_interval = g_conf().get_val<double>("pool_availability_update_interval"); 
+  }
 }
 
 void MgrStatMonitor::create_initial()
@@ -120,6 +131,18 @@ void MgrStatMonitor::clear_pool_availability(int64_t poolid)
   dout(20) << __func__ << " cleared availability score for pool: " << poolid << dendl;
 }
 
+bool MgrStatMonitor::should_calc_pool_availability() 
+{
+  dout(20) << __func__ << dendl;
+  std::scoped_lock l(lock); 
+
+  utime_t now = ceph_clock_now();
+  if ((now  - pool_availability_last_updated) >= pool_availability_update_interval) {
+    return true; 
+  }
+  return false; 
+}
+
 void MgrStatMonitor::calc_pool_availability()
 {
   dout(20) << __func__ << dendl;
@@ -202,6 +225,7 @@ void MgrStatMonitor::calc_pool_availability()
 
   }
   pending_pool_availability = pool_availability;
+  pool_availability_last_updated = now; 
 }
 
 void MgrStatMonitor::update_from_paxos(bool *need_bootstrap)
@@ -239,7 +263,8 @@ void MgrStatMonitor::update_from_paxos(bool *need_bootstrap)
   mon.osdmon()->notify_new_pg_digest();
 
   // only calculate pool_availability within leader mon
-  if (mon.is_leader()) {
+  // and if configured interval has elapsed
+  if (mon.is_leader() && should_calc_pool_availability()) {
       calc_pool_availability();
   }
 }

src/mon/MgrStatMonitor.h

@@ -56,8 +56,11 @@
 
   void calc_pool_availability();
   bool enable_availability_tracking = g_conf().get_val<bool>("enable_availability_tracking"); ///< tracking availability score feature 
-  
+  double pool_availability_update_interval = g_conf().get_val<double>("pool_availability_update_interval");
+  utime_t pool_availability_last_updated = ceph_clock_now(); 
+
   void clear_pool_availability(int64_t poolid);
+  bool should_calc_pool_availability();
 
   void check_sub(Subscription *sub);
   void check_subs();