MB-41760: Add disk-failures stats group

Provide ns_server with disk failure stats; can be fetched frequently
with minimal overhead.

$ cbstats -u ... -p ... localhost:12000 disk-failures
 ep_data_read_failed:  0
 ep_data_write_failed: 0

Change-Id: I90eabc27ccce03fcd2057cb1ca2353bd030b3e5f
Reviewed-on: http://review.couchbase.org/c/kv_engine/+/138633
Tested-by: Build Bot <[email protected]>
Reviewed-by: Richard de Mellow <[email protected]>

Author: jameseh96

engines/ep/management/cbstats

@@ -794,6 +794,10 @@ def stats_diskinfo(mc, with_detail=None):
 
     stats_formatter(stats_perform(mc, cmd_str))
 
+@cmd
+def stats_diskfailures(mc):
+    stats_formatter(stats_perform(mc, "disk-failures"))
+
 @cmd
 def stats_eviction(mc):
     if output_json:
@@ -949,6 +953,7 @@ def main():
     c.addCommand('checkpoint', stats_checkpoint, 'checkpoint [vbid]')
     c.addCommand('config', stats_config, 'config')
     c.addCommand('diskinfo', stats_diskinfo, 'diskinfo [detail]')
+    c.addCommand('disk-failures', stats_diskfailures, 'disk-failures')
     c.addCommand('durability-monitor', stats_durability_monitor, 'durability-monitor [vbid]')
     c.addCommand('eviction', stats_eviction, 'eviction')
     c.addCommand('scheduler', stats_scheduler, 'scheduler')

engines/ep/src/ep_engine.cc

@@ -2965,20 +2965,12 @@ ENGINE_ERROR_CODE EventuallyPersistentEngine::doEngineStats(
     collector.addStat(Key::ep_cursors_dropped, epstats.cursorsDropped);
     collector.addStat(Key::ep_cursor_memory_freed, epstats.cursorMemoryFreed);
 
+    doDiskFailureStats(collector);
+
     // Note: These are also reported per-shard in 'kvstore' stats, however
     // we want to be able to graph these over time, and hence need to expose
     // to ns_sever at the top-level.
     size_t value = 0;
-    if (kvBucket->getKVStoreStat("failure_compaction", value,
-                                 KVBucketIface::KVSOption::BOTH)) {
-        // Total data write failures is compaction failures plus commit failures
-        auto writeFailure = value + epstats.commitFailed;
-        collector.addStat(Key::ep_data_write_failed, writeFailure);
-    }
-    if (kvBucket->getKVStoreStat("failure_get", value,
-                                 KVBucketIface::KVSOption::BOTH)) {
-        collector.addStat(Key::ep_data_read_failed, value);
-    }
     if (kvBucket->getKVStoreStat("io_document_write_bytes",
                                  value,
                                  KVBucketIface::KVSOption::RW)) {
@@ -4513,6 +4505,22 @@ ENGINE_ERROR_CODE EventuallyPersistentEngine::doDiskinfoStats(
     return ENGINE_EINVAL;
 }
 
+void EventuallyPersistentEngine::doDiskFailureStats(
+        BucketStatCollector& collector) {
+    using namespace cb::stats;
+    size_t value = 0;
+    if (kvBucket->getKVStoreStat(
+                "failure_compaction", value, KVBucketIface::KVSOption::BOTH)) {
+        // Total data write failures is compaction failures plus commit failures
+        auto writeFailure = value + stats.commitFailed;
+        collector.addStat(Key::ep_data_write_failed, writeFailure);
+    }
+    if (kvBucket->getKVStoreStat(
+                "failure_get", value, KVBucketIface::KVSOption::BOTH)) {
+        collector.addStat(Key::ep_data_read_failed, value);
+    }
+}
+
 ENGINE_ERROR_CODE EventuallyPersistentEngine::doPrivilegedStats(
         const void* cookie, const AddStatFn& add_stat, std::string_view key) {
     // Privileged stats - need Stats priv (and not just SimpleStats).
@@ -4692,6 +4700,10 @@ ENGINE_ERROR_CODE EventuallyPersistentEngine::getStats(
         return doScopeStats(
                 cookie, add_stat, std::string(key.data(), key.size()));
     }
+    if (cb_isPrefix(key, "disk-failures")) {
+        doDiskFailureStats(bucketCollector);
+        return ENGINE_SUCCESS;
+    }
     if (key[0] == '_') {
         return doPrivilegedStats(cookie, add_stat, key);
     }

engines/ep/src/ep_engine.h

@@ -1010,6 +1010,8 @@ class EventuallyPersistentEngine : public EngineIface, public DcpIface {
                                       const AddStatFn& add_stat,
                                       std::string_view statKey);
 
+    void doDiskFailureStats(BucketStatCollector& collector);
+
     ENGINE_ERROR_CODE doPrivilegedStats(const void* cookie,
                                         const AddStatFn& add_stat,
                                         std::string_view statKey);