feat: prometheus metrics per database (#5770)

* feat: prometheus metrics per database

---------
Co-authored-by: Roman Gershman <[email protected]>

Author: qlimenoque

src/server/db_slice.cc

@@ -580,6 +580,7 @@ auto DbSlice::FindInternal(const Context& cntx, string_view key, optional<unsign
 
   if (!IsValid(res.it)) {
     events_.misses += miss_weight;
+    db.stats.events.misses += miss_weight;
     return OpStatus::KEY_NOTFOUND;
   }
 
@@ -589,13 +590,15 @@ auto DbSlice::FindInternal(const Context& cntx, string_view key, optional<unsign
 
   if (req_obj_type.has_value() && res.it->second.ObjType() != req_obj_type.value()) {
     events_.misses += miss_weight;
+    db.stats.events.misses += miss_weight;
     return OpStatus::WRONG_TYPE;
   }
 
   if (res.it->second.HasExpire()) {  // check expiry state
     res = ExpireIfNeeded(cntx, res.it);
     if (!IsValid(res.it)) {
       events_.misses += miss_weight;
+      db.stats.events.misses += miss_weight;
       return OpStatus::KEY_NOTFOUND;
     }
   }
@@ -612,6 +615,7 @@ auto DbSlice::FindInternal(const Context& cntx, string_view key, optional<unsign
       break;
     case UpdateStatsMode::kReadStats:
       events_.hits++;
+      db.stats.events.hits++;
       if (db.slots_stats) {
         db.slots_stats[KeySlot(key)].total_reads++;
       }
@@ -778,6 +782,7 @@ OpResult<DbSlice::ItAndUpdater> DbSlice::AddOrFindInternal(const Context& cntx,
   events_.garbage_collected = db.prime.garbage_collected();
   events_.stash_unloaded = db.prime.stash_unloaded();
   events_.evicted_keys += evp.evicted();
+  db.stats.events.evicted_keys += evp.evicted();
   events_.garbage_checked += evp.checked();
   if (db.slots_stats) {
     SlotId sid = KeySlot(key);
@@ -1264,6 +1269,7 @@ DbSlice::PrimeItAndExp DbSlice::ExpireIfNeeded(const Context& cntx, PrimeIterato
       ExpIterator(expire_it, StringOrView::FromView(key)), db.get());
 
   ++events_.expired_keys;
+  db->stats.events.expired_keys++;
 
   return {PrimeIterator{}, ExpireIterator{}};
 }
@@ -1493,6 +1499,7 @@ pair<uint64_t, size_t> DbSlice::FreeMemWithEvictionStepAtomic(DbIndex db_ind,
   SendQueuedInvalidationMessagesAsync();
   auto time_finish = absl::GetCurrentTimeNanos();
   events_.evicted_keys += evicted_items;
+  db_arr_[db_ind]->stats.events.evicted_keys += evicted_items;
   DVLOG(2) << "Eviction time (us): " << (time_finish - time_start) / 1000;
   return pair<uint64_t, size_t>{evicted_items, evicted_bytes};
 }
@@ -1582,6 +1589,11 @@ void DbSlice::ResetUpdateEvents() {
 
 void DbSlice::ResetEvents() {
   events_ = {};
+  for (auto& db : db_arr_) {
+    if (db) {
+      db->stats.events = {};
+    }
+  }
 }
 
 void DbSlice::SetNotifyKeyspaceEvents(std::string_view notify_keyspace_events) {

src/server/multi_test.cc

@@ -150,6 +150,60 @@ TEST_F(MultiTest, HitMissStats) {
   EXPECT_THAT(metrics.events.misses, 1);
 }
 
+TEST_F(MultiTest, PerDbHitMissStats) {
+  Run({"SELECT", "0"});
+  ASSERT_EQ(Run({"SET", "key1", "val1"}), "OK");
+  ASSERT_EQ(Run({"GET", "key1"}), "val1");
+  ASSERT_THAT(Run({"GET", "nonexistent1"}), ArgType(RespExpr::NIL));
+
+  Run({"SELECT", "1"});
+  ASSERT_EQ(Run({"SET", "key2", "val2"}), "OK");
+  ASSERT_EQ(Run({"GET", "key2"}), "val2");
+  ASSERT_THAT(Run({"GET", "nonexistent2"}), ArgType(RespExpr::NIL));
+
+  auto metrics = GetMetrics();
+
+  EXPECT_GE(metrics.db_stats.size(), 2u);
+  EXPECT_EQ(metrics.db_stats[0].events.hits, 1u);
+  EXPECT_EQ(metrics.db_stats[0].events.misses, 1u);
+  EXPECT_EQ(metrics.db_stats[1].events.hits, 1u);
+  EXPECT_EQ(metrics.db_stats[1].events.misses, 1u);
+
+  EXPECT_EQ(metrics.events.hits, 2u);
+  EXPECT_EQ(metrics.events.misses, 2u);
+}
+
+TEST_F(MultiTest, PerDbHitMissStatsReset) {
+  Run({"SELECT", "0"});
+  Run({"SET", "key1", "val1"});
+  Run({"GET", "key1"});
+  Run({"GET", "key2"});
+
+  auto before = GetMetrics();
+  ASSERT_GT(before.db_stats[0].events.hits, 0u);
+  ASSERT_GT(before.db_stats[0].events.misses, 0u);
+
+  EXPECT_EQ("OK", Run({"CONFIG", "RESETSTAT"}));
+
+  auto after = GetMetrics();
+  EXPECT_EQ(after.db_stats[0].events.hits, 0u);
+  EXPECT_EQ(after.db_stats[0].events.misses, 0u);
+}
+
+TEST_F(MultiTest, PerDbHitMissInfoOutput) {
+  Run({"SELECT", "0"});
+  Run({"SET", "testkey", "testval"});
+  Run({"GET", "testkey"});
+  Run({"GET", "missing"});
+
+  auto info_resp = Run({"INFO", "keyspace"});
+  ASSERT_TRUE(info_resp.type == RespExpr::STRING);
+  string info_str = info_resp.GetString();
+  EXPECT_THAT(info_str, HasSubstr("hits=1"));
+  EXPECT_THAT(info_str, HasSubstr("misses=1"));
+  EXPECT_THAT(info_str, HasSubstr("hit_ratio=50.00"));
+}
+
 TEST_F(MultiTest, MultiEmpty) {
   RespExpr resp = Run({"multi"});
   ASSERT_EQ(resp, "OK");

src/server/server_family.cc

@@ -1724,6 +1724,22 @@ void PrintPrometheusMetrics(uint64_t uptime, const Metrics& m, DflyCmd* dfly_cmd
                             &resp->body());
   AppendMetricWithoutLabels("evicted_keys_total", "", m.events.evicted_keys, MetricType::COUNTER,
                             &resp->body());
+  // Per-DB expired/evicted totals
+  {
+    string perdb_str;
+    AppendMetricHeader("expired_keys_total", "", MetricType::COUNTER, &perdb_str);
+    AppendMetricHeader("evicted_keys_total", "", MetricType::COUNTER, &perdb_str);
+    for (size_t i = 0; i < m.db_stats.size(); ++i) {
+      const auto& s = m.db_stats[i];
+      if (s.events.expired_keys > 0)
+        AppendMetricValue("expired_keys_total", s.events.expired_keys, {"db"}, {StrCat("db", i)},
+                          &perdb_str);
+      if (s.events.evicted_keys > 0)
+        AppendMetricValue("evicted_keys_total", s.events.evicted_keys, {"db"}, {StrCat("db", i)},
+                          &perdb_str);
+    }
+    absl::StrAppend(&resp->body(), perdb_str);
+  }
   // Memory stats
   if (legacy) {
     AppendMetricWithoutLabels("memory_fiberstack_vms_bytes",
@@ -1915,6 +1931,11 @@ void PrintPrometheusMetrics(uint64_t uptime, const Metrics& m, DflyCmd* dfly_cmd
 
     AppendMetricValue("db_keys_expiring", m.db_stats[i].expire_count, {"db"}, {StrCat("db", i)},
                       &db_key_expire_metrics);
+
+    AppendMetricValue("keyspace_hits_total", m.db_stats[i].events.hits, {"db"}, {StrCat("db", i)},
+                      &resp->body());
+    AppendMetricValue("keyspace_misses_total", m.db_stats[i].events.misses, {"db"},
+                      {StrCat("db", i)}, &resp->body());
   }
 
   absl::StrAppend(&resp->body(), db_key_metrics, db_key_expire_metrics, db_capacity_metrics,
@@ -2644,7 +2665,8 @@ void ServerFamily::ResetStat(Namespace* ns) {
         registry->ResetCallStats(index);
         EngineShard* shard = EngineShard::tlocal();
         if (shard) {
-          ns->GetDbSlice(shard->shard_id()).ResetEvents();
+          auto& db_slice = ns->GetDbSlice(shard->shard_id());
+          db_slice.ResetEvents();
         }
         facade::ResetStats();
         ServerState::tlocal()->exec_freq_count.clear();
@@ -3244,7 +3266,12 @@ string ServerFamily::FormatInfoMetrics(const Metrics& m, std::string_view sectio
       const auto& stats = m.db_stats[i];
       bool show = (i == 0) || (stats.key_count > 0);
       if (show) {
+        size_t total = stats.events.hits + stats.events.misses;
+        double hit_ratio =
+            (total > 0) ? static_cast<double>(stats.events.hits) / (total)*100.0 : 0.0;
         string val = StrCat("keys=", stats.key_count, ",expires=", stats.expire_count,
+                            ",hits=", stats.events.hits, ",misses=", stats.events.misses,
+                            ",hit_ratio=", absl::StrFormat("%.2f", hit_ratio),
                             ",avg_ttl=-1");  // TODO
         append(StrCat("db", i), val);
       }

src/server/table.cc

@@ -30,12 +30,16 @@ void DbTableStats::AddTypeMemoryUsage(unsigned type, int64_t delta) {
 
 DbTableStats& DbTableStats::operator+=(const DbTableStats& o) {
   constexpr size_t kDbSz = sizeof(DbTableStats) - sizeof(memory_usage_by_type);
-  static_assert(kDbSz == 32);
+  static_assert(kDbSz == 64);
 
   ADD(inline_keys);
   ADD(obj_memory_usage);
   ADD(tiered_entries);
   ADD(tiered_used_bytes);
+  ADD(events.hits);
+  ADD(events.misses);
+  ADD(events.expired_keys);
+  ADD(events.evicted_keys);
 
   for (size_t i = 0; i < o.memory_usage_by_type.size(); ++i) {
     memory_usage_by_type[i] += o.memory_usage_by_type[i];

src/server/table.h

@@ -70,6 +70,16 @@ struct DbTableStats {
   size_t tiered_entries = 0;
   size_t tiered_used_bytes = 0;
 
+  struct {
+    // Per-database hits/misses on keys
+    size_t hits = 0;
+    size_t misses = 0;
+
+    // Per-database expired/evicted keys
+    size_t expired_keys = 0;
+    size_t evicted_keys = 0;
+  } events;
+
   std::array<size_t, OBJ_TYPE_MAX> memory_usage_by_type = {};
 
   // Mostly used internally, exposed for tiered storage.