MB-59260: BP:58199: Add max_[user/system]_connections to stats

Add the max setting for user and system connections to
prometheus to allow for monitoring the current count with
the max limit.

Change-Id: Ia962e6445dd05c79e1fbae537df5548c06de780c
Reviewed-on: https://review.couchbase.org/c/kv_engine/+/199509
Well-Formed: Restriction Checker
Reviewed-by: Paolo Cocchi <[email protected]>
Tested-by: Trond Norbye <[email protected]>

Author: trondn

daemon/stats.cc

@@ -41,12 +41,17 @@ static void server_global_stats(const StatCollector& collector) {
     collector.addStat(Key::memcached_version, MEMCACHED_VERSION);
 
     collector.addStat(Key::daemon_connections, ServerSocket::getNumInstances());
-    collector.addStat(Key::curr_connections,
-                      stats.curr_conns.load(std::memory_order_relaxed));
-    collector.addStat(Key::system_connections,
-                      stats.system_conns.load(std::memory_order_relaxed));
+    const auto curr = stats.curr_conns.load();
+    const auto sys = stats.system_conns.load();
+    collector.addStat(Key::curr_connections, curr);
+    collector.addStat(Key::system_connections, sys);
+    collector.addStat(Key::user_connections, curr>sys ? curr - sys : 0);
     collector.addStat(Key::total_connections, stats.total_conns);
     collector.addStat(Key::connection_structures, stats.conn_structs);
+    collector.addStat(Key::max_user_connections,
+                      Settings::instance().getMaxUserConnections());
+    collector.addStat(Key::max_system_connections,
+                      Settings::instance().getSystemConnections());
 
     auto pool_size = Settings::instance().getFreeConnectionPoolSize();
     if (pool_size != 0) {

include/statistics/stats.def.h

@@ -471,8 +471,11 @@ STAT(memcached_version, , none, , ) // version string
 STAT(daemon_connections, , count, , )
 STAT(curr_connections, , count, , )
 STAT(system_connections, , count, , )
+STAT(user_connections, , count, , )
 STAT(total_connections, , count, , ) // total since start/reset
 STAT(connection_structures, , count, , )
+STAT(max_user_connections, , count, , )
+STAT(max_system_connections, , count, , )
 STAT(connection_recycle_high_watermark, , none, , )
 
 // Amount of memory allocated by the daemon.

tests/testapp/testapp_stats.cc

@@ -299,6 +299,7 @@ TEST_P(StatsTest, MB37147_TestEWBReturnFromStat) {
                                               sequence);
         auto stats = connection.stats("vbucket");
         EXPECT_FALSE(stats.empty());
+        connection.disableEwouldBlockEngine();
     });
 }
 
@@ -540,3 +541,17 @@ TEST_P(StatsTest, TestClocksStats) {
                       << value << "'";
     }
 }
+
+/// The stats should contain max user and system connections to allow
+/// for alerting by monitoring the current levels with the max
+TEST_P(StatsTest, MB59260) {
+    nlohmann::json stats;
+    adminConnection->executeInBucket(
+            bucketName, [&stats](auto& conn) { stats = conn.stats(""); });
+    ASSERT_TRUE(stats.contains("max_system_connections"));
+    ASSERT_TRUE(stats.contains("max_user_connections"));
+    const auto max_system = Testapp::MAX_CONNECTIONS / 4;
+    const auto max_user = Testapp::MAX_CONNECTIONS - max_system;
+    EXPECT_EQ(max_system, stats["max_system_connections"]);
+    EXPECT_EQ(max_user, stats["max_user_connections"]);
+}