add cbstats all to cbmonitor

Change-Id: I13c3fc25a61a8db82a2c13a743a25b2565e3ab07
Reviewed-on: https://review.couchbase.org/c/perfrunner/+/169533
Tested-by: Build Bot <[email protected]>
Reviewed-by: Korrigan Clark <[email protected]>

Author: bochun

cbagent/collectors/__init__.py

@@ -1,6 +1,6 @@
 from cbagent.collectors.active_tasks import ActiveTasks
 from cbagent.collectors.analytics import AnalyticsStats
-from cbagent.collectors.cbstats import CBStatsMemory
+from cbagent.collectors.cbstats import CBStatsAll, CBStatsMemory
 from cbagent.collectors.collector import Collector
 from cbagent.collectors.eventing_stats import (
     EventingConsumerStats,

cbagent/collectors/cbstats.py

@@ -70,3 +70,78 @@ def update_metadata(self):
             self.mc.add_bucket(bucket)
         for node in self.nodes:
             self.mc.add_server(node)
+
+
+class CBStatsAll(Collector):
+    COLLECTOR = "cbstats_all"
+    CB_STATS_PORT = 11209
+    METRICS = (
+        "mem_used_secondary",
+        "ep_magma_total_mem_used",
+        "ep_magma_mem_used_diff"
+    )
+
+    def __init__(self, settings, test):
+        super().__init__(settings)
+        extract_cb_any(filename='couchbase')
+        self.cluster_spec = test.cluster_spec
+
+    def _get_stats_from_server(self, bucket: str, server: str):
+        stats = {}
+        try:
+            result = get_cbstats(server, self.CB_STATS_PORT, "all", self.cluster_spec)
+            buckets_data = list(filter(lambda a: a != "", result.split("*")))
+            for data in buckets_data:
+                data = data.strip()
+                if data.startswith(bucket):
+                    data = data.split("\n", 1)[1]
+                    data = data.replace("\"{", "{")
+                    data = data.replace("}\"", "}")
+                    data = data.replace("\\", "")
+                    data = json.loads(data)
+                    for (metric, number) in data.items():
+                        if metric in self.METRICS:
+                            if metric in stats:
+                                stats[metric] += number
+                            else:
+                                stats[metric] = number
+                    ep_magma_mem_used_diff = stats['mem_used_secondary'] -\
+                        stats['ep_magma_total_mem_used']
+                    if 'ep_magma_mem_used_diff' in stats:
+                        stats['ep_magma_mem_used_diff'] += ep_magma_mem_used_diff
+                    else:
+                        stats['ep_magma_mem_used_diff'] = ep_magma_mem_used_diff
+                    break
+        except Exception:
+            pass
+
+        return stats
+
+    def _get_cbstats_all_stats(self, bucket: str, server: str):
+        node_stats = self._get_stats_from_server(bucket, server=server)
+        return node_stats
+
+    def sample(self):
+        for bucket in self.get_buckets():
+            stats = {}
+            for node in self.nodes:
+                temp_stats = self._get_cbstats_all_stats(bucket, node)
+                for st in temp_stats:
+                    if st in stats:
+                        stats[st] += temp_stats[st]
+                    else:
+                        stats[st] = temp_stats[st]
+
+            if stats:
+                self.update_metric_metadata(stats.keys(), bucket=bucket)
+                self.store.append(stats, cluster=self.cluster,
+                                  bucket=bucket,
+                                  collector=self.COLLECTOR)
+
+    def update_metadata(self):
+        self.mc.add_cluster()
+
+        for bucket in self.get_buckets():
+            self.mc.add_bucket(bucket)
+        for node in self.nodes:
+            self.mc.add_server(node)

cbagent/collectors/kvstore_stats.py

@@ -83,10 +83,8 @@ class KVStoreStats(Collector):
         "NumNormalFlushes",
         "NumPersistentFlushes",
         "NumSyncFlushes",
-        "HistogramMemUsed",
         "WALBufferMemUsed",
         "TreeSnapshotMemoryUsed",
-        "LSMTreeObjectMemUsed",
         "ReadAheadBufferMemUsed",
         "TableObjectMemUsed",
         "BlockCacheHitsPerSec",

perfrunner/helpers/cbmonitor.py

@@ -13,6 +13,7 @@
     VMSTAT,
     ActiveTasks,
     AnalyticsStats,
+    CBStatsAll,
     CBStatsMemory,
     Disk,
     DurabilityLatency,
@@ -157,6 +158,7 @@ def init_clusters(self, phase: str):
 
     def add_collectors(self,
                        analytics=False,
+                       cbstats_all=False,
                        cbstats_memory=False,
                        disk=False,
                        durability=False,
@@ -217,6 +219,8 @@ def add_collectors(self,
             self.add_collector(KVStoreStats, self.test)
         if cbstats_memory:
             self.add_collector(CBStatsMemory, self.test)
+        if cbstats_all:
+            self.add_collector(CBStatsAll, self.test)
 
         if not self.test.dynamic_infra:
             if self.test.remote.os != 'Cygwin':

perfrunner/tests/magma.py

@@ -327,7 +327,7 @@ def run(self):
 
 class KVTest(PerfTest):
     COLLECTORS = {'disk': True, 'latency': True, 'net': False, 'kvstore': True,
-                  'vmstat': True, 'cbstats_memory': True}
+                  'vmstat': True, 'cbstats_memory': True, 'cbstats_all': True}
     CB_STATS_PORT = 11209
 
     def __init__(self, *args):
@@ -861,8 +861,8 @@ def run(self):
 
 class YCSBThroughputHIDDTest(YCSBThroughputTest, KVTest):
 
-    COLLECTORS = {'disk': True, 'net': True, 'kvstore': True,
-                  'vmstat': True, 'cbstats_memory': True}
+    COLLECTORS = {'disk': True, 'net': True, 'kvstore': True, 'vmstat': True,
+                  'cbstats_memory': True, 'cbstats_all': True}
 
     def __init__(self, *args):
         KVTest.__init__(self, *args)
@@ -1138,7 +1138,7 @@ def run(self):
 class RebalanceKVDGMTest(RebalanceKVTest, StabilityBootstrap):
 
     COLLECTORS = {'disk': True, 'latency': True, 'net': False, 'kvstore': True,
-                  'vmstat': True, 'cbstats_memory': True}
+                  'vmstat': True, 'cbstats_memory': True, 'cbstats_all': True}
 
     def __init__(self, *args):
         RebalanceKVTest.__init__(self, *args)