add validation prometheus metrics

Author: yungwine

modules/prometheus.py

@@ -30,6 +30,12 @@ def to_format(self, value):
     'collated_master_err': Metric('validator_blocks_collated_master_err', 'Number of masterchain blocks failed to collate', 'gauge'),
     'collated_shard_ok': Metric('validator_blocks_collated_shard_ok', 'Number of shardchain blocks successfully collated', 'gauge'),
     'collated_shard_err': Metric('validator_blocks_collated_shard_err', 'Number of shardchain blocks failed to collate', 'gauge'),
+    'validated_master_ok': Metric('validator_blocks_validated_master_ok', 'Number of masterchain blocks successfully validated', 'gauge'),
+    'validated_master_err': Metric('validator_blocks_validated_master_err', 'Number of masterchain blocks failed to validate', 'gauge'),
+    'validated_shard_ok': Metric('validator_blocks_validated_shard_ok', 'Number of shardchain blocks successfully validated', 'gauge'),
+    'validated_shard_err': Metric('validator_blocks_validated_shard_err', 'Number of shardchain blocks failed to validate', 'gauge'),
+    'validator_groups_master': Metric('validator_active_groups_master', 'Number of masterchain validation groups validator participates in', 'gauge'),
+    'validator_groups_shard': Metric('validator_active_groups_shard', 'Number of shardchain validation groups validator participates in', 'gauge'),
     'ls_queries_ok': Metric('validator_ls_queries_ok', 'Number of Liteserver successful queries', 'gauge'),
     'ls_queries_err': Metric('validator_ls_queries_err', 'Number of Liteserver failed queries', 'gauge'),
 }
@@ -59,6 +65,9 @@ def get_validator_status_metrics(self, result: list):
                 result.append(METRICS['celldb_gc_state'].to_format(status.gcmasterchainblock - status.last_deleted_mc_state))
             else:
                 result.append(METRICS['celldb_gc_state'].to_format(-1))
+        if status.validator_groups_master is not None:
+            result.append(METRICS['validator_groups_master'].to_format(status.validator_groups_master))
+            result.append(METRICS['validator_groups_shard'].to_format(status.validator_groups_shard))
         result.append(METRICS['vc_up'].to_format(int(is_working)))
 
     def get_validator_validation_metrics(self, result: list):
@@ -81,6 +90,16 @@ def get_node_stats_metrics(self, result: list):
                 return
             result.append(METRICS['ls_queries_ok'].to_format(stats['ls_queries']['ok']))
             result.append(METRICS['ls_queries_err'].to_format(stats['ls_queries']['error']))
+        if stats and 'collated' in stats:
+            result.append(METRICS['collated_master_ok'].to_format(stats['collated']['master']['ok']))
+            result.append(METRICS['collated_master_err'].to_format(stats['collated']['master']['error']))
+            result.append(METRICS['collated_shard_ok'].to_format(stats['collated']['shard']['ok']))
+            result.append(METRICS['collated_shard_err'].to_format(stats['collated']['shard']['error']))
+        if stats and 'validated' in stats:
+            result.append(METRICS['validated_master_ok'].to_format(stats['validated']['master']['ok']))
+            result.append(METRICS['validated_master_err'].to_format(stats['validated']['master']['error']))
+            result.append(METRICS['validated_shard_ok'].to_format(stats['validated']['shard']['ok']))
+            result.append(METRICS['validated_shard_err'].to_format(stats['validated']['shard']['error']))
 
     def push_metrics(self):
         if not self.ton.using_prometheus():

mytoncore/functions.py

@@ -323,6 +323,9 @@ def get_ok_error(value: str):
                 data['ls_queries']['error'] = int(k.split(':')[1])
     statistics = local.db.get("statistics", dict())
 
+    if time.time() - int(status.start_time) <= 60:  # was node restart <60 sec ago, resetting node statistics
+        statistics['node'] = []
+
     # statistics['node'] = [stats_from_election_id, stats_from_prev_min, stats_now]
 
     election_id = ton.GetConfig34()['startWorkTime']

mytoncore/mytoncore.py

@@ -807,6 +807,10 @@ def GetValidatorStatus(self, no_cache=False):
 			status.last_deleted_mc_state = int(parse(result, "last_deleted_mc_state", '\n'))
 			status.stateserializerenabled = parse(result, "stateserializerenabled", '\n') == "true"
 			self.local.try_function(self.parse_stats_from_vc, args=[result, status])
+			if 'active_validator_groups' in status:
+				groups = status.active_validator_groups.split()  # master:1 shard:2
+				status.validator_groups_master = int(groups[0].split(':')[1])
+				status.validator_groups_shard = int(groups[1].split(':')[1])
 		except Exception as ex:
 			self.local.add_log(f"GetValidatorStatus warning: {ex}", "warning")
 			status.is_working = False
@@ -3043,27 +3047,34 @@ def get_node_statistics(self):
 		:return: stats for collated/validated blocks since round beggining and stats for ls queries for the last minute
 		"""
 		stats = self.local.db.get('statistics', {}).get('node')
-		result = {}
+		result = {
+			'collated': {
+				'ok': 0,
+				'error': 0,
+			},
+			'validated': {
+				'ok': 0,
+				'error': 0,
+			}
+		}
 		if stats is not None and len(stats) == 3 and stats[0] is not None:
-			collated_ok = 0
-			collated_error = 0
-			validated_ok = 0
-			validated_error = 0
 			for k in ['master', 'shard']:
-				collated_ok += stats[2]['collated_blocks'][k]['ok'] - stats[0]['collated_blocks'][k]['ok']
-				collated_error += stats[2]['collated_blocks'][k]['error'] - stats[0]['collated_blocks'][k]['error']
-				validated_ok += stats[2]['validated_blocks'][k]['ok'] - stats[0]['validated_blocks'][k]['ok']
-				validated_error += stats[2]['validated_blocks'][k]['error'] - stats[0]['validated_blocks'][k]['error']
-			result = {
-				'collated': {
+				collated_ok = stats[2]['collated_blocks'][k]['ok'] - stats[0]['collated_blocks'][k]['ok']
+				collated_error = stats[2]['collated_blocks'][k]['error'] - stats[0]['collated_blocks'][k]['error']
+				validated_ok = stats[2]['validated_blocks'][k]['ok'] - stats[0]['validated_blocks'][k]['ok']
+				validated_error = stats[2]['validated_blocks'][k]['error'] - stats[0]['validated_blocks'][k]['error']
+				result['collated'][k] = {
 					'ok': collated_ok,
 					'error': collated_error,
-				},
-				'validated': {
+				}
+				result['validated'][k] = {
 					'ok': validated_ok,
 					'error': validated_error,
-				},
-			}
+				}
+				result['collated']['ok'] += collated_ok
+				result['collated']['error'] += collated_error
+				result['validated']['ok'] += validated_ok
+				result['validated']['error'] += validated_error
 		if stats is not None and len(stats) >= 2:
 			result['ls_queries'] = {
 				'ok': stats[-1]['ls_queries']['ok'] - stats[-2]['ls_queries']['ok'],