add shard_collators_offline alert

Author: yungwine

modules/alert_bot.py

@@ -4,7 +4,7 @@
 
 from modules.module import MtcModule
 from mypylib.mypylib import get_timestamp, print_table, color_print
-from mytoncore import get_hostname
+from mytoncore import get_hostname, signed_int_to_hex64
 from mytonctrl.utils import timestamp2utcdatetime
 
 
@@ -121,7 +121,13 @@ def init_alerts():
             "Initial sync has been completed (info alert with no sound)",
             "Node initial sync has been completed",
             0
-        )
+        ),
+        "shard_collators_offline": Alert(
+            "high",
+            "All collators for specific shards are offline",
+            "All collators for shards <code>{shards}</code> are offline.",
+            3600
+        ),
     }
 
 
@@ -452,6 +458,27 @@ def check_initial_sync(self):
             self.initial_sync = False
             self.send_alert("initial_sync_completed")
 
+    def check_online_collators(self):
+        if not self.ton.using_validator():
+            return
+        collators_list = self.validator_module.get_collators_list()
+        if not collators_list or not collators_list['shards']:
+            return
+        collators_stats = self.validator_module.get_collators_stats()
+        offline_shards = []
+
+        for shard in collators_list['shards']:
+            if not shard['collators']:
+                continue
+            collators_alive = []
+            for c in shard['collators']:
+                collators_alive.append(collators_stats.get(c['adnl_id']))
+            if not any(collators_alive):
+                offline_shards.append(f"{shard['shard_id']['workchain']}:{signed_int_to_hex64(int(shard['shard_id']['shard']))}")
+
+        if offline_shards:
+            self.send_alert("shard_collators_offline", shards=' '.join(offline_shards))
+
     def check_status(self):
         if not self.ton.using_alert_bot():
             return
@@ -471,6 +498,7 @@ def check_status(self):
         self.local.try_function(self.check_stake_returned)
         self.local.try_function(self.check_voting)
         self.local.try_function(self.check_initial_sync)
+        self.local.try_function(self.check_online_collators)
 
     def add_console_commands(self, console):
         console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd"))

modules/validator.py

@@ -234,7 +234,7 @@ def delete_collator(self, args: list):
             self.set_collators_list(collators_list)
         color_print("delete_collator - {green}OK{endc}")
 
-    def _get_collators_stats(self):
+    def get_collators_stats(self):
         output = self.ton.validatorConsole.Run('collation-manager-stats')
         if 'No stats' in output:
             return {}
@@ -257,7 +257,7 @@ def print_collators(self, args: list):
             if 'collators list is empty' in result:
                 print("No collators found")
                 return
-            collators_stats = self._get_collators_stats()
+            collators_stats = self.get_collators_stats()
             for adnl, alive in collators_stats.items():
                 if adnl in result:
                     status = '{green}online{endc}' if alive else '{red}offline{endc}'