Merge remote-tracking branch 'tonblkch/dev' into dev

Author: yungwine

modules/__init__.py

@@ -9,6 +9,7 @@
 from modules.controller import ControllerModule
 from modules.liteserver import LiteserverModule
 from modules.alert_bot import AlertBotModule
+from modules.prometheus import PrometheusModule
 
 
 MODES = {
@@ -17,7 +18,8 @@
     'single-nominator': SingleNominatorModule,
     'liquid-staking': ControllerModule,
     'liteserver': LiteserverModule,
-    'alert-bot': AlertBotModule
+    'alert-bot': AlertBotModule,
+    'prometheus': PrometheusModule
 }
 
 
@@ -58,7 +60,11 @@ class Setting:
     'debug': Setting(None, False, 'Debug mtc console mode. Prints Traceback on errors'),
     'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'),
     'BotToken': Setting('alert-bot', None, 'Alerting Telegram bot token'),
-    'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id')
+    'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id'),
+    'auto_backup': Setting('validator', None, 'Make validator backup every election'),
+    'auto_backup_path': Setting('validator', '/tmp/mytoncore/auto_backups/', 'Path to store auto-backups'),
+    'prometheus_url': Setting('prometheus', None, 'Prometheus pushgateway url'),
+    'onlyNode': Setting(None, None, 'MyTonCtrl will work only for collecting validator telemetry (if `sendTelemetry` is True), without participating in Elections and etc.')
 }
 
 

modules/alert_bot.py

@@ -18,59 +18,90 @@ class Alert:
 HOUR = 3600
 VALIDATION_PERIOD = 65536
 FREEZE_PERIOD = 32768
-
-
-ALERTS = {
-    "low_wallet_balance": Alert(
-        "low",
-        "Validator wallet {wallet} balance is low: {balance} TON.",
-        18*HOUR
-    ),
-    "db_usage_80": Alert(
-        "high",
-        """TON DB usage > 80%. Clean the TON database: 
-        https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming 
-        or (and) set node\'s archive ttl to lower value.""",
-        24*HOUR
-    ),
-    "db_usage_95": Alert(
-        "critical",
-        """TON DB usage > 95%. Disk is almost full, clean the TON database immediately: 
-        https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming 
-        or (and) set node\'s archive ttl to lower value.""",
-        6*HOUR
-    ),
-    "low_efficiency": Alert(
-        "high",
-        """Validator efficiency is low: {efficiency}%.""",
-        VALIDATION_PERIOD // 3
-    ),
-    "out_of_sync": Alert(
-        "critical",
-        "Node is out of sync on {sync} sec.",
-        300
-    ),
-    "service_down": Alert(
-        "critical",
-        "validator.service is down.",
-        300
-    ),
-    "adnl_connection_failed": Alert(
-        "high",
-        "ADNL connection to node failed",
-        3*HOUR
-    ),
-    "zero_block_created": Alert(
-        "critical",
-        "Validator has not created any blocks in the last {hours} hours.",
-        VALIDATION_PERIOD // 3
-    ),
-    "validator_slashed": Alert(
-        "high",
-        "Validator has been slashed in previous round for {amount} TON",
-        FREEZE_PERIOD
-    ),
-}
+ELECTIONS_START_BEFORE = 8192
+
+
+ALERTS = {}
+
+
+def init_alerts():
+    global ALERTS
+    ALERTS = {
+        "low_wallet_balance": Alert(
+            "low",
+            "Validator wallet {wallet} balance is low: {balance} TON.",
+            18 * HOUR
+        ),
+        "db_usage_80": Alert(
+            "high",
+            """TON DB usage > 80%. Clean the TON database: 
+            https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming 
+            or (and) set node\'s archive ttl to lower value.""",
+            24 * HOUR
+        ),
+        "db_usage_95": Alert(
+            "critical",
+            """TON DB usage > 95%. Disk is almost full, clean the TON database immediately: 
+            https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming 
+            or (and) set node\'s archive ttl to lower value.""",
+            6 * HOUR
+        ),
+        "low_efficiency": Alert(
+            "high",
+            """Validator efficiency is low: {efficiency}%.""",
+            VALIDATION_PERIOD // 3
+        ),
+        "out_of_sync": Alert(
+            "critical",
+            "Node is out of sync on {sync} sec.",
+            300
+        ),
+        "service_down": Alert(
+            "critical",
+            "validator.service is down.",
+            300
+        ),
+        "adnl_connection_failed": Alert(
+            "high",
+            "ADNL connection to node failed",
+            3 * HOUR
+        ),
+        "zero_block_created": Alert(
+            "critical",
+            "Validator has not created any blocks in the last {hours} hours.",
+            VALIDATION_PERIOD // 3
+        ),
+        "validator_slashed": Alert(
+            "high",
+            "Validator has been slashed in previous round for {amount} TON",
+            FREEZE_PERIOD
+        ),
+        "stake_not_accepted": Alert(
+            "high",
+            "Validator's stake has not been accepted",
+            ELECTIONS_START_BEFORE
+        ),
+        "stake_accepted": Alert(
+            "info",
+            "Validator's stake {stake} TON has been accepted",
+            ELECTIONS_START_BEFORE
+        ),
+        "stake_returned": Alert(
+            "info",
+            "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.",
+            60
+        ),
+        "stake_not_returned": Alert(
+            "high",
+            "Validator's stake has not been returned on address {address}.",
+            60
+        ),
+        "voting": Alert(
+            "high",
+            "Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.",
+            VALIDATION_PERIOD
+        ),
+    }
 
 
 class AlertBotModule(MtcModule):
@@ -83,17 +114,18 @@ def __init__(self, ton, local, *args, **kwargs):
         self.validator_module = None
         self.inited = False
         self.hostname = None
+        self.ip = None
         self.token = None
         self.chat_id = None
         self.last_db_check = 0
 
-    def send_message(self, text: str):
+    def send_message(self, text: str, silent: bool = False):
         if self.token is None:
             raise Exception("send_message error: token is not initialized")
         if self.chat_id is None:
             raise Exception("send_message error: chat_id is not initialized")
         request_url = f"https://api.telegram.org/bot{self.token}/sendMessage"
-        data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML'}
+        data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML', 'disable_notification': silent}
         response = requests.post(request_url, data=data, timeout=3)
         if response.status_code != 200:
             raise Exception(f"send_message error: {response.text}")
@@ -113,22 +145,24 @@ def send_alert(self, alert_name: str, *args, **kwargs):
 ❗️ <b>MyTonCtrl Alert {alert_name}</b> ❗️
 
 Hostname: <code>{self.hostname}</code>
+Node IP: <code>{self.ip}</code>
 Time: <code>{time_}</code> (<code>{int(time.time())}</code>)
 Severity: <code>{alert.severity}</code>
 
 Alert text:
 <blockquote> {alert.text.format(*args, **kwargs)} </blockquote>
 '''
         if time.time() - last_sent > alert.timeout:
-            self.send_message(text)
+            self.send_message(text, alert.severity == "info")  # send info alerts without sound
             self.set_alert_sent(alert_name)
 
     def set_global_vars(self):
         # set global vars for correct alerts timeouts for current network
         config15 = self.ton.GetConfig15()
-        global VALIDATION_PERIOD, FREEZE_PERIOD
+        global VALIDATION_PERIOD, FREEZE_PERIOD, ELECTIONS_START_BEFORE
         VALIDATION_PERIOD = config15["validatorsElectedFor"]
         FREEZE_PERIOD = config15["stakeHeldFor"]
+        ELECTIONS_START_BEFORE = config15["electionsStartBefore"]
 
     def init(self):
         if not self.ton.get_mode_value('alert-bot'):
@@ -140,7 +174,9 @@ def init(self):
         from modules.validator import ValidatorModule
         self.validator_module = ValidatorModule(self.ton, self.local)
         self.hostname = get_hostname()
+        self.ip = self.ton.get_node_ip()
         self.set_global_vars()
+        init_alerts()
         self.inited = True
 
     def get_alert_from_db(self, alert_name: str):
@@ -206,6 +242,9 @@ def check_db_usage(self):
     def check_validator_wallet_balance(self):
         if not self.ton.using_validator():
             return
+        validator_status = self.ton.GetValidatorStatus()
+        if not validator_status.is_working or validator_status.out_of_sync >= 20:
+            return
         validator_wallet = self.ton.GetValidatorWallet()
         validator_account = self.ton.GetAccount(validator_wallet.addrB64)
         if validator_account.balance < 10:
@@ -265,6 +304,66 @@ def check_adnl_connection_failed(self):
         if not ok:
             self.send_alert("adnl_connection_failed")
 
+    def get_myself_from_election(self, config: dict):
+        if not config["validators"]:
+            return
+        adnl = self.ton.GetAdnlAddr()
+        save_elections = self.ton.GetSaveElections()
+        elections = save_elections.get(str(config["startWorkTime"]))
+        if elections is None:
+            return
+        if adnl not in elections:  # didn't participate in elections
+            return
+        validator = self.validator_module.find_myself(config["validators"])
+        if validator is None:
+            return False
+        validator['stake'] = elections[adnl].get('stake')
+        validator['walletAddr'] = elections[adnl].get('walletAddr')
+        return validator
+
+    def check_stake_sent(self):
+        if not self.ton.using_validator():
+            return
+        config = self.ton.GetConfig36()
+        res = self.get_myself_from_election(config)
+        if res is None:
+            return
+        if res is False:
+            self.send_alert("stake_not_accepted")
+            return
+        self.send_alert("stake_accepted", stake=round(res.get('stake'), 2))
+
+    def check_stake_returned(self):
+        if not self.ton.using_validator():
+            return
+        config = self.ton.GetConfig32()
+        if not (config['endWorkTime'] + FREEZE_PERIOD + 1800 <= time.time() < config['endWorkTime'] + FREEZE_PERIOD + 1860):  # check between 25th and 26th minutes after stakes have been unfrozen
+            return
+        res = self.get_myself_from_election(config)
+        if not res:
+            return
+        trs = self.ton.GetAccountHistory(self.ton.GetAccount(res["walletAddr"]), limit=10)
+
+        for tr in trs:
+            if tr.time >= config['endWorkTime'] + FREEZE_PERIOD and tr.srcAddr == '3333333333333333333333333333333333333333333333333333333333333333' and tr.body.startswith('F96F7324'):  # Elector Recover Stake Response
+                self.send_alert("stake_returned", stake=round(tr.value, 2), address=res["walletAddr"], reward=round(tr.value - res.get('stake', 0), 2))
+                return
+        self.send_alert("stake_not_returned", address=res["walletAddr"])
+
+    def check_voting(self):
+        if not self.ton.using_validator():
+            return
+        validator_index = self.ton.GetValidatorIndex()
+        if validator_index == -1:
+            return
+        need_to_vote = []
+        offers = self.ton.GetOffers()
+        for offer in offers:
+            if not offer['isPassed'] and offer['approvedPercent'] >= 50 and validator_index not in offer['votedValidators']:
+                need_to_vote.append(offer['hash'])
+        if need_to_vote:
+            self.send_alert("voting", hashes=' '.join(need_to_vote))
+
     def check_status(self):
         if not self.ton.using_alert_bot():
             return
@@ -279,6 +378,9 @@ def check_status(self):
         self.local.try_function(self.check_sync)
         self.local.try_function(self.check_slashed)
         self.local.try_function(self.check_adnl_connection_failed)
+        self.local.try_function(self.check_stake_sent)
+        self.local.try_function(self.check_stake_returned)
+        self.local.try_function(self.check_voting)
 
     def add_console_commands(self, console):
         console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd"))