Merge branch 'dev' into btc-teleport

Author: yungwine

modules/__init__.py

@@ -62,7 +62,8 @@ class Setting:
     'auto_backup': Setting('validator', None, 'Make validator backup every election'),
     'auto_backup_path': Setting('validator', '/tmp/mytoncore/auto_backups/', 'Path to store auto-backups'),
     'prometheus_url': Setting('prometheus', None, 'Prometheus pushgateway url'),
-    'onlyNode': Setting(None, None, 'MyTonCtrl will work only for collecting validator telemetry (if `sendTelemetry` is True), without participating in Elections and etc.')
+    'onlyNode': Setting(None, None, 'MyTonCtrl will work only for collecting validator telemetry (if `sendTelemetry` is True), without participating in Elections and etc.'),
+    'importGc': Setting(None, None, 'Delete imported archive blocks files. Restart mytoncore to apply this setting'),
 }
 
 

modules/alert_bot.py

@@ -116,6 +116,12 @@ def init_alerts():
             "Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.",
             VALIDATION_PERIOD
         ),
+        "initial_sync_completed": Alert(
+            "info",
+            "Initial sync has been completed (info alert with no sound)",
+            "Node initial sync has been completed",
+            0
+        )
     }
 
 
@@ -134,6 +140,7 @@ def __init__(self, ton, local, *args, **kwargs):
         self.token = None
         self.chat_id = None
         self.last_db_check = 0
+        self.initial_sync = None
 
     def send_message(self, text: str, silent: bool = False, disable_web_page_preview: bool = False):
         if self.token is None:
@@ -207,6 +214,7 @@ def init(self):
         self.wallet = self.ton.GetValidatorWallet().addrB64
         self.ip = self.ton.get_node_ip()
         self.set_global_vars()
+        self.initial_sync = self.ton.in_initial_sync()
         init_alerts()
         self.inited = True
 
@@ -271,6 +279,7 @@ def setup_alert_bot(self, args):
             self.send_welcome_message()
             self.ton.local.db['BotToken'] = args[0]
             self.ton.local.db['ChatId'] = args[1]
+            self.ton.local.save()
             color_print("setup_alert_bot - {green}OK{endc}")
         except Exception as e:
             self.local.add_log(f"Error while sending welcome message: {e}", "error")
@@ -314,7 +323,7 @@ def check_validator_wallet_balance(self):
             return
         validator_wallet = self.ton.GetValidatorWallet()
         validator_account = self.ton.GetAccount(validator_wallet.addrB64)
-        if validator_account.balance < 10:
+        if validator_account.status != "empty" and validator_account.balance < 10:
             self.send_alert("low_wallet_balance", wallet=validator_wallet.addrB64, balance=validator_account.balance)
 
     def check_efficiency(self):
@@ -334,12 +343,12 @@ def check_efficiency(self):
 
     def check_validator_working(self):
         validator_status = self.ton.GetValidatorStatus()
-        if not validator_status.is_working:
+        if not self.initial_sync and not validator_status.is_working:
             self.send_alert("service_down")
 
     def check_sync(self):
         validator_status = self.ton.GetValidatorStatus()
-        if validator_status.is_working and validator_status.out_of_sync >= 20:
+        if not self.initial_sync and validator_status.is_working and validator_status.out_of_sync >= 20:
             self.send_alert("out_of_sync", sync=validator_status.out_of_sync)
 
     def check_zero_blocks_created(self):
@@ -435,6 +444,13 @@ def check_voting(self):
         if need_to_vote:
             self.send_alert("voting", hashes=' '.join(need_to_vote))
 
+    def check_initial_sync(self):
+        if not self.initial_sync:
+            return
+        if not self.ton.in_initial_sync():
+            self.initial_sync = False
+            self.send_alert("initial_sync_completed")
+
     def check_status(self):
         if not self.ton.using_alert_bot():
             return
@@ -453,6 +469,7 @@ def check_status(self):
         self.local.try_function(self.check_stake_sent)
         self.local.try_function(self.check_stake_returned)
         self.local.try_function(self.check_voting)
+        self.local.try_function(self.check_initial_sync)
 
     def add_console_commands(self, console):
         console.AddItem("enable_alert", self.enable_alert, self.local.translate("enable_alert_cmd"))

modules/prometheus.py

@@ -26,6 +26,18 @@ def to_format(self, value):
     'stake': Metric('validator_stake', 'Validator stake', 'gauge'),
     'celldb_gc_block': Metric('validator_celldb_gc_block', 'Celldb GC block latency', 'gauge'),
     'celldb_gc_state': Metric('validator_celldb_gc_state', 'Celldb GC queue size', 'gauge'),
+    'collated_master_ok': Metric('validator_blocks_collated_master_ok', 'Number of masterchain blocks successfully collated', 'gauge'),
+    'collated_master_err': Metric('validator_blocks_collated_master_err', 'Number of masterchain blocks failed to collate', 'gauge'),
+    'collated_shard_ok': Metric('validator_blocks_collated_shard_ok', 'Number of shardchain blocks successfully collated', 'gauge'),
+    'collated_shard_err': Metric('validator_blocks_collated_shard_err', 'Number of shardchain blocks failed to collate', 'gauge'),
+    'validated_master_ok': Metric('validator_blocks_validated_master_ok', 'Number of masterchain blocks successfully validated', 'gauge'),
+    'validated_master_err': Metric('validator_blocks_validated_master_err', 'Number of masterchain blocks failed to validate', 'gauge'),
+    'validated_shard_ok': Metric('validator_blocks_validated_shard_ok', 'Number of shardchain blocks successfully validated', 'gauge'),
+    'validated_shard_err': Metric('validator_blocks_validated_shard_err', 'Number of shardchain blocks failed to validate', 'gauge'),
+    'validator_groups_master': Metric('validator_active_groups_master', 'Number of masterchain validation groups validator participates in', 'gauge'),
+    'validator_groups_shard': Metric('validator_active_groups_shard', 'Number of shardchain validation groups validator participates in', 'gauge'),
+    'ls_queries_ok': Metric('validator_ls_queries_ok', 'Number of Liteserver successful queries', 'gauge'),
+    'ls_queries_err': Metric('validator_ls_queries_err', 'Number of Liteserver failed queries', 'gauge'),
 }
 
 
@@ -44,7 +56,7 @@ def get_validator_status_metrics(self, result: list):
             result.append(METRICS['master_out_of_sync'].to_format(status.masterchain_out_of_sync))
         if status.shardchain_out_of_sync is not None:
             result.append(METRICS['shard_out_of_sync'].to_format(status.shardchain_out_of_sync))
-        if status.masterchain_out_of_ser is not None and status.stateserializermasterchainseqno != 0:
+        if status.stateserializerenabled and status.masterchain_out_of_ser is not None and status.stateserializermasterchainseqno != 0:
             result.append(METRICS['out_of_ser'].to_format(status.masterchain_out_of_ser))
         if status.masterchainblock is not None and status.gcmasterchainblock is not None:
             result.append(METRICS['celldb_gc_block'].to_format(status.masterchainblock - status.gcmasterchainblock))
@@ -53,6 +65,9 @@ def get_validator_status_metrics(self, result: list):
                 result.append(METRICS['celldb_gc_state'].to_format(status.gcmasterchainblock - status.last_deleted_mc_state))
             else:
                 result.append(METRICS['celldb_gc_state'].to_format(-1))
+        if status.validator_groups_master is not None:
+            result.append(METRICS['validator_groups_master'].to_format(status.validator_groups_master))
+            result.append(METRICS['validator_groups_shard'].to_format(status.validator_groups_shard))
         result.append(METRICS['vc_up'].to_format(int(is_working)))
 
     def get_validator_validation_metrics(self, result: list):
@@ -67,6 +82,25 @@ def get_validator_validation_metrics(self, result: list):
             if stake:
                 result.append(METRICS['stake'].to_format(round(stake, 2)))
 
+    def get_node_stats_metrics(self, result: list):
+        stats = self.ton.get_node_statistics()
+        if stats and 'ls_queries' in stats:
+            if stats['ls_queries']['time'] < 50:
+                self.local.add_log(f'Liteserver queries time is too low: {stats}')
+                return
+            result.append(METRICS['ls_queries_ok'].to_format(stats['ls_queries']['ok']))
+            result.append(METRICS['ls_queries_err'].to_format(stats['ls_queries']['error']))
+        if stats and 'collated' in stats:
+            result.append(METRICS['collated_master_ok'].to_format(stats['collated']['master']['ok']))
+            result.append(METRICS['collated_master_err'].to_format(stats['collated']['master']['error']))
+            result.append(METRICS['collated_shard_ok'].to_format(stats['collated']['shard']['ok']))
+            result.append(METRICS['collated_shard_err'].to_format(stats['collated']['shard']['error']))
+        if stats and 'validated' in stats:
+            result.append(METRICS['validated_master_ok'].to_format(stats['validated']['master']['ok']))
+            result.append(METRICS['validated_master_err'].to_format(stats['validated']['master']['error']))
+            result.append(METRICS['validated_shard_ok'].to_format(stats['validated']['shard']['ok']))
+            result.append(METRICS['validated_shard_err'].to_format(stats['validated']['shard']['error']))
+
     def push_metrics(self):
         if not self.ton.using_prometheus():
             return
@@ -77,6 +111,7 @@ def push_metrics(self):
         metrics = []
         self.local.try_function(self.get_validator_status_metrics, args=[metrics])
         self.local.try_function(self.get_validator_validation_metrics, args=[metrics])
+        self.local.try_function(self.get_node_stats_metrics, args=[metrics])
         requests.post(url, data='\n'.join(metrics).encode())
 
     def add_console_commands(self, console):

mytoncore/fift.py

@@ -19,7 +19,7 @@ def Run(self, args, **kwargs):
 		process = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
 		output = process.stdout.decode("utf-8")
 		err = process.stderr.decode("utf-8")
-		if len(err) > 0:
+		if process.returncode != 0 and len(err) > 0:
 			self.local.add_log("args: {args}".format(args=args), "error")
 			raise Exception("Fift error: {err}".format(err=err))
 		return output

mytoncore/functions.py

@@ -53,8 +53,6 @@ def Event(local, event_name):
         EnableVcEvent(local)
     elif event_name == "validator down":
         ValidatorDownEvent(local)
-    elif event_name == "enable_ton_storage_provider":
-        enable_ton_storage_provider_event(local)
     elif event_name.startswith("enable_mode"):
         enable_mode(local, event_name)
     elif event_name == "enable_btc_teleport":
@@ -86,15 +84,6 @@ def ValidatorDownEvent(local):
 # end define
 
 
-def enable_ton_storage_provider_event(local):
-    config_path = local.db.ton_storage.provider.config_path
-    config = GetConfig(path=config_path)
-    key_bytes = base64.b64decode(config.ProviderKey)
-    ton = MyTonCore(local)
-    ton.import_wallet_with_version(key_bytes[:32], version="v3r2", wallet_name="provider_wallet_001")
-#end define
-
-
 def enable_mode(local, event_name):
     ton = MyTonCore(local)
     mode = event_name.split("_")[-1]
@@ -292,6 +281,64 @@ def CalculateNetworkStatistics(zerodata, data):
 # end define
 
 
+def save_node_statistics(local, ton):
+    status = ton.GetValidatorStatus(no_cache=True)
+    if status.unixtime is None:
+        return
+    data = {'timestamp': status.unixtime}
+
+    def get_ok_error(value: str):
+        ok, error = value.split()
+        return int(ok.split(':')[1]), int(error.split(':')[1])
+
+    if 'total.collated_blocks.master' in status:
+        master_ok, master_error = get_ok_error(status['total.collated_blocks.master'])
+        shard_ok, shard_error = get_ok_error(status['total.collated_blocks.shard'])
+        data['collated_blocks'] = {
+            'master': {'ok': master_ok, 'error': master_error},
+            'shard': {'ok': shard_ok, 'error': shard_error},
+        }
+    if 'total.validated_blocks.master' in status:
+        master_ok, master_error = get_ok_error(status['total.validated_blocks.master'])
+        shard_ok, shard_error = get_ok_error(status['total.validated_blocks.shard'])
+        data['validated_blocks'] = {
+            'master': {'ok': master_ok, 'error': master_error},
+            'shard': {'ok': shard_ok, 'error': shard_error},
+        }
+    if 'total.ext_msg_check' in status:
+        ok, error = get_ok_error(status['total.ext_msg_check'])
+        data['ext_msg_check'] = {'ok': ok, 'error': error}
+    if 'total.ls_queries_ok' in status and 'total.ls_queries_error' in status:
+        data['ls_queries'] = {}
+        for k in status['total.ls_queries_ok'].split():
+            if k.startswith('TOTAL'):
+                data['ls_queries']['ok'] = int(k.split(':')[1])
+        for k in status['total.ls_queries_error'].split():
+            if k.startswith('TOTAL'):
+                data['ls_queries']['error'] = int(k.split(':')[1])
+    statistics = local.db.get("statistics", dict())
+
+    if time.time() - int(status.start_time) <= 60:  # was node restart <60 sec ago, resetting node statistics
+        statistics['node'] = []
+
+    # statistics['node'] = [stats_from_election_id, stats_from_prev_min, stats_now]
+
+    election_id = ton.GetConfig34()['startWorkTime']
+    if 'node' not in statistics or len(statistics['node']) == 0:
+        statistics['node'] = [None, data]
+    elif len(statistics['node']) < 3:
+        statistics['node'].append(data)
+    if len(statistics['node']) == 3:
+        if statistics['node'][0] is None:
+            if 0 < data['timestamp'] - election_id < 90:
+                statistics['node'][0] = data
+        elif statistics['node'][0]['timestamp'] < election_id:
+            statistics['node'][0] = data
+        statistics['node'] = statistics.get('node', []) + [data]
+        statistics['node'].pop(1)
+    local.db["statistics"] = statistics
+
+
 def ReadTransData(local, scanner):
     transData = local.buffer.transData
     SetToTimeData(transData, scanner.transNum)
@@ -417,6 +464,7 @@ def Telemetry(local, ton):
     data["vprocess"] = GetValidatorProcessInfo()
     data["dbStats"] = local.try_function(get_db_stats)
     data["nodeArgs"] = local.try_function(get_node_args)
+    data["modes"] = local.try_function(ton.get_modes)
     data["cpuInfo"] = {'cpuName': local.try_function(get_cpu_name), 'virtual': local.try_function(is_host_virtual)}
     data["validatorDiskName"] = local.try_function(get_validator_disk_name)
     data["pings"] = local.try_function(get_pings_values)
@@ -550,6 +598,46 @@ def ScanLiteServers(local, ton):
 # end define
 
 
+def check_initial_sync(local, ton):
+    if not ton.in_initial_sync():
+        return
+    validator_status = ton.GetValidatorStatus()
+    if validator_status.initial_sync:
+        return
+    if validator_status.out_of_sync < 20:
+        ton.set_initial_sync_off()
+        return
+
+
+def gc_import(local, ton):
+    if not ton.local.db.get('importGc', False):
+        return
+    local.add_log("GC import is running", "debug")
+    import_path = '/var/ton-work/db/import'
+    files = os.listdir(import_path)
+    if not files:
+        local.add_log("No files left to import", "debug")
+        ton.local.db['importGc'] = False
+        return
+    try:
+        status = ton.GetValidatorStatus()
+        node_seqno = int(status.shardclientmasterchainseqno)
+    except Exception as e:
+        local.add_log(f"Failed to get shardclientmasterchainseqno: {e}", "warning")
+        return
+    removed = 0
+    for file in files:
+        file_seqno = int(file.split('.')[1])
+        if node_seqno > file_seqno + 101:
+            try:
+                os.remove(os.path.join(import_path, file))
+                removed += 1
+            except PermissionError:
+                local.add_log(f"Failed to remove file {file}: Permission denied", "error")
+                continue
+    local.add_log(f"Removed {removed} import files up to {node_seqno} seqno", "debug")
+
+
 def General(local):
     local.add_log("start General function", "debug")
     ton = MyTonCore(local)
@@ -576,6 +664,8 @@ def General(local):
 
     local.start_cycle(ScanLiteServers, sec=60, args=(local, ton,))
 
+    local.start_cycle(save_node_statistics, sec=60, args=(local, ton, ))
+
     from modules.custom_overlays import CustomOverlayModule
     local.start_cycle(CustomOverlayModule(ton, local).custom_overlays, sec=60, args=())
 
@@ -588,6 +678,11 @@ def General(local):
     from modules.btc_teleport import BtcTeleportModule
     local.start_cycle(BtcTeleportModule(ton, local).auto_vote_offers, sec=600, args=())
 
+    if ton.in_initial_sync():
+        local.start_cycle(check_initial_sync, sec=120, args=(local, ton))
+
+    if ton.local.db.get('importGc'):
+        local.start_cycle(gc_import, sec=300, args=(local, ton))
 
     thr_sleep()
 # end define

mytoncore/liteclient.py

@@ -22,7 +22,7 @@ def Run(self, cmd, **kwargs):
 		if index is not None:
 			index = str(index)
 			args += ["-i", index]
-		elif useLocalLiteServer and self.pubkeyPath and validator_status.out_of_sync and validator_status.out_of_sync < 20:
+		elif useLocalLiteServer and self.pubkeyPath and validator_status.out_of_sync is not None and validator_status.out_of_sync < 20:
 			args = [self.appPath, "--addr", self.addr, "--pub", self.pubkeyPath, "--verbosity", "0", "--cmd", cmd]
 		else:
 			liteServers = self.local.db.get("liteServers")