Merge pull request #6 from yungwine/mytonctrl2_dev

merge mtc2dev to modes

Author: yungwine

docs/ru/ton-storage-provider.md

@@ -0,0 +1,32 @@
+# TON storage provider
+
+MyTonCtrl поддерживает установку дополнительных компонентов таких как `TON storage` и `TON storage provider`. С их помощью вы можете хранить пользовательские файлы у себя на сервере и получать за это вознаграждение.
+
+## Установка дополнительных компонентов
+
+Войдите в MyTonCtrl
+```
+mytonctrl
+```
+
+Затем войдите в режим установщика
+```
+installer
+```
+
+После включите нужный функционал TON storage provider:
+```
+enable TSP
+```
+
+## Активация контракта провайдера
+
+В ходе установки дополнительных компонетов был создан специальный кошелек провайдера `provider_wallet_001` откуда будут отправляться доказательства и куда будет приходить награда.
+Для его работы сначала нужно пополнить этот кошелек на 1 монету и активировать его командой внутри MyTonCtrl, в ходе которого он так же зарегистрируется в списке провайдеров:
+```
+activate_ton_storage_provider
+```
+
+## Панель управления TON storage provider
+
+TODO

modules/custom_overlays.py

@@ -0,0 +1,181 @@
+import json
+import requests
+
+from mypylib.mypylib import color_print
+from modules.module import MtcModule
+from mytoncore.utils import hex2base64
+
+
+class CustomOverlayModule(MtcModule):
+
+    @staticmethod
+    def parse_config(name: str, config: dict, vset: list = None):
+        """
+        Converts config to validator-console friendly format
+        :param name: custom overlay name
+        :param config: config
+        :param vset: list of validators adnl addresses, can be None if `@validators` not in config
+        :return:
+        """
+        result = {
+            "name": name,
+            "nodes": []
+        }
+        for k, v in config.items():
+            if k == '@validators' and v:
+                if vset is None:
+                    raise Exception("Validators set is not defined but @validators is in config")
+                for v_adnl in vset:
+                    result["nodes"].append({
+                        "adnl_id": hex2base64(v_adnl),
+                        "msg_sender": False,
+                    })
+            else:
+                result["nodes"].append({
+                    "adnl_id": hex2base64(k),
+                    "msg_sender": v["msg_sender"],
+                })
+                if v["msg_sender"]:
+                    result["nodes"][-1]["msg_sender_priority"] = v["msg_sender_priority"]
+        return result
+
+    def add_custom_overlay(self, args):
+        if len(args) != 2:
+            color_print("{red}Bad args. Usage:{endc} add_custom_overlay <name> <path_to_config>")
+            return
+        path = args[1]
+        with open(path, 'r') as f:
+            config = json.load(f)
+        self.ton.set_custom_overlay(args[0], config)
+        if '@validators' in config:
+            print('Dynamic overlay will be added within 1 minute')
+        else:
+            result = self.add_custom_overlay_to_vc(self.parse_config(args[0], config))
+            if not result:
+                print('Failed to add overlay to validator console')
+                color_print("add_custom_overlay - {red}ERROR{endc}")
+                return
+        color_print("add_custom_overlay - {green}OK{endc}")
+
+    def list_custom_overlays(self, args):
+        if not self.ton.get_custom_overlays():
+            color_print("{red}No custom overlays{endc}")
+            return
+        for k, v in self.ton.get_custom_overlays().items():
+            color_print(f"Custom overlay {{bold}}{k}{{endc}}:")
+            print(json.dumps(v, indent=4))
+
+    def delete_custom_overlay(self, args):
+        if len(args) != 1:
+            color_print("{red}Bad args. Usage:{endc} delete_custom_overlay <name>")
+            return
+        if '@validators' in self.ton.get_custom_overlays().get(args[0], {}):
+            self.ton.delete_custom_overlay(args[0])
+            print('Dynamic overlay will be deleted within 1 minute')
+        else:
+            self.ton.delete_custom_overlay(args[0])
+            result = self.delete_custom_overlay_from_vc(args[0])
+            if not result:
+                print('Failed to delete overlay from validator console')
+                color_print("delete_custom_overlay - {red}ERROR{endc}")
+                return
+        color_print("delete_custom_overlay - {green}OK{endc}")
+
+    def check_node_eligible_for_custom_overlay(self, config: dict):
+        vconfig = self.ton.GetValidatorConfig()
+        my_adnls = vconfig.adnl
+        node_adnls = [i["adnl_id"] for i in config["nodes"]]
+        for adnl in my_adnls:
+            if adnl.id in node_adnls:
+                return True
+        return False
+
+    def delete_custom_overlay_from_vc(self, name: str):
+        result = self.ton.validatorConsole.Run(f"delcustomoverlay {name}")
+        return 'success' in result
+
+    def add_custom_overlay_to_vc(self, config: dict):
+        if not self.check_node_eligible_for_custom_overlay(config):
+            self.ton.local.add_log(f"Node has no adnl address required for custom overlay {config.get('name')}", "debug")
+            return False
+        self.ton.local.add_log(f"Adding custom overlay {config.get('name')}", "debug")
+        path = self.ton.tempDir + f'/custom_overlay_{config["name"]}.json'
+        with open(path, 'w') as f:
+            json.dump(config, f)
+        result = self.ton.validatorConsole.Run(f"addcustomoverlay {path}")
+        return 'success' in result
+
+    def custom_overlays(self):
+        config = self.get_default_custom_overlay()
+        if config is not None:
+            self.ton.set_custom_overlay('default', config)
+        self.deploy_custom_overlays()
+
+    def deploy_custom_overlays(self):
+        result = self.ton.validatorConsole.Run("showcustomoverlays")
+        if 'unknown command' in result:
+            return  # node old version
+        names = []
+        for line in result.split('\n'):
+            if line.startswith('Overlay'):
+                names.append(line.split(' ')[1].replace('"', '').replace(':', ''))
+
+        config34 = self.ton.GetConfig34()
+        current_el_id = config34['startWorkTime']
+        current_vset = [i["adnlAddr"] for i in config34['validators']]
+
+        config36 = self.ton.GetConfig36()
+        next_el_id = config36['startWorkTime'] if config36['validators'] else 0
+        next_vset = [i["adnlAddr"] for i in config36['validators']]
+
+        for name in names:
+            # check that overlay still exists in mtc db
+            pure_name = name
+            suffix = name.split('_')[-1]
+            if suffix.startswith('elid') and suffix.split('elid')[-1].isdigit():  # probably election id
+                pure_name = '_'.join(name.split('_')[:-1])
+                el_id = int(suffix.split('elid')[-1])
+                if el_id not in (current_el_id, next_el_id):
+                    self.ton.local.add_log(f"Overlay {name} is not in current or next election, deleting", "debug")
+                    self.delete_custom_overlay_from_vc(name)  # delete overlay if election id is not in current or next election
+                    continue
+
+            if pure_name not in self.ton.get_custom_overlays():
+                self.ton.local.add_log(f"Overlay {name} ({pure_name}) is not in mtc db, deleting", "debug")
+                self.delete_custom_overlay_from_vc(name)  # delete overlay if it's not in mtc db
+
+        for name, config in self.ton.get_custom_overlays().items():
+            if name in names:
+                continue
+            if '@validators' in config:
+                new_name = name + '_elid' + str(current_el_id)
+                if new_name not in names:
+                    node_config = self.parse_config(new_name, config, current_vset)
+                    self.add_custom_overlay_to_vc(node_config)
+
+                if next_el_id != 0:
+                    new_name = name + '_elid' + str(next_el_id)
+                    if new_name not in names:
+                        node_config = self.parse_config(new_name, config, next_vset)
+                        self.add_custom_overlay_to_vc(node_config)
+            else:
+                node_config = self.parse_config(name, config)
+                self.add_custom_overlay_to_vc(node_config)
+
+    def get_default_custom_overlay(self):
+        if not self.ton.local.db.get('useDefaultCustomOverlays', True):
+            return None
+        network = self.ton.GetNetworkName()
+        default_url = 'https://ton-blockchain.github.io/fallback_custom_overlays.json'
+        url = self.ton.local.db.get('defaultCustomOverlaysUrl', default_url)
+        resp = requests.get(url)
+        if resp.status_code != 200:
+            self.ton.local.add_log(f"Failed to get default custom overlays from {url}", "error")
+            return None
+        config = resp.json()
+        return config.get(network)
+
+    def add_console_commands(self, console):
+        console.AddItem("add_custom_overlay", self.add_custom_overlay, self.local.translate("add_custom_overlay_cmd"))
+        console.AddItem("list_custom_overlays", self.list_custom_overlays, self.local.translate("list_custom_overlays_cmd"))
+        console.AddItem("delete_custom_overlay", self.delete_custom_overlay, self.local.translate("delete_custom_overlay_cmd"))

mytoncore/functions.py

@@ -5,11 +5,13 @@
 import psutil
 import time
 import json
+import base64
 import requests
 import subprocess
 
 from mytoncore.mytoncore import MyTonCore
 from mytoncore.utils import parse_db_stats
+from mytoninstaller.config import GetConfig
 from mypylib.mypylib import (
     b2mb,
     get_timestamp,
@@ -20,14 +22,15 @@
     thr_sleep,
     Dict
 )
+from mytoninstaller.node_args import get_node_args
 
 
 def Init(local):
     # Event reaction
     if ("-e" in sys.argv):
         x = sys.argv.index("-e")
-        eventName = sys.argv[x+1]
-        Event(local, eventName)
+        event_name = sys.argv[x+1]
+        Event(local, event_name)
     # end if
 
     local.run()
@@ -46,11 +49,13 @@ def Init(local):
 # end define
 
 
-def Event(local, eventName):
-    if eventName == "enableVC":
+def Event(local, event_name):
+    if event_name == "enableVC":
         EnableVcEvent(local)
-    elif eventName == "validator down":
+    elif event_name == "validator down":
         ValidatorDownEvent(local)
+    elif event_name == "enable_ton_storage_provider":
+        enable_ton_storage_provider_event(local)
     local.exit()
 # end define
 
@@ -78,6 +83,15 @@ def ValidatorDownEvent(local):
 # end define
 
 
+def enable_ton_storage_provider_event(local):
+    config_path = local.db.ton_storage.provider.config_path
+    config = GetConfig(path=config_path)
+    key_bytes = base64.b64decode(config.ProviderKey)
+    ton = MyTonCore(local)
+    ton.import_wallet_with_version(key_bytes[:32], version="v3r2", wallet_name="provider_wallet_001")
+#end define
+
+
 def Elections(local, ton):
     use_pool = ton.using_pool()
     use_liquid_staking = ton.using_liquid_staking()
@@ -460,6 +474,49 @@ def get_db_stats():
 # end define
 
 
+def get_cpu_name():
+    with open('/proc/cpuinfo') as f:
+        for line in f:
+            if line.strip():
+                if line.rstrip('\n').startswith('model name'):
+                    return line.rstrip('\n').split(':')[1].strip()
+    return None
+
+
+def is_host_virtual():
+    try:
+        with open('/sys/class/dmi/id/product_name') as f:
+            product_name = f.read().strip().lower()
+            if 'virtual' in product_name or 'kvm' in product_name or 'qemu' in product_name or 'vmware' in product_name:
+                return {'virtual': True, 'product_name': product_name}
+            return {'virtual': False, 'product_name': product_name}
+    except FileNotFoundError:
+        return {'virtual': None, 'product_name': None}
+
+
+def do_beacon_ping(host, count, timeout):
+    args = ['ping', '-c', str(count), '-W', str(timeout), host]
+    process = subprocess.run(args, stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
+    output = process.stdout.decode("utf-8")
+    avg = output.split('\n')[-2].split('=')[1].split('/')[1]
+    return float(avg)
+
+
+def get_pings_values():
+    return {
+        'beacon-eu-01.toncenter.com': do_beacon_ping('beacon-eu-01.toncenter.com', 5, 10),
+        'beacon-apac-01.toncenter.com': do_beacon_ping('beacon-apac-01.toncenter.com', 5, 10)
+    }
+
+
+def get_validator_disk_name():
+    process = subprocess.run("df -h /var/ton-work/ | sed -n '2 p' | awk '{print $1}'", stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3, shell=True)
+    output = process.stdout.decode("utf-8")
+    return output.strip()
+
+
 def Telemetry(local, ton):
     sendTelemetry = local.db.get("sendTelemetry")
     if sendTelemetry is not True:
@@ -484,6 +541,10 @@ def Telemetry(local, ton):
     data["uname"] = GetUname()
     data["vprocess"] = GetValidatorProcessInfo()
     data["dbStats"] = get_db_stats()
+    data["nodeArgs"] = get_node_args()
+    data["cpuInfo"] = {'cpuName': get_cpu_name(), 'virtual': is_host_virtual()}
+    data["validatorDiskName"] = get_validator_disk_name()
+    data["pings"] = get_pings_values()
     elections = local.try_function(ton.GetElectionEntries)
     complaints = local.try_function(ton.GetComplaints)
 
@@ -629,6 +690,10 @@ def General(local):
     local.start_cycle(Telemetry, sec=60, args=(local, ton, ))
     local.start_cycle(OverlayTelemetry, sec=7200, args=(local, ton, ))
     local.start_cycle(ScanLiteServers, sec=60, args=(local, ton,))
+
+    from modules.custom_overlays import CustomOverlayModule
+    local.start_cycle(CustomOverlayModule(ton, local).custom_overlays, sec=60, args=())
+
     thr_sleep()
 # end define