Merge branch 'dev' into backup

Author: yungwine

modules/prometheus.py

@@ -44,11 +44,11 @@ def get_validator_status_metrics(self, result: list):
             result.append(METRICS['master_out_of_sync'].to_format(status.masterchain_out_of_sync))
         if status.shardchain_out_of_sync is not None:
             result.append(METRICS['shard_out_of_sync'].to_format(status.shardchain_out_of_sync))
-        if status.masterchain_out_of_ser is not None:
+        if status.masterchain_out_of_ser is not None and status.stateserializermasterchainseqno != 0:
             result.append(METRICS['out_of_ser'].to_format(status.masterchain_out_of_ser))
         if status.masterchainblock is not None and status.gcmasterchainblock is not None:
             result.append(METRICS['celldb_gc_block'].to_format(status.masterchainblock - status.gcmasterchainblock))
-        if status.gcmasterchainblock is not None and status.last_deleted_mc_state is not None:
+        if status.gcmasterchainblock is not None and status.last_deleted_mc_state is not None and status.last_deleted_mc_state != 0:
             result.append(METRICS['celldb_gc_state'].to_format(status.gcmasterchainblock - status.last_deleted_mc_state))
         result.append(METRICS['vc_up'].to_format(int(is_working)))
 

mytoninstaller/config.py

@@ -43,17 +43,6 @@ def backup_config(local, config_path):
 #end define
 
 
-def BackupVconfig(local):
-	if local.buffer.only_mtc:
-		return
-	local.add_log("Backup validator config file 'config.json' to 'config.json.backup'", "debug")
-	vconfig_path = local.buffer.vconfig_path
-	backupPath = vconfig_path + ".backup"
-	args = ["cp", vconfig_path, backupPath]
-	subprocess.run(args)
-#end define
-
-
 def BackupMconfig(local):
 	local.add_log("Backup mytoncore config file 'mytoncore.db' to 'mytoncore.db.backup'", "debug")
 	mconfig_path = local.buffer.mconfig_path

mytoninstaller/mytoninstaller.py

@@ -34,7 +34,6 @@
 )
 from mytoninstaller.config import (
 	CreateLocalConfig,
-	BackupVconfig,
 	BackupMconfig,
 )
 
@@ -144,7 +143,7 @@ def set_node_argument(local, args):
 					"Examples: 'set_node_argument --archive-ttl 86400' or 'set_node_argument --archive-ttl -d' or 'set_node_argument -M' or 'set_node_argument --add-shard 0:2000000000000000 0:a000000000000000'")
 		return
 	arg_name = args[0]
-	args = [arg_name, args[1] if len(args) > 1 else ""]
+	args = [arg_name, " ".join(args[1:])]
 	script_path = pkg_resources.resource_filename('mytoninstaller.scripts', 'set_node_argument.py')
 	run_as_root(['python3', script_path] + args)
 	color_print("set_node_argument - {green}OK{endc}")
@@ -295,7 +294,6 @@ def General(local, console):
 	FirstNodeSettings(local)
 	EnableValidatorConsole(local)
 	EnableLiteServer(local)
-	BackupVconfig(local)
 	BackupMconfig(local)
 	CreateSymlinks(local)
 	EnableMode(local)

mytoninstaller/node_args.py

@@ -2,8 +2,8 @@
 
 def get_validator_service():
     path = '/etc/systemd/system/validator.service'
-    with open(path, 'r') as f:
-        return f.read()
+    with open(path, 'r') as file:
+        return file.read()
 #end define
 
 
@@ -14,43 +14,19 @@ def get_node_start_command():
             return line.split('=')[1].strip()
 #end define
 
-
-"""
-def get_node_args(command: str = None):
-    if command is None:
-        command = get_node_start_command()
-    result = []
-    key = ''
-    for c in command.split(' ')[1:]:
-        if c.startswith('--') or c.startswith('-'):
-            if key:
-                result.append([key, ''])
-            key = c
-        elif key:
-            result.append([key, c])
-            key = ''
-    if key:
-        result.append([key, ''])
-    return result
-#end define
-"""
-
-
-def get_node_args(command: str = None):
-    if command is None:
-        command = get_node_start_command()
-    result = {}  # {key: [value1, value2]}
-    key = ''
-    for c in command.split(' ')[1:]:
-        if c.startswith('--') or c.startswith('-'):
-            if key:
-                result[key] = result.get(key, []) + ['']
-            key = c
-        elif key:
-            result[key] = result.get(key, []) + [c]
-            key = ''
-    if key:
-        result[key] = result.get(key, []) + ['']
+def get_node_args(start_command: str = None):
+    if start_command is None:
+        start_command = get_node_start_command()
+    #end if
+
+    result = dict() # {key: [value1, value2]}
+    node_args = start_command.split(' ')[1:]
+    key = None
+    for item in node_args:
+        if item.startswith('-'):
+            key = item
+            result[key] = list()
+        else:
+            result[key].append(item)
     return result
 #end define
-

mytoninstaller/scripts/set_node_argument.py

@@ -11,23 +11,35 @@ def set_node_arg(arg_name: str, arg_value: str = ''):
     """
     assert arg_name.startswith('-'), 'arg_name must start with "-" or "--"'
     service = get_validator_service()
-    command = get_node_start_command()
-    if command.split(' ')[0] != '/usr/bin/ton/validator-engine/validator-engine':
-        raise Exception('Invalid node start command in service file')
-    if command is None:
+    start_command = get_node_start_command()
+    if start_command is None:
         raise Exception('Cannot find node start command in service file')
-    args = get_node_args(command)
+    first_arg = start_command.split(' ')[0]
+    if first_arg != '/usr/bin/ton/validator-engine/validator-engine':
+        raise Exception('Invalid node start command in service file')
+    #end if
+    
+    node_args = get_node_args(start_command)
     if arg_value == '-d':
-        args.pop(arg_name, None)
+        node_args.pop(arg_name, None)
     else:
         if ' ' in arg_value:
-            args[arg_name] = arg_value.split()
+            node_args[arg_name] = arg_value.split()
         else:
-            args[arg_name] = [arg_value]
-    new_command = command.split(' ')[0] + ' ' + ' '.join([f'{k} {v}' for k, vs in args.items() for v in vs])
-    new_service = service.replace(command, new_command)
-    with open('/etc/systemd/system/validator.service', 'w') as f:
-        f.write(new_service)
+            node_args[arg_name] = [arg_value]
+    #end if
+
+    buffer = list()
+    buffer.append(first_arg)
+    for key, value_list in node_args.items():
+        if len(value_list) == 0:
+            buffer.append(f"{key}")
+        for value in value_list:
+            buffer.append(f"{key} {value}")
+    new_start_command = ' '.join(buffer)
+    new_service = service.replace(start_command, new_start_command)
+    with open('/etc/systemd/system/validator.service', 'w') as file:
+        file.write(new_service)
     restart_node()
 #end define
 

mytoninstaller/settings.py

@@ -66,6 +66,13 @@ def FirstNodeSettings(local):
 	# Прописать автозагрузку
 	cpus = psutil.cpu_count() - 1
 	cmd = f"{validatorAppPath} --threads {cpus} --daemonize --global-config {globalConfigPath} --db {ton_db_dir} --logname {tonLogPath} --archive-ttl {archive_ttl} --verbosity 1"
+
+	if os.getenv('ADD_SHARD'):
+		add_shard = os.getenv('ADD_SHARD')
+		cmd += f' -M'
+		for shard in add_shard.split():
+			cmd += f' --add-shard {shard}'
+
 	add2systemd(name="validator", user=vuser, start=cmd) # post="/usr/bin/python3 /usr/src/mytonctrl/mytoncore.py -e \"validator down\""
 
 	# Получить внешний ip адрес

scripts/install.py

@@ -38,6 +38,13 @@ def run_cli():
             "dump",
             message="Do you want to download blockchain's dump? "
                     "This reduces synchronization time but requires to download a large file",
+        ),
+        inquirer.Text(
+            "add-shard",
+            message="Set shards node will sync. Skip to sync all shards. "
+                    "Format: <workchain>:<shard>. Divide multiple shards with space. "
+                    "Example: `0:2000000000000000 0:6000000000000000`",
+            validate=lambda _, x: not x or all([":" in i for i in x.split()])
         )
     ]
 
@@ -51,6 +58,7 @@ def parse_args(answers: dict):
     network = answers["network"].lower()
     config = answers["config"]
     archive_ttl = answers["archive-ttl"]
+    add_shard = answers["add-shard"]
     validator_mode = answers["validator-mode"]
     dump = answers["dump"]
 
@@ -61,6 +69,8 @@ def parse_args(answers: dict):
 
     if archive_ttl:
         os.putenv('ARCHIVE_TTL', archive_ttl)  # set env variable
+    if add_shard:
+        os.putenv('ADD_SHARD', add_shard)
 
     if validator_mode and validator_mode not in ('Skip', 'Validator wallet'):
         if validator_mode == 'Nominator pool':

scripts/install.sh

@@ -88,7 +88,8 @@ fi
 if [ "${mode}" = "none" ] && [ "$backup" = "none" ]; then  # no mode or backup was provided
     echo "Running cli installer"
     wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/install.py
-    pip3 install inquirer==3.4.0
+    python3 -m pip install --upgrade pip
+    pip3 install inquirer==3.4.0 --break-system-packages
     python3 install.py
     exit
 fi
@@ -123,6 +124,16 @@ if [[ "$OSTYPE" =~ darwin.* ]]; then
     mkdir -p ${SOURCES_DIR}
 fi
 
+
+if [ ! -f ~/.config/pip/pip.conf ]; then  # create pip config
+    mkdir -p ~/.config/pip
+cat > ~/.config/pip/pip.conf <<EOF
+[global]
+break-system-packages = true
+EOF
+fi
+
+
 # check TON components
 file1=${BIN_DIR}/ton/crypto/fift
 file2=${BIN_DIR}/ton/lite-client/lite-client