add gc import/ dir

Author: yungwine

modules/__init__.py

@@ -62,7 +62,8 @@ class Setting:
     'auto_backup': Setting('validator', None, 'Make validator backup every election'),
     'auto_backup_path': Setting('validator', '/tmp/mytoncore/auto_backups/', 'Path to store auto-backups'),
     'prometheus_url': Setting('prometheus', None, 'Prometheus pushgateway url'),
-    'onlyNode': Setting(None, None, 'MyTonCtrl will work only for collecting validator telemetry (if `sendTelemetry` is True), without participating in Elections and etc.')
+    'onlyNode': Setting(None, None, 'MyTonCtrl will work only for collecting validator telemetry (if `sendTelemetry` is True), without participating in Elections and etc.'),
+    'importGc': Setting(None, None, 'Delete imported archive blocks files. Restart mytoncore to apply this setting'),
 }
 
 

mytoncore/functions.py

@@ -602,6 +602,35 @@ def check_initial_sync(local, ton):
         return
 
 
+def gc_import(local, ton):
+    if not ton.local.db.get('importGc', False):
+        return
+    local.add_log("GC import is running", "debug")
+    import_path = '/var/ton-work/db/import'
+    files = os.listdir(import_path)
+    if not files:
+        local.add_log("No files left to import", "debug")
+        ton.local.db['importGc'] = False
+        return
+    try:
+        status = ton.GetValidatorStatus()
+        node_seqno = int(status.shardclientmasterchainseqno)
+    except Exception as e:
+        local.add_log(f"Failed to get shardclientmasterchainseqno: {e}", "warning")
+        return
+    removed = 0
+    for file in files:
+        file_seqno = int(file.split('.')[1])
+        if node_seqno > file_seqno + 101:
+            try:
+                os.remove(os.path.join(import_path, file))
+                removed += 1
+            except PermissionError:
+                local.add_log(f"Failed to remove file {file}: Permission denied", "error")
+                continue
+    local.add_log(f"Removed {removed} import files up to {node_seqno} seqno", "debug")
+
+
 def General(local):
     local.add_log("start General function", "debug")
     ton = MyTonCore(local)
@@ -642,6 +671,9 @@ def General(local):
     if ton.in_initial_sync():
         local.start_cycle(check_initial_sync, sec=120, args=(local, ton))
 
+    if ton.local.db.get('importGc'):
+        local.start_cycle(gc_import, sec=300, args=(local, ton))
+
     thr_sleep()
 # end define
 

mytoninstaller/settings.py

@@ -269,7 +269,7 @@ def download_archive_from_ts(local):
 	for bag in block_bags + master_block_bags:
 		subprocess.run(f'mv {downloads_path}/{bag["bag"]}/packages/*/* {import_dir}', shell=True)
 		# subprocess.run(['rm', '-rf', f"{downloads_path}/{bag['bag']}"])
-	subprocess.run(['rm', '-rf', downloads_path + '*'])
+	subprocess.run(f'rm -rf {downloads_path}*', shell=True)
 
 	stop_service(local, "ton_storage")  # stop TS
 	disable_service(local, "ton_storage")
@@ -285,6 +285,13 @@ def download_archive_from_ts(local):
 	if c['validator']['hardforks'] and c['validator']['hardforks'][-1]['seqno'] > block_from:
 		run_process_hardforks(local, block_from)
 
+	local.add_log(f"Changing permissions on imported files", "info")
+	subprocess.run(["chmod", "o+w", import_dir])
+	mconfig_path = local.buffer.mconfig_path
+	mconfig = GetConfig(path=mconfig_path)
+	mconfig.importGc = True
+	SetConfig(path=mconfig_path, data=mconfig)
+
 
 def run_process_hardforks(local, from_seqno: int):
 	script_path = os.path.join(local.buffer.mtc_src_dir, 'mytoninstaller', 'scripts', 'process_hardforks.py')

scripts/install.py

@@ -46,6 +46,9 @@ def validate_archive_blocks(value):
             return "Block number cannot be negative"
         elif not part.isdigit() and not is_valid_date_format(part):
             return "Incorrect date format, use YYYY-MM-DD"
+    if len(parts) == 2 and parts[1].isdigit() and parts[0].isdigit():
+        if int(parts[1]) < int(parts[0]):
+            return "End block seqno cannot be less than start block seqno"
     return True