Merge pull request #400 from yungwine/auto-backup

Auto backups

Author: igroman787

modules/__init__.py

@@ -58,7 +58,9 @@ class Setting:
     'debug': Setting(None, False, 'Debug mtc console mode. Prints Traceback on errors'),
     'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'),
     'BotToken': Setting('alert-bot', None, 'Alerting Telegram bot token'),
-    'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id')
+    'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id'),
+    'auto_backup': Setting('validator', None, 'Make validator backup every election'),
+    'auto_backup_path': Setting('validator', '/tmp/mytoncore/auto_backups/', 'Path to store auto-backups'),
 }
 
 

modules/backups.py

@@ -0,0 +1,76 @@
+import os
+import shutil
+import subprocess
+import time
+
+import pkg_resources
+
+from modules.module import MtcModule
+from mypylib.mypylib import color_print, ip2int, run_as_root, parse
+from mytoninstaller.config import get_own_ip
+
+
+class BackupModule(MtcModule):
+
+    def create_keyring(self, dir_name):
+        keyring_dir = dir_name + '/keyring'
+        self.ton.validatorConsole.Run(f'exportallprivatekeys {keyring_dir}')
+
+    def create_tmp_ton_dir(self):
+        result = self.ton.validatorConsole.Run("getconfig")
+        text = parse(result, "---------", "--------")
+        dir_name = self.ton.tempDir + f'/ton_backup_{int(time.time() * 1000)}'
+        dir_name_db = dir_name + '/db'
+        os.makedirs(dir_name_db)
+        with open(dir_name_db + '/config.json', 'w') as f:
+            f.write(text)
+        self.create_keyring(dir_name_db)
+        return dir_name
+
+    def create_backup(self, args):
+        if len(args) > 1:
+            color_print("{red}Bad args. Usage:{endc} create_backup [filename]")
+            return
+        tmp_dir = self.create_tmp_ton_dir()
+        command_args = ["-m", self.ton.local.buffer.my_work_dir, "-t", tmp_dir]
+        if len(args) == 1:
+            command_args += ["-d", args[0]]
+        backup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/create_backup.sh')
+        process = subprocess.run(["bash", backup_script_path] + command_args, timeout=5)
+
+        if process.returncode == 0:
+            color_print("create_backup - {green}OK{endc}")
+        else:
+            color_print("create_backup - {red}Error{endc}")
+        shutil.rmtree(tmp_dir)
+        return process.returncode
+    # end define
+
+    def restore_backup(self, args):
+        if len(args) == 0 or len(args) > 2:
+            color_print("{red}Bad args. Usage:{endc} restore_backup <filename> [-y]")
+            return
+        if '-y' not in args:
+            res = input(
+                f'This action will overwrite existing configuration with contents of backup archive, please make sure that donor node is not in operation prior to this action. Proceed [y/n]')
+            if res.lower() != 'y':
+                print('aborted.')
+                return
+        else:
+            args.pop(args.index('-y'))
+        print('Before proceeding, mtc will create a backup of current configuration.')
+        self.create_backup([])
+        ip = str(ip2int(get_own_ip()))
+        command_args = ["-m", self.ton.local.buffer.my_work_dir, "-n", args[0], "-i", ip]
+
+        restore_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/restore_backup.sh')
+        if run_as_root(["bash", restore_script_path] + command_args) == 0:
+            color_print("restore_backup - {green}OK{endc}")
+            self.local.exit()
+        else:
+            color_print("restore_backup - {red}Error{endc}")
+    # end define
+
+    def add_console_commands(self, console):
+        console.AddItem("create_backup", self.create_backup, self.local.translate("create_backup_cmd"))
+        console.AddItem("restore_backup", self.restore_backup, self.local.translate("restore_backup_cmd"))

mytoncore/mytoncore.py

@@ -1455,21 +1455,47 @@ def ElectionEntry(self, args=None):
 		self.local.add_log("ElectionEntry completed. Start work time: " + str(startWorkTime))
 
 		self.clear_tmp()
+		self.make_backup(startWorkTime)
 
 	#end define
 
-	def clear_tmp(self):
+	def clear_dir(self, dir_name):
 		start = time.time()
 		count = 0
 		week_ago = 60 * 60 * 24 * 7
-		dir = self.tempDir
-		for f in os.listdir(dir):
-			ts = os.path.getmtime(os.path.join(dir, f))
+		for f in os.listdir(dir_name):
+			ts = os.path.getmtime(os.path.join(dir_name, f))
 			if ts < time.time() - week_ago:
 				count += 1
-				os.remove(os.path.join(dir, f))
+				if os.path.isfile(os.path.join(dir_name, f)):
+					os.remove(os.path.join(dir_name, f))
+		self.local.add_log(f"Removed {count} old files from {dir_name} directory for {int(time.time() - start)} seconds", "info")
+
+	def clear_tmp(self):
+		self.clear_dir(self.tempDir)
 
-		self.local.add_log(f"Removed {count} old files from tmp dir for {int(time.time() - start)} seconds", "info")
+	def make_backup(self, election_id: str):
+		if not self.local.db.get("auto_backup"):
+			return
+		from modules.backups import BackupModule
+		module = BackupModule(self, self.local)
+		args = []
+		name = f"/mytonctrl_backup_elid{election_id}.zip"
+		backups_dir = self.tempDir + "/auto_backups"
+		if self.local.db.get("auto_backup_path"):
+			backups_dir = self.local.db.get("auto_backup_path")
+		os.makedirs(backups_dir, exist_ok=True)
+		args.append(backups_dir + name)
+		self.clear_dir(backups_dir)
+		exit_code = module.create_backup(args)
+		if exit_code != 0:
+			self.local.add_log(f"Backup failed with exit code {exit_code}", "error")
+			# try one more time
+			exit_code = module.create_backup(args)
+			if exit_code != 0:
+				self.local.add_log(f"Backup failed with exit code {exit_code}", "error")
+		if exit_code == 0:
+			self.local.add_log(f"Backup created successfully", "info")
 
 	def GetValidatorKeyByTime(self, startWorkTime, endWorkTime):
 		self.local.add_log("start GetValidatorKeyByTime function", "debug")

mytonctrl/mytonctrl.py

@@ -85,15 +85,17 @@ def inject_globals(func):
 	console.AddItem("get", inject_globals(GetSettings), local.translate("get_cmd"))
 	console.AddItem("set", inject_globals(SetSettings), local.translate("set_cmd"))
 	console.AddItem("rollback", inject_globals(rollback_to_mtc1), local.translate("rollback_cmd"))
-	console.AddItem("create_backup", inject_globals(create_backup), local.translate("create_backup_cmd"))
-	console.AddItem("restore_backup", inject_globals(restore_backup), local.translate("restore_backup_cmd"))
 
 	#console.AddItem("xrestart", inject_globals(Xrestart), local.translate("xrestart_cmd"))
 	#console.AddItem("xlist", inject_globals(Xlist), local.translate("xlist_cmd"))
 	#console.AddItem("gpk", inject_globals(GetPubKey), local.translate("gpk_cmd"))
 	#console.AddItem("ssoc", inject_globals(SignShardOverlayCert), local.translate("ssoc_cmd"))
 	#console.AddItem("isoc", inject_globals(ImportShardOverlayCert), local.translate("isoc_cmd"))
 
+	from modules.backups import BackupModule
+	module = BackupModule(ton, local)
+	module.add_console_commands(console)
+
 	from modules.custom_overlays import CustomOverlayModule
 	module = CustomOverlayModule(ton, local)
 	module.add_console_commands(console)
@@ -932,53 +934,6 @@ def disable_mode(local, ton, args):
 #end define
 
 
-def create_backup(local, ton, args):
-	if len(args) > 2:
-		color_print("{red}Bad args. Usage:{endc} create_backup [path_to_archive] [-y]")
-		return
-	if '-y' not in args:
-		res = input(f'Mytoncore service will be stopped for few seconds while backup is created, Proceed [y/n]?')
-		if res.lower() != 'y':
-			print('aborted.')
-			return
-	else:
-		args.pop(args.index('-y'))
-	command_args = ["-m", ton.local.buffer.my_work_dir]
-	if len(args) == 1:
-		command_args += ["-d", args[0]]
-	backup_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/create_backup.sh')
-	if run_as_root(["bash", backup_script_path] + command_args) == 0:
-		color_print("create_backup - {green}OK{endc}")
-	else:
-		color_print("create_backup - {red}Error{endc}")
-#end define
-
-
-def restore_backup(local, ton, args):
-	if len(args) == 0 or len(args) > 2:
-		color_print("{red}Bad args. Usage:{endc} restore_backup <path_to_archive> [-y]")
-		return
-	if '-y' not in args:
-		res = input(f'This action will overwrite existing configuration with contents of backup archive, please make sure that donor node is not in operation prior to this action. Proceed [y/n]')
-		if res.lower() != 'y':
-			print('aborted.')
-			return
-	else:
-		args.pop(args.index('-y'))
-	print('Before proceeding, mtc will create a backup of current configuration.')
-	create_backup(local, ton, ['-y'])
-	ip = str(ip2int(get_own_ip()))
-	command_args = ["-m", ton.local.buffer.my_work_dir, "-n", args[0], "-i", ip]
-
-	restore_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/restore_backup.sh')
-	if run_as_root(["bash", restore_script_path] + command_args) == 0:
-		color_print("restore_backup - {green}OK{endc}")
-		local.exit()
-	else:
-		color_print("restore_backup - {red}Error{endc}")
-#end define
-
-
 def Xrestart(inputArgs):
 	if len(inputArgs) < 2:
 		color_print("{red}Bad args. Usage:{endc} xrestart <timestamp> <args>")

mytonctrl/scripts/create_backup.sh

@@ -1,12 +1,16 @@
 dest="mytonctrl_backup_$(hostname)_$(date +%s).tar.gz"
 mtc_dir="$HOME/.local/share/mytoncore"
 user=$(logname)
+ton_dir="/var/ton-work"
+keys_dir="/var/ton-work/keys"
 # Get arguments
-while getopts d:m: flag
+while getopts d:m:t:k: flag
 do
 	case "${flag}" in
 		d) dest=${OPTARG};;
     m) mtc_dir=${OPTARG};;
+    t) ton_dir=${OPTARG};;
+    k) keys_dir=${OPTARG};;
     *)
         echo "Flag -${flag} is not recognized. Aborting"
         exit 1 ;;
@@ -16,32 +20,27 @@ done
 COLOR='\033[92m'
 ENDC='\033[0m'
 
-systemctl stop mytoncore
-
-echo -e "${COLOR}[1/4]${ENDC} Stopped mytoncore service"
-
-
-tmp_dir="/tmp/mytoncore/backup"
+tmp_dir="/tmp/mytoncore/backupv2"
 rm -rf $tmp_dir
 mkdir $tmp_dir
 mkdir $tmp_dir/db
 
-cp /var/ton-work/db/config.json ${tmp_dir}/db
-cp -r /var/ton-work/db/keyring ${tmp_dir}/db
-cp -r /var/ton-work/keys ${tmp_dir}
+cp $ton_dir/db/config.json ${tmp_dir}/db
+cp -r $ton_dir/db/keyring ${tmp_dir}/db
+cp -r $keys_dir ${tmp_dir}
 cp -r $mtc_dir $tmp_dir
 
 python3 -c "import json;f=open('${tmp_dir}/db/config.json');json.load(f);f.close()" || exit 1  # Check if config.json is copied correctly
+python3 -c "import json;f=open('${tmp_dir}/mytoncore/mytoncore.db');json.load(f);f.close()" || exit 2  # Check if mytoncore.db is copied correctly
 
-echo -e "${COLOR}[2/4]${ENDC} Copied files to ${tmp_dir}"
-
-systemctl start mytoncore
-
-echo -e "${COLOR}[3/4]${ENDC} Started mytoncore service"
+echo -e "${COLOR}[1/2]${ENDC} Copied files to ${tmp_dir}"
 
 tar -zcf $dest -C $tmp_dir .
 
 chown $user:$user $dest
 
-echo -e "${COLOR}[4/4]${ENDC} Backup successfully created in ${dest}!"
+echo -e "${COLOR}[2/2]${ENDC} Backup successfully created in ${dest}!"
+
+rm -rf $tmp_dir
+
 echo -e "If you wish to use archive package to migrate node to different machine please make sure to stop validator and mytoncore on donor (this) host prior to migration."