From be888953310931384052cc883712e5d11b75509c Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Wed, 21 Sep 2022 16:33:02 +0530 Subject: [PATCH 01/10] Removed VSCode Files --- VMBackup/UpgradeLog.htm | 281 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100644 VMBackup/UpgradeLog.htm diff --git a/VMBackup/UpgradeLog.htm b/VMBackup/UpgradeLog.htm new file mode 100644 index 000000000..0f5657e6e --- /dev/null +++ b/VMBackup/UpgradeLog.htm @@ -0,0 +1,281 @@ + + + + Migration Report +

+ Migration Report - VMBackup

Overview

ProjectPathErrorsWarningsMessages
VMBackupVMBackup.pyproj021
SolutionVMBackup.sln012

Solution and projects

\ No newline at end of file From b4507ec2d66bc2c9172c1c8859c7b3c1e8a4a502 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Wed, 19 Oct 2022 16:24:11 +0530 Subject: [PATCH 02/10] AddedChecks --- VMBackup/main/MachineIdentity.py | 20 +++++++++++++------- VMBackup/main/Utils/HandlerUtil.py | 9 +++++---- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/VMBackup/main/MachineIdentity.py b/VMBackup/main/MachineIdentity.py index dbccba9d7..95bbd947b 100644 --- a/VMBackup/main/MachineIdentity.py +++ b/VMBackup/main/MachineIdentity.py @@ -20,24 +20,30 @@ import subprocess import xml import xml.dom.minidom +import traceback class MachineIdentity: def __init__(self): self.store_identity_file = './machine_identity_FD76C85E-406F-4CFA-8EB0-CF18B123365C' def current_identity(self): - file = open("/var/lib/waagent/HostingEnvironmentConfig.xml",'r') - xmlText = file.read() - dom = xml.dom.minidom.parseString(xmlText) - deployment = dom.getElementsByTagName("Role") - identity=deployment[0].getAttribute("guid") + try: + identity = None + file = open("/var/lib/waagent/HostingEnvironmentConfig.xml",'r') + xmlText = file.read() + dom = xml.dom.minidom.parseString(xmlText) + deployment = dom.getElementsByTagName("Role") + identity=deployment[0].getAttribute("guid") + except Exception as e: + errorMsg = "Unable to open file, error: %s, stack trace: %s" % (str(e), traceback.format_exc()) return identity def save_identity(self): file = open(self.store_identity_file,'w') machine_identity = self.current_identity() - file.write(machine_identity) - file.close() + if( machine_identity != None ): + file.write(machine_identity) + file.close() def stored_identity(self): identity_stored = None diff --git a/VMBackup/main/Utils/HandlerUtil.py b/VMBackup/main/Utils/HandlerUtil.py index c1139764b..59ea89155 100755 --- a/VMBackup/main/Utils/HandlerUtil.py +++ b/VMBackup/main/Utils/HandlerUtil.py @@ -413,10 +413,11 @@ def get_machine_id(self): file_pointer.close() else: mi = MachineIdentity() - machine_id = mi.stored_identity()[1:-1] - file_pointer = open(machine_id_file, "w") - file_pointer.write(machine_id) - file_pointer.close() + if(mi.stored_identity() != None): + machine_id = mi.stored_identity()[1:-1] + file_pointer = open(machine_id_file, "w") + file_pointer.write(machine_id) + file_pointer.close() except Exception as e: errMsg = 'Failed to retrieve the unique machine id with error: %s, stack trace: %s' % (str(e), traceback.format_exc()) self.log(errMsg, 'Error') From 69343cf1469175a6b14e6318f8ac8d658011d50d Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Fri, 21 Oct 2022 12:05:14 +0530 Subject: [PATCH 03/10] UpdatedForLoggingTheError --- VMBackup/main/MachineIdentity.py | 10 ++++------ VMBackup/main/Utils/HandlerUtil.py | 18 +++++++++++------ VMBackup/main/handle.py | 32 +++++++++++++++++------------- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/VMBackup/main/MachineIdentity.py b/VMBackup/main/MachineIdentity.py index 95bbd947b..d7b6a2b1d 100644 --- a/VMBackup/main/MachineIdentity.py +++ b/VMBackup/main/MachineIdentity.py @@ -20,22 +20,20 @@ import subprocess import xml import xml.dom.minidom -import traceback class MachineIdentity: def __init__(self): self.store_identity_file = './machine_identity_FD76C85E-406F-4CFA-8EB0-CF18B123365C' def current_identity(self): - try: - identity = None + identity = None + if os.path.exists("/var/lib/waagent/HostingEnvironmentConfig.xml"): file = open("/var/lib/waagent/HostingEnvironmentConfig.xml",'r') xmlText = file.read() dom = xml.dom.minidom.parseString(xmlText) deployment = dom.getElementsByTagName("Role") identity=deployment[0].getAttribute("guid") - except Exception as e: - errorMsg = "Unable to open file, error: %s, stack trace: %s" % (str(e), traceback.format_exc()) + file.close() return identity def save_identity(self): @@ -43,7 +41,7 @@ def save_identity(self): machine_identity = self.current_identity() if( machine_identity != None ): file.write(machine_identity) - file.close() + file.close() def stored_identity(self): identity_stored = None diff --git a/VMBackup/main/Utils/HandlerUtil.py b/VMBackup/main/Utils/HandlerUtil.py index 59ea89155..3cc2c4dd5 100755 --- a/VMBackup/main/Utils/HandlerUtil.py +++ b/VMBackup/main/Utils/HandlerUtil.py @@ -403,24 +403,30 @@ def set_value_to_configfile(self, key, value): def get_machine_id(self): machine_id_file = "/etc/azure/machine_identity_FD76C85E-406F-4CFA-8EB0-CF18B123358B" machine_id = "" + self.file_pointer = None try: if not os.path.exists(os.path.dirname(machine_id_file)): os.makedirs(os.path.dirname(machine_id_file)) if os.path.exists(machine_id_file): - file_pointer = open(machine_id_file, "r") - machine_id = file_pointer.readline() - file_pointer.close() + self.file_pointer = open(machine_id_file, "r") + machine_id = self.file_pointer.readline() + self.file_pointer.close() + else: mi = MachineIdentity() if(mi.stored_identity() != None): machine_id = mi.stored_identity()[1:-1] - file_pointer = open(machine_id_file, "w") - file_pointer.write(machine_id) - file_pointer.close() + self.file_pointer = open(machine_id_file, "w") + self.file_pointer.write(machine_id) + self.file_pointer.close() except Exception as e: errMsg = 'Failed to retrieve the unique machine id with error: %s, stack trace: %s' % (str(e), traceback.format_exc()) self.log(errMsg, 'Error') + finally : + if self.file_pointer != None : + if self.file_pointer.closed == False : + self.file_pointer.close() self.log("Unique Machine Id : {0}".format(machine_id)) return machine_id diff --git a/VMBackup/main/handle.py b/VMBackup/main/handle.py index 8de4f4a0d..ab374b672 100755 --- a/VMBackup/main/handle.py +++ b/VMBackup/main/handle.py @@ -130,7 +130,7 @@ def get_status_to_report(status, status_code, message, snapshot_info = None): file_report_msg = None try: if total_used_size == -1 : - sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , hutil = hutil, logger = backup_logger , para_parser = para_parser) + sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , logger = backup_logger , para_parser = para_parser) total_used_size,size_calculation_failed = sizeCalculation.get_total_used_size() number_of_blobs = len(para_parser.includeLunList) maximum_possible_size = number_of_blobs * 1099511627776 @@ -236,20 +236,24 @@ def daemon(): hutil.do_parse_context('Executing', configSeqNo) try: - backup_logger.log('starting daemon initially', True) + backup_logger.log('starting daemon', True) backup_logger.log("patch_class_name: "+str(patch_class_name)+" and orig_distro: "+str(orig_distro),True) # handle the restoring scenario. mi = MachineIdentity() stored_identity = mi.stored_identity() - if(stored_identity is None): - mi.save_identity() - else: - current_identity = mi.current_identity() - if(current_identity != stored_identity): - current_seq_no = -1 - backup_logger.log("machine identity not same, set current_seq_no to " + str(current_seq_no) + " " + str(stored_identity) + " " + str(current_identity), True) - hutil.set_last_seq(current_seq_no) + try: + if(stored_identity is None): mi.save_identity() + else: + current_identity = mi.current_identity() + if(current_identity != stored_identity): + current_seq_no = -1 + backup_logger.log("machine identity not same, set current_seq_no to " + str(current_seq_no) + " " + str(stored_identity) + " " + str(current_identity), True) + hutil.set_last_seq(current_seq_no) + mi.save_identity() + except Exception as e: + errMsg = "Unable to open file, error: %s, stack trace: %s" % (str(e), traceback.format_exc()) + backup_logger.log(errMsg, True, 'Error') except Exception as e: errMsg = 'Failed to validate sequence number with error: %s, stack trace: %s' % (str(e), traceback.format_exc()) backup_logger.log(errMsg, True, 'Error') @@ -300,7 +304,7 @@ def daemon(): try: # we need to freeze the file system first - backup_logger.log('starting daemon for freezing the file system', True) + backup_logger.log('starting daemon', True) """ protectedSettings is the privateConfig passed from Powershell. WATCHOUT that, the _context_config are using the most freshest timestamp. @@ -335,7 +339,7 @@ def daemon(): commandToExecute = para_parser.commandToExecute #validate all the required parameter here - backup_logger.log('The command '+ commandToExecute+ ' is being validated',True) + backup_logger.log(commandToExecute,True) if(CommonVariables.iaas_install_command in commandToExecute.lower()): backup_logger.log('install succeed.',True) run_status = 'success' @@ -350,7 +354,7 @@ def daemon(): error_msg = 'required field empty or not correct' backup_logger.log(error_msg, True, 'Error') else: - backup_logger.log('commandToExecute for backup is ' + commandToExecute, True) + backup_logger.log('commandToExecute is ' + commandToExecute, True) """ make sure the log is not doing when the file system is freezed. """ @@ -365,7 +369,7 @@ def daemon(): backup_logger.commit_to_blob(para_parser.logsBlobUri) else: backup_logger.log("the logs blob uri is not there, so do not upload log.") - backup_logger.log('commandToExecute after commiting the blob is ' + commandToExecute, True) + backup_logger.log('commandToExecute is ' + commandToExecute, True) workload_patch = WorkloadPatch.WorkloadPatch(backup_logger) #new flow only if workload name is present in workload.conf From d88f30eb6a8d0d500d200333233e2219e742809b Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Fri, 21 Oct 2022 12:28:18 +0530 Subject: [PATCH 04/10] UpdatedCode --- VMBackup/main/handle.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/VMBackup/main/handle.py b/VMBackup/main/handle.py index ab374b672..0baecb53e 100755 --- a/VMBackup/main/handle.py +++ b/VMBackup/main/handle.py @@ -130,7 +130,7 @@ def get_status_to_report(status, status_code, message, snapshot_info = None): file_report_msg = None try: if total_used_size == -1 : - sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , logger = backup_logger , para_parser = para_parser) + sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , hutil = hutil, logger = backup_logger , para_parser = para_parser) total_used_size,size_calculation_failed = sizeCalculation.get_total_used_size() number_of_blobs = len(para_parser.includeLunList) maximum_possible_size = number_of_blobs * 1099511627776 @@ -236,7 +236,7 @@ def daemon(): hutil.do_parse_context('Executing', configSeqNo) try: - backup_logger.log('starting daemon', True) + backup_logger.log('starting daemon initially', True) backup_logger.log("patch_class_name: "+str(patch_class_name)+" and orig_distro: "+str(orig_distro),True) # handle the restoring scenario. mi = MachineIdentity() @@ -304,7 +304,7 @@ def daemon(): try: # we need to freeze the file system first - backup_logger.log('starting daemon', True) + backup_logger.log('starting daemon for freezing the file system', True) """ protectedSettings is the privateConfig passed from Powershell. WATCHOUT that, the _context_config are using the most freshest timestamp. @@ -339,7 +339,7 @@ def daemon(): commandToExecute = para_parser.commandToExecute #validate all the required parameter here - backup_logger.log(commandToExecute,True) + backup_logger.log('The command '+ commandToExecute+ ' is being validated',True) if(CommonVariables.iaas_install_command in commandToExecute.lower()): backup_logger.log('install succeed.',True) run_status = 'success' @@ -354,7 +354,7 @@ def daemon(): error_msg = 'required field empty or not correct' backup_logger.log(error_msg, True, 'Error') else: - backup_logger.log('commandToExecute is ' + commandToExecute, True) + backup_logger.log('commandToExecute for backup is ' + commandToExecute, True) """ make sure the log is not doing when the file system is freezed. """ @@ -369,7 +369,7 @@ def daemon(): backup_logger.commit_to_blob(para_parser.logsBlobUri) else: backup_logger.log("the logs blob uri is not there, so do not upload log.") - backup_logger.log('commandToExecute is ' + commandToExecute, True) + backup_logger.log('commandToExecute after commiting the blob is ' + commandToExecute, True) workload_patch = WorkloadPatch.WorkloadPatch(backup_logger) #new flow only if workload name is present in workload.conf From 38d75dc70835b3921cb711df5a57ece7b506f8f1 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Wed, 26 Oct 2022 11:21:01 +0530 Subject: [PATCH 05/10] Revert "UpdatedCode" This reverts commit d88f30eb6a8d0d500d200333233e2219e742809b. --- VMBackup/main/handle.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/VMBackup/main/handle.py b/VMBackup/main/handle.py index 0baecb53e..ab374b672 100755 --- a/VMBackup/main/handle.py +++ b/VMBackup/main/handle.py @@ -130,7 +130,7 @@ def get_status_to_report(status, status_code, message, snapshot_info = None): file_report_msg = None try: if total_used_size == -1 : - sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , hutil = hutil, logger = backup_logger , para_parser = para_parser) + sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , logger = backup_logger , para_parser = para_parser) total_used_size,size_calculation_failed = sizeCalculation.get_total_used_size() number_of_blobs = len(para_parser.includeLunList) maximum_possible_size = number_of_blobs * 1099511627776 @@ -236,7 +236,7 @@ def daemon(): hutil.do_parse_context('Executing', configSeqNo) try: - backup_logger.log('starting daemon initially', True) + backup_logger.log('starting daemon', True) backup_logger.log("patch_class_name: "+str(patch_class_name)+" and orig_distro: "+str(orig_distro),True) # handle the restoring scenario. mi = MachineIdentity() @@ -304,7 +304,7 @@ def daemon(): try: # we need to freeze the file system first - backup_logger.log('starting daemon for freezing the file system', True) + backup_logger.log('starting daemon', True) """ protectedSettings is the privateConfig passed from Powershell. WATCHOUT that, the _context_config are using the most freshest timestamp. @@ -339,7 +339,7 @@ def daemon(): commandToExecute = para_parser.commandToExecute #validate all the required parameter here - backup_logger.log('The command '+ commandToExecute+ ' is being validated',True) + backup_logger.log(commandToExecute,True) if(CommonVariables.iaas_install_command in commandToExecute.lower()): backup_logger.log('install succeed.',True) run_status = 'success' @@ -354,7 +354,7 @@ def daemon(): error_msg = 'required field empty or not correct' backup_logger.log(error_msg, True, 'Error') else: - backup_logger.log('commandToExecute for backup is ' + commandToExecute, True) + backup_logger.log('commandToExecute is ' + commandToExecute, True) """ make sure the log is not doing when the file system is freezed. """ @@ -369,7 +369,7 @@ def daemon(): backup_logger.commit_to_blob(para_parser.logsBlobUri) else: backup_logger.log("the logs blob uri is not there, so do not upload log.") - backup_logger.log('commandToExecute after commiting the blob is ' + commandToExecute, True) + backup_logger.log('commandToExecute is ' + commandToExecute, True) workload_patch = WorkloadPatch.WorkloadPatch(backup_logger) #new flow only if workload name is present in workload.conf From e643dd6f3975268c74f80dbe86a57c07dc1c3394 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Wed, 26 Oct 2022 11:27:18 +0530 Subject: [PATCH 06/10] Revert "updated lsblk command by using more flags" This reverts commit 9ccc15571438f4eda8b0465b3e9c83416777e7b5. --- VMBackup/main/Utils/SizeCalculation.py | 35 +++++++++++++------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/VMBackup/main/Utils/SizeCalculation.py b/VMBackup/main/Utils/SizeCalculation.py index 22238ab49..1a9e2dc82 100644 --- a/VMBackup/main/Utils/SizeCalculation.py +++ b/VMBackup/main/Utils/SizeCalculation.py @@ -53,16 +53,17 @@ def __init__(self,patching, hutil, logger,para_parser): self.root_devices = [] self.root_mount_points = ['/' , '/boot/efi'] try: - self.lsscsi_list = (os.popen(self.command).read()).splitlines() + self.output_lsscsi = (os.popen(self.command).read()) + self.lsscsi_list = self.output_lsscsi.splitlines() except Exception as e: error_msg = "Failed to execute the command lsscsi because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.output_lsscsi = "" self.lsscsi_list = [] try: - self.output_lsblk = json.loads(os.popen("lsblk --json --output name,mountpoint").read()) + self.output_lsblk = json.loads(os.popen("lsblk --json").read()) except: - error_msg = "Failed to execute the command lsblk --json --output name,mountpoint because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) + error_msg = "Failed to execute the command lsblk --json because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.output_lsblk = {} self.devicesToInclude = [] #partitions to be included @@ -74,6 +75,7 @@ def __init__(self,patching, hutil, logger,para_parser): def get_loop_devices(self): global disk_util disk_util = DiskUtil.get_instance(patching = self.patching,logger = self.logger) + self.logger.log("disk_util: {0}".format(str(disk_util)),True) if len(self.file_systems_info) == 0 : self.file_systems_info = disk_util.get_mount_file_systems() self.logger.log("file_systems list : ",True) @@ -93,16 +95,16 @@ def disk_list_for_billing(self): # item_split is the list of elements present in the one row of the cmd sudo lsscsi self.item_split = item.split() #storing the corresponding device name from the list - device_name = self.item_split[len(self.item_split)-1] + self.device_name = self.item_split[len(self.item_split)-1] for device in self.root_devices : - if device_name in device : + if self.device_name in device : lunNumber = -1 # Changing the Lun# of OS Disk to -1 if lunNumber in self.includedLunList : - self.disksToBeIncluded.append(device_name) - self.logger.log("LUN Number {0}, disk {1}".format(lunNumber,device_name)) + self.disksToBeIncluded.append(self.device_name) + self.logger.log("LUN Number {0}, disk {1}".format(lunNumber,self.device_name)) self.logger.log("Disks to be included {0}".format(self.disksToBeIncluded)) return self.disksToBeIncluded @@ -140,18 +142,17 @@ def device_list_for_billing(self): Sample output for lsblk --json command { "blockdevices": [ - {"name": "sda", "mountpoint": null, - "children": [ - {"name": "sda1", "mountpoint": null} - ] + {"name": "sdb", "maj:min": "8:16", "rm": "0", "size": "32G", "ro": "0", "type": "disk", "mountpoint": null, + "children": [ + {"name": "sdb1", "maj:min": "8:17", "rm": "0", "size": "976.6M", "ro": "0", "type": "part", "mountpoint": null} + ] }, - {"name": "sdb", "mountpoint": null}, - {"name": "sdc", "mountpoint": null, - "children": [ - {"name": "sdc1", "mountpoint": null} - ] + {"name": "sdc", "maj:min": "8:32", "rm": "0", "size": "128G", "ro": "0", "type": "disk", "mountpoint": null, + "children": [ + {"name": "data--vg01-data--lv01", "maj:min": "253:0", "rm": "0", "size": "16G", "ro": "0", "type": "lvm", "mountpoint": null} + ] }, - ... + .... ] } ''' From 752e62767ea461471d6931ced693fb46fb6536ea Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Wed, 26 Oct 2022 11:36:42 +0530 Subject: [PATCH 07/10] reverting to master --- VMBackup/UpgradeLog.htm | 281 ------------------------- VMBackup/main/MachineIdentity.py | 16 +- VMBackup/main/Utils/HandlerUtil.py | 21 +- VMBackup/main/Utils/SizeCalculation.py | 35 ++- VMBackup/main/handle.py | 32 ++- 5 files changed, 44 insertions(+), 341 deletions(-) delete mode 100644 VMBackup/UpgradeLog.htm diff --git a/VMBackup/UpgradeLog.htm b/VMBackup/UpgradeLog.htm deleted file mode 100644 index 0f5657e6e..000000000 --- a/VMBackup/UpgradeLog.htm +++ /dev/null @@ -1,281 +0,0 @@ - - - - Migration Report -

- Migration Report - VMBackup

Overview

ProjectPathErrorsWarningsMessages
VMBackupVMBackup.pyproj021
SolutionVMBackup.sln012

Solution and projects

\ No newline at end of file diff --git a/VMBackup/main/MachineIdentity.py b/VMBackup/main/MachineIdentity.py index d7b6a2b1d..dbccba9d7 100644 --- a/VMBackup/main/MachineIdentity.py +++ b/VMBackup/main/MachineIdentity.py @@ -26,21 +26,17 @@ def __init__(self): self.store_identity_file = './machine_identity_FD76C85E-406F-4CFA-8EB0-CF18B123365C' def current_identity(self): - identity = None - if os.path.exists("/var/lib/waagent/HostingEnvironmentConfig.xml"): - file = open("/var/lib/waagent/HostingEnvironmentConfig.xml",'r') - xmlText = file.read() - dom = xml.dom.minidom.parseString(xmlText) - deployment = dom.getElementsByTagName("Role") - identity=deployment[0].getAttribute("guid") - file.close() + file = open("/var/lib/waagent/HostingEnvironmentConfig.xml",'r') + xmlText = file.read() + dom = xml.dom.minidom.parseString(xmlText) + deployment = dom.getElementsByTagName("Role") + identity=deployment[0].getAttribute("guid") return identity def save_identity(self): file = open(self.store_identity_file,'w') machine_identity = self.current_identity() - if( machine_identity != None ): - file.write(machine_identity) + file.write(machine_identity) file.close() def stored_identity(self): diff --git a/VMBackup/main/Utils/HandlerUtil.py b/VMBackup/main/Utils/HandlerUtil.py index 3cc2c4dd5..c1139764b 100755 --- a/VMBackup/main/Utils/HandlerUtil.py +++ b/VMBackup/main/Utils/HandlerUtil.py @@ -403,30 +403,23 @@ def set_value_to_configfile(self, key, value): def get_machine_id(self): machine_id_file = "/etc/azure/machine_identity_FD76C85E-406F-4CFA-8EB0-CF18B123358B" machine_id = "" - self.file_pointer = None try: if not os.path.exists(os.path.dirname(machine_id_file)): os.makedirs(os.path.dirname(machine_id_file)) if os.path.exists(machine_id_file): - self.file_pointer = open(machine_id_file, "r") - machine_id = self.file_pointer.readline() - self.file_pointer.close() - + file_pointer = open(machine_id_file, "r") + machine_id = file_pointer.readline() + file_pointer.close() else: mi = MachineIdentity() - if(mi.stored_identity() != None): - machine_id = mi.stored_identity()[1:-1] - self.file_pointer = open(machine_id_file, "w") - self.file_pointer.write(machine_id) - self.file_pointer.close() + machine_id = mi.stored_identity()[1:-1] + file_pointer = open(machine_id_file, "w") + file_pointer.write(machine_id) + file_pointer.close() except Exception as e: errMsg = 'Failed to retrieve the unique machine id with error: %s, stack trace: %s' % (str(e), traceback.format_exc()) self.log(errMsg, 'Error') - finally : - if self.file_pointer != None : - if self.file_pointer.closed == False : - self.file_pointer.close() self.log("Unique Machine Id : {0}".format(machine_id)) return machine_id diff --git a/VMBackup/main/Utils/SizeCalculation.py b/VMBackup/main/Utils/SizeCalculation.py index 1a9e2dc82..22238ab49 100644 --- a/VMBackup/main/Utils/SizeCalculation.py +++ b/VMBackup/main/Utils/SizeCalculation.py @@ -53,17 +53,16 @@ def __init__(self,patching, hutil, logger,para_parser): self.root_devices = [] self.root_mount_points = ['/' , '/boot/efi'] try: - self.output_lsscsi = (os.popen(self.command).read()) - self.lsscsi_list = self.output_lsscsi.splitlines() + self.lsscsi_list = (os.popen(self.command).read()).splitlines() except Exception as e: error_msg = "Failed to execute the command lsscsi because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.output_lsscsi = "" self.lsscsi_list = [] try: - self.output_lsblk = json.loads(os.popen("lsblk --json").read()) + self.output_lsblk = json.loads(os.popen("lsblk --json --output name,mountpoint").read()) except: - error_msg = "Failed to execute the command lsblk --json because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) + error_msg = "Failed to execute the command lsblk --json --output name,mountpoint because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.output_lsblk = {} self.devicesToInclude = [] #partitions to be included @@ -75,7 +74,6 @@ def __init__(self,patching, hutil, logger,para_parser): def get_loop_devices(self): global disk_util disk_util = DiskUtil.get_instance(patching = self.patching,logger = self.logger) - self.logger.log("disk_util: {0}".format(str(disk_util)),True) if len(self.file_systems_info) == 0 : self.file_systems_info = disk_util.get_mount_file_systems() self.logger.log("file_systems list : ",True) @@ -95,16 +93,16 @@ def disk_list_for_billing(self): # item_split is the list of elements present in the one row of the cmd sudo lsscsi self.item_split = item.split() #storing the corresponding device name from the list - self.device_name = self.item_split[len(self.item_split)-1] + device_name = self.item_split[len(self.item_split)-1] for device in self.root_devices : - if self.device_name in device : + if device_name in device : lunNumber = -1 # Changing the Lun# of OS Disk to -1 if lunNumber in self.includedLunList : - self.disksToBeIncluded.append(self.device_name) - self.logger.log("LUN Number {0}, disk {1}".format(lunNumber,self.device_name)) + self.disksToBeIncluded.append(device_name) + self.logger.log("LUN Number {0}, disk {1}".format(lunNumber,device_name)) self.logger.log("Disks to be included {0}".format(self.disksToBeIncluded)) return self.disksToBeIncluded @@ -142,17 +140,18 @@ def device_list_for_billing(self): Sample output for lsblk --json command { "blockdevices": [ - {"name": "sdb", "maj:min": "8:16", "rm": "0", "size": "32G", "ro": "0", "type": "disk", "mountpoint": null, - "children": [ - {"name": "sdb1", "maj:min": "8:17", "rm": "0", "size": "976.6M", "ro": "0", "type": "part", "mountpoint": null} - ] + {"name": "sda", "mountpoint": null, + "children": [ + {"name": "sda1", "mountpoint": null} + ] }, - {"name": "sdc", "maj:min": "8:32", "rm": "0", "size": "128G", "ro": "0", "type": "disk", "mountpoint": null, - "children": [ - {"name": "data--vg01-data--lv01", "maj:min": "253:0", "rm": "0", "size": "16G", "ro": "0", "type": "lvm", "mountpoint": null} - ] + {"name": "sdb", "mountpoint": null}, + {"name": "sdc", "mountpoint": null, + "children": [ + {"name": "sdc1", "mountpoint": null} + ] }, - .... + ... ] } ''' diff --git a/VMBackup/main/handle.py b/VMBackup/main/handle.py index ab374b672..8de4f4a0d 100755 --- a/VMBackup/main/handle.py +++ b/VMBackup/main/handle.py @@ -130,7 +130,7 @@ def get_status_to_report(status, status_code, message, snapshot_info = None): file_report_msg = None try: if total_used_size == -1 : - sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , logger = backup_logger , para_parser = para_parser) + sizeCalculation = SizeCalculation.SizeCalculation(patching = MyPatching , hutil = hutil, logger = backup_logger , para_parser = para_parser) total_used_size,size_calculation_failed = sizeCalculation.get_total_used_size() number_of_blobs = len(para_parser.includeLunList) maximum_possible_size = number_of_blobs * 1099511627776 @@ -236,24 +236,20 @@ def daemon(): hutil.do_parse_context('Executing', configSeqNo) try: - backup_logger.log('starting daemon', True) + backup_logger.log('starting daemon initially', True) backup_logger.log("patch_class_name: "+str(patch_class_name)+" and orig_distro: "+str(orig_distro),True) # handle the restoring scenario. mi = MachineIdentity() stored_identity = mi.stored_identity() - try: - if(stored_identity is None): + if(stored_identity is None): + mi.save_identity() + else: + current_identity = mi.current_identity() + if(current_identity != stored_identity): + current_seq_no = -1 + backup_logger.log("machine identity not same, set current_seq_no to " + str(current_seq_no) + " " + str(stored_identity) + " " + str(current_identity), True) + hutil.set_last_seq(current_seq_no) mi.save_identity() - else: - current_identity = mi.current_identity() - if(current_identity != stored_identity): - current_seq_no = -1 - backup_logger.log("machine identity not same, set current_seq_no to " + str(current_seq_no) + " " + str(stored_identity) + " " + str(current_identity), True) - hutil.set_last_seq(current_seq_no) - mi.save_identity() - except Exception as e: - errMsg = "Unable to open file, error: %s, stack trace: %s" % (str(e), traceback.format_exc()) - backup_logger.log(errMsg, True, 'Error') except Exception as e: errMsg = 'Failed to validate sequence number with error: %s, stack trace: %s' % (str(e), traceback.format_exc()) backup_logger.log(errMsg, True, 'Error') @@ -304,7 +300,7 @@ def daemon(): try: # we need to freeze the file system first - backup_logger.log('starting daemon', True) + backup_logger.log('starting daemon for freezing the file system', True) """ protectedSettings is the privateConfig passed from Powershell. WATCHOUT that, the _context_config are using the most freshest timestamp. @@ -339,7 +335,7 @@ def daemon(): commandToExecute = para_parser.commandToExecute #validate all the required parameter here - backup_logger.log(commandToExecute,True) + backup_logger.log('The command '+ commandToExecute+ ' is being validated',True) if(CommonVariables.iaas_install_command in commandToExecute.lower()): backup_logger.log('install succeed.',True) run_status = 'success' @@ -354,7 +350,7 @@ def daemon(): error_msg = 'required field empty or not correct' backup_logger.log(error_msg, True, 'Error') else: - backup_logger.log('commandToExecute is ' + commandToExecute, True) + backup_logger.log('commandToExecute for backup is ' + commandToExecute, True) """ make sure the log is not doing when the file system is freezed. """ @@ -369,7 +365,7 @@ def daemon(): backup_logger.commit_to_blob(para_parser.logsBlobUri) else: backup_logger.log("the logs blob uri is not there, so do not upload log.") - backup_logger.log('commandToExecute is ' + commandToExecute, True) + backup_logger.log('commandToExecute after commiting the blob is ' + commandToExecute, True) workload_patch = WorkloadPatch.WorkloadPatch(backup_logger) #new flow only if workload name is present in workload.conf From 0011cd4446f8e186470226e909930ca78cd1a125 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Mon, 15 May 2023 12:57:12 +0530 Subject: [PATCH 08/10] removed sudo --- VMBackup/main/Utils/SizeCalculation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/VMBackup/main/Utils/SizeCalculation.py b/VMBackup/main/Utils/SizeCalculation.py index 9077811f3..e70a25741 100644 --- a/VMBackup/main/Utils/SizeCalculation.py +++ b/VMBackup/main/Utils/SizeCalculation.py @@ -68,7 +68,7 @@ def __init__(self,patching, hutil, logger,para_parser): self.logger.log("As the LunList is empty including all disks") def get_lsscsi_list(self): - command = "sudo lsscsi" + command = "lsscsi" try: self.lsscsi_list = (os.popen(command).read()).splitlines() except Exception as e: @@ -86,10 +86,10 @@ def get_lsblk_list(self): def get_pvs_list(self): try: - self.pvs_output = os.popen("sudo pvs").read().strip().split("\n") + self.pvs_output = os.popen("pvs").read().strip().split("\n") self.pvs_output = self.pvs_output[1:] except Exception as e: - error_msg = "Failed to execute the command sudo pvs because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) + error_msg = "Failed to execute the command pvs because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.pvs_output = [] @@ -112,7 +112,7 @@ def disk_list_for_billing(self): idxOfColon = item.rindex(':',0,item.index(']'))# to get the index of last ':' idxOfColon += 1 lunNumber = int(item[idxOfColon:item.index(']')]) - # item_split is the list of elements present in the one row of the cmd sudo lsscsi + # item_split is the list of elements present in the one row of the cmd lsscsi self.item_split = item.split() #storing the corresponding device name from the list device_name = self.item_split[len(self.item_split)-1] @@ -128,7 +128,7 @@ def disk_list_for_billing(self): self.logger.log("Disks to be included {0}".format(self.disksToBeIncluded)) else: self.size_calc_failed = True - self.logger.log("There is some glitch in executing the command 'sudo lsscsi' and therefore size calculation is marked as failed.") + self.logger.log("There is some glitch in executing the command 'lsscsi' and therefore size calculation is marked as failed.") def get_logicalVolumes_for_billing(self): try: From d89fbfdb7dde935ee55808d1a958649714208726 Mon Sep 17 00:00:00 2001 From: arisettisanjana Date: Mon, 15 May 2023 12:58:06 +0530 Subject: [PATCH 09/10] Revert "removed sudo" This reverts commit 0011cd4446f8e186470226e909930ca78cd1a125. --- VMBackup/main/Utils/SizeCalculation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/VMBackup/main/Utils/SizeCalculation.py b/VMBackup/main/Utils/SizeCalculation.py index e70a25741..9077811f3 100644 --- a/VMBackup/main/Utils/SizeCalculation.py +++ b/VMBackup/main/Utils/SizeCalculation.py @@ -68,7 +68,7 @@ def __init__(self,patching, hutil, logger,para_parser): self.logger.log("As the LunList is empty including all disks") def get_lsscsi_list(self): - command = "lsscsi" + command = "sudo lsscsi" try: self.lsscsi_list = (os.popen(command).read()).splitlines() except Exception as e: @@ -86,10 +86,10 @@ def get_lsblk_list(self): def get_pvs_list(self): try: - self.pvs_output = os.popen("pvs").read().strip().split("\n") + self.pvs_output = os.popen("sudo pvs").read().strip().split("\n") self.pvs_output = self.pvs_output[1:] except Exception as e: - error_msg = "Failed to execute the command pvs because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) + error_msg = "Failed to execute the command sudo pvs because of error %s , stack trace: %s" % (str(e), traceback.format_exc()) self.logger.log(error_msg, True ,'Error') self.pvs_output = [] @@ -112,7 +112,7 @@ def disk_list_for_billing(self): idxOfColon = item.rindex(':',0,item.index(']'))# to get the index of last ':' idxOfColon += 1 lunNumber = int(item[idxOfColon:item.index(']')]) - # item_split is the list of elements present in the one row of the cmd lsscsi + # item_split is the list of elements present in the one row of the cmd sudo lsscsi self.item_split = item.split() #storing the corresponding device name from the list device_name = self.item_split[len(self.item_split)-1] @@ -128,7 +128,7 @@ def disk_list_for_billing(self): self.logger.log("Disks to be included {0}".format(self.disksToBeIncluded)) else: self.size_calc_failed = True - self.logger.log("There is some glitch in executing the command 'lsscsi' and therefore size calculation is marked as failed.") + self.logger.log("There is some glitch in executing the command 'sudo lsscsi' and therefore size calculation is marked as failed.") def get_logicalVolumes_for_billing(self): try: From 1a0c6eac3de006303f59320f50d2d3bd753e2210 Mon Sep 17 00:00:00 2001 From: "Arisetti, Sanjana" Date: Wed, 16 Apr 2025 11:05:28 +0530 Subject: [PATCH 10/10] disabling console logging --- VMBackup/main/backuplogger.py | 25 ++++++++++++++----------- VMBackup/main/handle.py | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/VMBackup/main/backuplogger.py b/VMBackup/main/backuplogger.py index 6f83a1d55..ba577c2e8 100644 --- a/VMBackup/main/backuplogger.py +++ b/VMBackup/main/backuplogger.py @@ -32,6 +32,8 @@ def __init__(self, hutil): self.enforced_local_flag_value = True self.hutil = hutil self.prev_log = '' + #setting the ConsoleLoggingOff flag to true by default + self.console_logging_off = self.hutil.get_intvalue_from_configfile('ConsoleLoggingOff', 0) self.logging_off = False def enforce_local_flag(self, enforced_local): @@ -68,16 +70,17 @@ def log(self, msg, local=False, level='Info'): self.hutil.log(str(msg),level) def log_to_con(self, msg): - try: - with open(self.con_path, "wb") as C : - message = "".join(list(filter(lambda x : x in string.printable, msg))) - C.write(message.encode('ascii','ignore')) - except IOError as e: - pass - except Exception as e: - pass + if self.console_logging_off != 0: + try: + with open(self.con_path, "wb") as C : + message = "".join(list(filter(lambda x : x in string.printable, msg))) + C.write(message.encode('ascii','ignore')) + except IOError as e: + pass + except Exception as e: + pass - def log_to_con_py3(self, msg, level='Info'): + def log_to_con_py3(self, msg, level='Info'): log_msg = "" try: if type(msg) is not str: @@ -85,8 +88,8 @@ def log_to_con_py3(self, msg, level='Info'): time = datetime.datetime.utcnow().strftime(u'%Y/%m/%d %H:%M:%S.%f') log_msg = u"{0} {1} {2} \n".format(time , level , msg) log_msg= str(log_msg.encode('ascii', "backslashreplace"), - encoding="ascii") - if(self.enforced_local_flag_value != False): + encoding="ascii") + if(self.enforced_local_flag_value != False and self.console_logging_off != 0): with open(self.con_path, "w") as C : C.write(log_msg) except IOError: diff --git a/VMBackup/main/handle.py b/VMBackup/main/handle.py index d99adbfe1..7dbd1bd07 100755 --- a/VMBackup/main/handle.py +++ b/VMBackup/main/handle.py @@ -72,7 +72,7 @@ def main(): total_used_size = 0 size_calculation_failed = False eventlogger = None - HandlerUtil.waagent.LoggerInit('/dev/console','/dev/stdout') + HandlerUtil.waagent.LoggerInit(None,'/dev/stdout') hutil = HandlerUtil.HandlerUtility(HandlerUtil.waagent.Log, HandlerUtil.waagent.Error, CommonVariables.extension_name) backup_logger = Backuplogger(hutil) MyPatching, patch_class_name, orig_distro = GetMyPatching(backup_logger)