From 64db486b0e32fdd76481fae4d7f7b9c13a0c1a85 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 10 Sep 2024 13:26:00 -0400 Subject: [PATCH 01/21] NAS B&R Plugin enhancements --- .../cluster/KubernetesClusterManagerImpl.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 39e4bbf935fc..139b5c79b802 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -1469,6 +1469,9 @@ public boolean deleteKubernetesCluster(DeleteKubernetesClusterCmd cmd) throws Cl } List vmMapList = kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId); + if (checkIfVmsAssociatedWithBackupOffering(vmMapList)) { + throw new CloudRuntimeException("Unable to delete Kubernetes cluster, as node(s) are associated to a backup offering"); + } for (KubernetesClusterVmMapVO vmMap : vmMapList) { try { userVmService.destroyVm(vmMap.getVmId(), expunge); @@ -1491,6 +1494,16 @@ public Boolean doInTransaction(TransactionStatus status) { } } + private boolean checkIfVmsAssociatedWithBackupOffering(List vmMapList) { + for(KubernetesClusterVmMapVO vmMap : vmMapList) { + VMInstanceVO vm = vmInstanceDao.findById(vmMap.getVmId()); + if (Objects.nonNull(vm.getBackupOfferingId())) { + return true; + } + } + return false; + } + @Override public ListResponse listKubernetesClusters(ListKubernetesClustersCmd cmd) { if (!KubernetesServiceEnabled.value()) { From d5a259da0a8205f65ebe82eaa832ee2aba2771f4 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Fri, 20 Sep 2024 10:44:04 -0400 Subject: [PATCH 02/21] Prevent printing mount opts which may include password by removing from response --- .../response/BackupRepositoryResponse.java | 12 ------ scripts/vm/hypervisor/kvm/nasbackup.sh | 6 +++ .../java/com/cloud/api/ApiResponseHelper.java | 1 - .../cloudstack/backup/BackupManagerImpl.java | 42 +++++++++++++++++++ tools/marvin/setup.py | 2 +- ui/src/config/section/config.js | 2 +- 6 files changed, 50 insertions(+), 15 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java index 3847176608c0..0db51f040349 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java @@ -57,10 +57,6 @@ public class BackupRepositoryResponse extends BaseResponse { @Param(description = "backup type") private String type; - @SerializedName(ApiConstants.MOUNT_OPTIONS) - @Param(description = "mount options for the backup repository") - private String mountOptions; - @SerializedName(ApiConstants.CAPACITY_BYTES) @Param(description = "capacity of the backup repository") private Long capacityBytes; @@ -112,14 +108,6 @@ public void setAddress(String address) { this.address = address; } - public String getMountOptions() { - return mountOptions; - } - - public void setMountOptions(String mountOptions) { - this.mountOptions = mountOptions; - } - public String getProviderName() { return providerName; } diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index 5b264321bd8d..8b6c0b8c52d7 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -100,6 +100,12 @@ mount_operation() { mount_point=$(mktemp -d -t csbackup.XXXXX) dest="$mount_point/${BACKUP_DIR}" mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) + if [ $? -eq 0 ]; then + echo "Successfully mounted ${NAS_TYPE} store" + else + echo "Failed to mount ${NAS_TYPE} store" + exit 1 + fi } function usage { diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 810f0abd7e00..76110dc58f76 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -5439,7 +5439,6 @@ public BackupRepositoryResponse createBackupRepositoryResponse(BackupRepository response.setAddress(backupRepository.getAddress()); response.setProviderName(backupRepository.getProvider()); response.setType(backupRepository.getType()); - response.setMountOptions(backupRepository.getMountOptions()); response.setCapacityBytes(backupRepository.getCapacityBytes()); response.setObjectName("backuprepository"); DataCenter zone = ApiDBUtils.findZoneById(backupRepository.getZoneId()); diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index b86b65bd465d..7509a4183b3f 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -19,24 +19,33 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.TimeZone; import java.util.Timer; import java.util.TimerTask; import java.util.stream.Collectors; import com.amazonaws.util.CollectionUtils; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeApiService; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.vm.NicVO; import com.cloud.vm.UserVmManager; +import com.cloud.vm.UserVmService; import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachineManager; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.vm.dao.NicDao; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.backup.DeleteBackupOfferingCmd; @@ -166,6 +175,14 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { private VolumeApiService volumeApiService; @Inject private VolumeOrchestrationService volumeOrchestrationService; + @Inject + private VMTemplateDao templateDao; + @Inject + private NicDao nicDao; + @Inject + private NetworkDao networkDao; + @Inject + protected UserVmService userVmService; private AsyncJobDispatcher asyncJobDispatcher; private Timer backupTimer; @@ -645,6 +662,9 @@ public boolean restoreBackup(final Long backupId) { !vm.getState().equals(VirtualMachine.State.Destroyed)) { throw new CloudRuntimeException("Existing VM should be stopped before being restored from backup"); } + if (VirtualMachine.State.Expunging.equals(vm.getState()) && vm.getRemoved() != null) { + restoreExpungedVm(vm); + } // This is done to handle historic backups if any with Veeam / Networker plugins List backupVolumes = CollectionUtils.isNullOrEmpty(backup.getBackedUpVolumes()) ? vm.getBackupVolumeList() : backup.getBackedUpVolumes(); @@ -705,6 +725,28 @@ protected void tryRestoreVM(BackupVO backup, VMInstanceVO vm, BackupOffering off } } + private void restoreExpungedVm(VMInstanceVO vm) { + VMTemplateVO template = templateDao.findById(vm.getTemplateId()); + if (Objects.isNull(template)) { + throw new CloudRuntimeException("Failed to find VM template to restore the VM"); + } + List networkIds = nicDao.listByVmId(vm.getId()).stream().sorted(Comparator.comparing(NicVO::getDeviceId)).map(NicVO::getNetworkId).collect(Collectors.toList()); + for (Long networkId : networkIds) { + NetworkVO networkVO = networkDao.findById(networkId); + if (Objects.isNull(networkVO)) { + throw new CloudRuntimeException("Failed to find network all networks the VM was previously attached to"); + } + } + + CallContext ctx = CallContext.current(); + final Account owner = ctx.getCallingAccount(); +// userVmService.createAdvancedVirtualMachine(vm.getDataCenterId(), vm.getServiceOfferingId(), template.getId(), networkIds, owner, +// vm.getHostName(), vm.getHostName(), null, null, null, +// Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, null, null, null, keypairs, +// requestedIps, addrs, null, null, null, customParameterMap, null, null, null, null, true, null, null); + + } + /** * Tries to update the state of given VM, given specified event * @param vm The VM to update its state diff --git a/tools/marvin/setup.py b/tools/marvin/setup.py index 0618d84370a4..679b1d5920d8 100644 --- a/tools/marvin/setup.py +++ b/tools/marvin/setup.py @@ -27,7 +27,7 @@ raise RuntimeError("python setuptools is required to build Marvin") -VERSION = "4.20.0.0-SNAPSHOT" +VERSION = "4.20.0.0" setup(name="Marvin", version=VERSION, diff --git a/ui/src/config/section/config.js b/ui/src/config/section/config.js index 1736adf79c4f..2a4dbb84a6ec 100644 --- a/ui/src/config/section/config.js +++ b/ui/src/config/section/config.js @@ -151,7 +151,7 @@ export default { ], mapping: { type: { - options: ['nfs'] + options: ['nfs', 'cifs'] }, provider: { value: (record) => { return 'nas' } From 6dfb0ea2ceda4cc5be84771f892f7ffd04a96abf Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Fri, 20 Sep 2024 12:08:40 -0400 Subject: [PATCH 03/21] revert marvin change --- tools/marvin/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/marvin/setup.py b/tools/marvin/setup.py index 679b1d5920d8..0618d84370a4 100644 --- a/tools/marvin/setup.py +++ b/tools/marvin/setup.py @@ -27,7 +27,7 @@ raise RuntimeError("python setuptools is required to build Marvin") -VERSION = "4.20.0.0" +VERSION = "4.20.0.0-SNAPSHOT" setup(name="Marvin", version=VERSION, From b1ccf9d44328239807fd8a1905523423a3c902e3 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 23 Sep 2024 10:11:56 -0400 Subject: [PATCH 04/21] add sanity checks to validate minimum qemu and libvirt versions --- scripts/vm/hypervisor/kvm/nasbackup.sh | 43 ++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index 8b6c0b8c52d7..ed98db53db78 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -32,6 +32,46 @@ MOUNT_OPTS="" BACKUP_DIR="" DISK_PATHS="" +vercomp() { + local IFS=. + local i ver1=($1) ver2=($3) + + # Compare each segment of the version numbers + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + ver2[i]=0 + fi + + if ((10#${ver1[i]} > 10#${ver2[i]})); then + return 1 # Version 1 is greater + elif ((10#${ver1[i]} < 10#${ver2[i]})); then + return 2 # Version 2 is greater + fi + done + return 0 # Versions are equal +} + +sanity_checks() { + hvVersion=$(virsh version | grep hypervisor | awk '{print $(NF)}') + libvVersion=$(virsh version | grep libvirt | awk '{print $(NF)}' | tail -n 1) + apiVersion=$(virsh version | grep API | awk '{print $(NF)}') + + # Compare qemu version (hvVersion >= 4.2.0) + vercomp "$hvVersion" ">=" "4.2.0" + hvStatus=$? + + # Compare libvirt version (libvVersion >= 7.2.0) + vercomp "$libvVersion" ">=" "7.2.0" + libvStatus=$? + + if [[ ($hvStatus -eq 0 || $hvStatus -eq 1) && ($libvStatus -eq 0 || $libvStatus -eq 1) ]]; then + echo "Success \t\t [ QEMU: $hvVersion Libvirt: $libvVersion apiVersion: $apiVersion ]" + else + echo "Failure... \tYour QEMU version $hvVersion or libvirt version $libvVersion is unsupported. Consider upgrading to the required minimum version of QEMU: 4.2.0 and Libvirt: 7.2.0" + exit 1 + fi +} + ### Operation methods ### backup_running_vm() { @@ -163,6 +203,9 @@ while [[ $# -gt 0 ]]; do esac done +# Perform Initial sanity checks +sanity_checks + if [ "$OP" = "backup" ]; then STATE=$(virsh -c qemu:///system list | grep $VM | awk '{print $3}') if [ "$STATE" = "running" ]; then From 343e7dca2740f22ce8415276b90ace1195204737 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 23 Sep 2024 10:17:02 -0400 Subject: [PATCH 05/21] check is user running script is part of libvirt group --- scripts/vm/hypervisor/kvm/nasbackup.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index ed98db53db78..b55ec4d616df 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -65,11 +65,20 @@ sanity_checks() { libvStatus=$? if [[ ($hvStatus -eq 0 || $hvStatus -eq 1) && ($libvStatus -eq 0 || $libvStatus -eq 1) ]]; then - echo "Success \t\t [ QEMU: $hvVersion Libvirt: $libvVersion apiVersion: $apiVersion ]" + echo "Success... [ QEMU: $hvVersion Libvirt: $libvVersion apiVersion: $apiVersion ]" else - echo "Failure... \tYour QEMU version $hvVersion or libvirt version $libvVersion is unsupported. Consider upgrading to the required minimum version of QEMU: 4.2.0 and Libvirt: 7.2.0" + echo "Failure... Your QEMU version $hvVersion or libvirt version $libvVersion is unsupported. Consider upgrading to the required minimum version of QEMU: 4.2.0 and Libvirt: 7.2.0" exit 1 fi + + echo "Checking Permissions" + if groups $USER | grep -q '\blibvirt\b'; then + echo "Success... User $USER is part of libvirt group" + else + echo "Failure - User $USER is not part of libvirt group" + exit 1 + fi + echo "Environment Sanity Checks successfully passed" } ### Operation methods ### From f2f81c82f4d998594fbe28ba9b3a083328c0b244 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 23 Sep 2024 10:40:21 -0400 Subject: [PATCH 06/21] revert changes of retore expunged VM --- .../cloudstack/backup/BackupManagerImpl.java | 48 +------------------ 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index b39230f21ad4..8430a59f5cef 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -19,33 +19,22 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.TimeZone; import java.util.Timer; import java.util.TimerTask; import java.util.stream.Collectors; import com.amazonaws.util.CollectionUtils; -import com.cloud.network.dao.NetworkDao; -import com.cloud.network.dao.NetworkVO; -import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeApiService; -import com.cloud.storage.dao.VMTemplateDao; import com.cloud.utils.fsm.NoTransitionException; -import com.cloud.vm.NicVO; -import com.cloud.vm.UserVmManager; -import com.cloud.vm.UserVmService; -import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachineManager; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.vm.dao.NicDao; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.backup.DeleteBackupOfferingCmd; @@ -72,7 +61,6 @@ import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.BackupScheduleDao; import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.jobs.AsyncJobDispatcher; import org.apache.cloudstack.framework.jobs.AsyncJobManager; @@ -173,16 +161,6 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { private VirtualMachineManager virtualMachineManager; @Inject private VolumeApiService volumeApiService; - @Inject - private VolumeOrchestrationService volumeOrchestrationService; - @Inject - private VMTemplateDao templateDao; - @Inject - private NicDao nicDao; - @Inject - private NetworkDao networkDao; - @Inject - protected UserVmService userVmService; private AsyncJobDispatcher asyncJobDispatcher; private Timer backupTimer; @@ -642,9 +620,7 @@ public boolean restoreBackup(final Long backupId) { !vm.getState().equals(VirtualMachine.State.Destroyed)) { throw new CloudRuntimeException("Existing VM should be stopped before being restored from backup"); } - if (VirtualMachine.State.Expunging.equals(vm.getState()) && vm.getRemoved() != null) { - restoreExpungedVm(vm); - } + // This is done to handle historic backups if any with Veeam / Networker plugins List backupVolumes = CollectionUtils.isNullOrEmpty(backup.getBackedUpVolumes()) ? vm.getBackupVolumeList() : backup.getBackedUpVolumes(); @@ -705,28 +681,6 @@ protected void tryRestoreVM(BackupVO backup, VMInstanceVO vm, BackupOffering off } } - private void restoreExpungedVm(VMInstanceVO vm) { - VMTemplateVO template = templateDao.findById(vm.getTemplateId()); - if (Objects.isNull(template)) { - throw new CloudRuntimeException("Failed to find VM template to restore the VM"); - } - List networkIds = nicDao.listByVmId(vm.getId()).stream().sorted(Comparator.comparing(NicVO::getDeviceId)).map(NicVO::getNetworkId).collect(Collectors.toList()); - for (Long networkId : networkIds) { - NetworkVO networkVO = networkDao.findById(networkId); - if (Objects.isNull(networkVO)) { - throw new CloudRuntimeException("Failed to find network all networks the VM was previously attached to"); - } - } - - CallContext ctx = CallContext.current(); - final Account owner = ctx.getCallingAccount(); -// userVmService.createAdvancedVirtualMachine(vm.getDataCenterId(), vm.getServiceOfferingId(), template.getId(), networkIds, owner, -// vm.getHostName(), vm.getHostName(), null, null, null, -// Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, null, null, null, keypairs, -// requestedIps, addrs, null, null, null, customParameterMap, null, null, null, null, true, null, null); - - } - /** * Tries to update the state of given VM, given specified event * @param vm The VM to update its state From be9eba39c9037c89116ab2f41ae0786e69aa9920 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 30 Sep 2024 07:55:46 -0400 Subject: [PATCH 07/21] add code coverage ignore file --- codecov.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000000..a9c6aca16918 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,3 @@ +ignore: + - api/src/main/java/org/apache/cloudstack/api/response/* + - plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/* From 585126b4b1cfed668e5b73b1fa4df643a9d2c6ad Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 30 Sep 2024 09:50:58 -0400 Subject: [PATCH 08/21] remove check --- scripts/vm/hypervisor/kvm/nasbackup.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index b55ec4d616df..dad94f33bc52 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -71,14 +71,7 @@ sanity_checks() { exit 1 fi - echo "Checking Permissions" - if groups $USER | grep -q '\blibvirt\b'; then - echo "Success... User $USER is part of libvirt group" - else - echo "Failure - User $USER is not part of libvirt group" - exit 1 - fi - echo "Environment Sanity Checks successfully passed" + echo "Environment Sanity Checks successfully passed" } ### Operation methods ### From d024a964c41d4196c845f87f27dc09ae324bc4ab Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 30 Sep 2024 10:57:30 -0400 Subject: [PATCH 09/21] issue with listing schedules and add defensive checks --- .../api/command/user/backup/ListBackupScheduleCmd.java | 2 +- .../org/apache/cloudstack/backup/NASBackupProvider.java | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java index a76107174358..fa6e3ea5d455 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java @@ -82,7 +82,7 @@ public void execute() throws ResourceUnavailableException, InsufficientCapacityE List schedules = backupManager.listBackupSchedule(getVmId()); ListResponse response = new ListResponse<>(); List scheduleResponses = new ArrayList<>(); - if (CollectionUtils.isNullOrEmpty(schedules)) { + if (!CollectionUtils.isNullOrEmpty(schedules)) { for (BackupSchedule schedule : schedules) { scheduleResponses.add(_responseGenerator.createBackupScheduleResponse(schedule)); } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 4a6725abdca5..f900125a462f 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -373,8 +373,12 @@ public Map getBackupMetrics(Long zoneId, List Date: Mon, 30 Sep 2024 23:10:07 -0400 Subject: [PATCH 10/21] redirect logs to agent log file --- scripts/vm/hypervisor/kvm/nasbackup.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index dad94f33bc52..91b3e5bada5e 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -31,6 +31,16 @@ NAS_ADDRESS="" MOUNT_OPTS="" BACKUP_DIR="" DISK_PATHS="" +logFile="/var/log/cloudstack/agent/agent.log" + +log() { + [[ "$verb" -eq 1 ]] && builtin echo "$@" + if [[ "$1" == "-ne" || "$1" == "-e" || "$1" == "-n" ]]; then + builtin echo -e "$(date '+%Y-%m-%d %H-%M-%S>')" "${@: 2}" >> "$logFile" + else + builtin echo "$(date '+%Y-%m-%d %H-%M-%S>')" "$@" >> "$logFile" + fi +} vercomp() { local IFS=. @@ -43,7 +53,7 @@ vercomp() { fi if ((10#${ver1[i]} > 10#${ver2[i]})); then - return 1 # Version 1 is greater + return 0 # Version 1 is greater elif ((10#${ver1[i]} < 10#${ver2[i]})); then return 2 # Version 2 is greater fi @@ -64,14 +74,14 @@ sanity_checks() { vercomp "$libvVersion" ">=" "7.2.0" libvStatus=$? - if [[ ($hvStatus -eq 0 || $hvStatus -eq 1) && ($libvStatus -eq 0 || $libvStatus -eq 1) ]]; then - echo "Success... [ QEMU: $hvVersion Libvirt: $libvVersion apiVersion: $apiVersion ]" + if [[ $hvStatus -eq 0 && $libvStatus -eq 0 ]]; then + log -ne "Success... [ QEMU: $hvVersion Libvirt: $libvVersion apiVersion: $apiVersion ]" else echo "Failure... Your QEMU version $hvVersion or libvirt version $libvVersion is unsupported. Consider upgrading to the required minimum version of QEMU: 4.2.0 and Libvirt: 7.2.0" exit 1 fi - echo "Environment Sanity Checks successfully passed" + log -ne "Environment Sanity Checks successfully passed" } ### Operation methods ### @@ -143,7 +153,7 @@ mount_operation() { dest="$mount_point/${BACKUP_DIR}" mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) if [ $? -eq 0 ]; then - echo "Successfully mounted ${NAS_TYPE} store" + log -ne "Successfully mounted ${NAS_TYPE} store" else echo "Failed to mount ${NAS_TYPE} store" exit 1 From 0760ef596873b465dbea5af9fbf0b9f562336ff9 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 1 Oct 2024 00:04:39 -0400 Subject: [PATCH 11/21] add some more debugging --- scripts/vm/hypervisor/kvm/nasbackup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index 91b3e5bada5e..a128915e2c24 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -131,7 +131,7 @@ backup_stopped_vm() { name="root" for disk in $DISK_PATHS; do volUuid="${disk##*/}" - qemu-img convert -O qcow2 $disk $dest/$name.$volUuid.qcow2 + qemu-img convert -O qcow2 $disk $dest/$name.$volUuid.qcow2 | tee -a "$logFile" name="datadisk" done sync @@ -151,7 +151,7 @@ delete_backup() { mount_operation() { mount_point=$(mktemp -d -t csbackup.XXXXX) dest="$mount_point/${BACKUP_DIR}" - mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) + mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) | tee -a "$logFile" if [ $? -eq 0 ]; then log -ne "Successfully mounted ${NAS_TYPE} store" else From bf19dea4920894090b035f329a739e0702f26e18 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 1 Oct 2024 00:07:20 -0400 Subject: [PATCH 12/21] remove test file --- codecov.yml | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index a9c6aca16918..000000000000 --- a/codecov.yml +++ /dev/null @@ -1,3 +0,0 @@ -ignore: - - api/src/main/java/org/apache/cloudstack/api/response/* - - plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/* From 5bfed063f6d9fa35a9c130dddcc336775ebeeafe Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 1 Oct 2024 09:58:59 -0400 Subject: [PATCH 13/21] prevent deletion of cks cluster when vms associated to a backup offering --- .../kubernetes/cluster/KubernetesClusterManagerImpl.java | 9 +++++---- .../actionworkers/KubernetesClusterDestroyWorker.java | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 7655de5afa58..77d90c14b3e7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -36,6 +36,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -1468,7 +1469,8 @@ public boolean deleteKubernetesCluster(DeleteKubernetesClusterCmd cmd) throws Cl } List vmMapList = kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId); - if (checkIfVmsAssociatedWithBackupOffering(vmMapList)) { + List vms = vmMapList.stream().map(vmMap -> vmInstanceDao.findById(vmMap.getVmId())).collect(Collectors.toList()); + if (checkIfVmsAssociatedWithBackupOffering(vms)) { throw new CloudRuntimeException("Unable to delete Kubernetes cluster, as node(s) are associated to a backup offering"); } for (KubernetesClusterVmMapVO vmMap : vmMapList) { @@ -1493,9 +1495,8 @@ public Boolean doInTransaction(TransactionStatus status) { } } - private boolean checkIfVmsAssociatedWithBackupOffering(List vmMapList) { - for(KubernetesClusterVmMapVO vmMap : vmMapList) { - VMInstanceVO vm = vmInstanceDao.findById(vmMap.getVmId()); + public static boolean checkIfVmsAssociatedWithBackupOffering(List vms) { + for(VMInstanceVO vm : vms) { if (Objects.nonNull(vm.getBackupOfferingId())) { return true; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index 50d7fb14085a..cef2e91042ac 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -245,6 +245,10 @@ public boolean destroy() throws CloudRuntimeException { init(); validateClusterSate(); this.clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + List vms = this.clusterVMs.stream().map(vmMap -> vmInstanceDao.findById(vmMap.getVmId())).collect(Collectors.toList()); + if (KubernetesClusterManagerImpl.checkIfVmsAssociatedWithBackupOffering(vms)) { + throw new CloudRuntimeException("Unable to delete Kubernetes cluster, as node(s) are associated to a backup offering"); + } boolean cleanupNetwork = true; final KubernetesClusterDetailsVO clusterDetails = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "networkCleanup"); if (clusterDetails != null) { From 3ae36018c84990b0cc74aadd96de347d89dad010 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Thu, 3 Oct 2024 22:55:51 -0400 Subject: [PATCH 14/21] delete all snapshot policies when bkp offering is disassociated from a VM --- .../user/backup/DeleteBackupScheduleCmd.java | 14 ++++++- .../cloudstack/backup/BackupManager.java | 5 ++- .../cloudstack/backup/BackupManagerImpl.java | 37 +++++++++++++------ tools/marvin/setup.py | 2 +- 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/DeleteBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/DeleteBackupScheduleCmd.java index 0245f228b895..04c30da0b5ca 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/DeleteBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/DeleteBackupScheduleCmd.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupScheduleResponse; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.backup.BackupManager; @@ -58,6 +59,14 @@ public class DeleteBackupScheduleCmd extends BaseCmd { description = "ID of the VM") private Long vmId; + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = BackupScheduleResponse.class, + required = true, + description = "ID of the schedule", + since = "4.20.1") + private Long id; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -66,6 +75,9 @@ public Long getVmId() { return vmId; } + public Long getId() { return id; } + + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -73,7 +85,7 @@ public Long getVmId() { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { try { - boolean result = backupManager.deleteBackupSchedule(getVmId()); + boolean result = backupManager.deleteBackupSchedule(this); if (result) { SuccessResponse response = new SuccessResponse(getCommandName()); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 8b45bb4ee5ef..78d189c3bf1d 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -22,6 +22,7 @@ import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; +import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.ListBackupOfferingsCmd; import org.apache.cloudstack.api.command.user.backup.ListBackupsCmd; import org.apache.cloudstack.framework.config.ConfigKey; @@ -111,10 +112,10 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer /** * Deletes VM backup schedule for a VM - * @param vmId + * @param cmd * @return */ - boolean deleteBackupSchedule(Long vmId); + boolean deleteBackupSchedule(DeleteBackupScheduleCmd cmd); /** * Creates backup of a VM diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 8430a59f5cef..0f35e3c77cbe 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.TimeZone; import java.util.Timer; import java.util.TimerTask; @@ -393,8 +394,8 @@ public boolean removeVMFromBackupOffering(final Long vmId, final boolean forced) UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(), vm.getBackupOfferingId(), null, null, Backup.class.getSimpleName(), vm.getUuid()); - final BackupSchedule backupSchedule = backupScheduleDao.findByVM(vm.getId()); - if (backupSchedule != null) { + final List backupSchedules = backupScheduleDao.listByVM(vm.getId()); + for(BackupSchedule backupSchedule: backupSchedules) { backupScheduleDao.remove(backupSchedule.getId()); } result = true; @@ -453,7 +454,7 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { schedule.setTimezone(timezoneId); schedule.setScheduledTimestamp(nextDateTime); backupScheduleDao.update(schedule.getId(), schedule); - return backupScheduleDao.findByVM(vmId); + return backupScheduleDao.findById(schedule.getId()); } @Override @@ -467,16 +468,30 @@ public List listBackupSchedule(final Long vmId) { @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_SCHEDULE_DELETE, eventDescription = "deleting VM backup schedule") - public boolean deleteBackupSchedule(final Long vmId) { - final VMInstanceVO vm = findVmById(vmId); - validateForZone(vm.getDataCenterId()); - accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); + public boolean deleteBackupSchedule(DeleteBackupScheduleCmd cmd) { + Long vmId = cmd.getVmId(); + Long id = cmd.getId(); + if (Objects.nonNull(vmId)) { + final VMInstanceVO vm = findVmById(vmId); + validateForZone(vm.getDataCenterId()); + accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); + return deleteAllVMBackupSchedules(vm.getId()); + } else { + final BackupSchedule schedule = backupScheduleDao.findById(id); + if (schedule == null) { + throw new CloudRuntimeException("Could not find the requested backup schedule."); + } + return backupScheduleDao.remove(schedule.getId()); + } + } - final BackupSchedule schedule = backupScheduleDao.findByVM(vmId); - if (schedule == null) { - throw new CloudRuntimeException("VM has no backup schedule defined, no need to delete anything."); + private boolean deleteAllVMBackupSchedules(long vmId) { + List vmBackupSchedules = backupScheduleDao.listByVM(vmId); + boolean success = true; + for (BackupScheduleVO vmBackupSchedule : vmBackupSchedules) { + success = success && backupScheduleDao.remove(vmBackupSchedule.getId()); } - return backupScheduleDao.remove(schedule.getId()); + return success; } @Override diff --git a/tools/marvin/setup.py b/tools/marvin/setup.py index 0618d84370a4..679b1d5920d8 100644 --- a/tools/marvin/setup.py +++ b/tools/marvin/setup.py @@ -27,7 +27,7 @@ raise RuntimeError("python setuptools is required to build Marvin") -VERSION = "4.20.0.0-SNAPSHOT" +VERSION = "4.20.0.0" setup(name="Marvin", version=VERSION, From d6181d542108d02cca31daa758234bb29e1b317a Mon Sep 17 00:00:00 2001 From: Lucas Martins <56271185+lucas-a-martins@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:41:30 -0300 Subject: [PATCH 15/21] Fix `updateTemplatePermission` when the UI is set to a language other than English (#9766) * Fix updateTemplatePermission UI in non-english language * Improve fix --------- Co-authored-by: Lucas Martins --- ui/public/locales/pt_BR.json | 4 ++-- .../image/UpdateTemplateIsoPermissions.vue | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ui/public/locales/pt_BR.json b/ui/public/locales/pt_BR.json index f02aee747eb8..6955d83a510e 100644 --- a/ui/public/locales/pt_BR.json +++ b/ui/public/locales/pt_BR.json @@ -1482,8 +1482,8 @@ "label.setting": "Configura\u00e7\u00e3o", "label.settings": "Configura\u00e7\u00f5es", "label.setup": "Configura\u00e7\u00e3o", -"label.shared": "Compatilhado", -"label.sharedexecutable": "Compatilhado", +"label.shared": "Compartilhado", +"label.sharedexecutable": "Compartilhado", "label.sharedmountpoint": "SharedMountPoint", "label.sharedrouterip": "Endere\u00e7os IPv4 para o roteador dentro da rede compartilhada", "label.sharedrouteripv6": "Endere\u00e7os IPv6 para o roteador dentro da rede compartilhada", diff --git a/ui/src/views/image/UpdateTemplateIsoPermissions.vue b/ui/src/views/image/UpdateTemplateIsoPermissions.vue index c7b0f9e5cf19..557574156b17 100644 --- a/ui/src/views/image/UpdateTemplateIsoPermissions.vue +++ b/ui/src/views/image/UpdateTemplateIsoPermissions.vue @@ -25,7 +25,7 @@

{{ $t('label.operation') }}

- {{ $t('label.add') }} - {{ $t('label.remove') }} - {{ $t('label.reset') }} + {{ $t('label.add') }} + {{ $t('label.remove') }} + {{ $t('label.reset') }} -