From 35ed72c523e479361cae22b85b0e125246cef873 Mon Sep 17 00:00:00 2001 From: abh1sar Date: Wed, 25 Dec 2024 08:24:43 +0530 Subject: [PATCH 001/147] Initial api and service layer changes --- .../main/java/com/cloud/event/EventTypes.java | 1 + .../main/java/com/cloud/vm/UserVmService.java | 8 +- .../user/vm/CreateVMFromBackupCmd.java | 86 +++++++++ .../cloudstack/backup/BackupManager.java | 8 + .../cloudstack/backup/BackupProvider.java | 2 + .../backup/DummyBackupProvider.java | 5 + .../backup/NetworkerBackupProvider.java | 5 + .../backup/VeeamBackupProvider.java | 5 + .../java/com/cloud/vm/UserVmManagerImpl.java | 168 ++++++++++++++++++ .../cloudstack/backup/BackupManagerImpl.java | 103 +++++++++-- 10 files changed, 376 insertions(+), 15 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 81ed185dae5a..ebc99b23761f 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -606,6 +606,7 @@ public class EventTypes { public static final String EVENT_VM_BACKUP_OFFERING_REMOVE = "BACKUP.OFFERING.REMOVE"; public static final String EVENT_VM_BACKUP_CREATE = "BACKUP.CREATE"; public static final String EVENT_VM_BACKUP_RESTORE = "BACKUP.RESTORE"; + public static final String EVENT_VM_BACKUP_RESTORE_TO_NEW_VM = "BACKUP.RESTORE.TO.NEW.VM"; public static final String EVENT_VM_BACKUP_DELETE = "BACKUP.DELETE"; public static final String EVENT_VM_BACKUP_RESTORE_VOLUME_TO_VM = "BACKUP.RESTORE.VOLUME.TO.VM"; public static final String EVENT_VM_BACKUP_SCHEDULE_CONFIGURE = "BACKUP.SCHEDULE.CONFIGURE"; diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 5a9301eef7fa..e2c93c66652f 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd; import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; +import org.apache.cloudstack.api.command.user.vm.CreateVMFromBackupCmd; import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; @@ -412,8 +413,7 @@ UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serviceOffe void deletePrivateTemplateRecord(Long templateId); HypervisorType getHypervisorTypeOfUserVM(long vmid); - - UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, +UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, StorageUnavailableException, ResourceAllocationException; /** @@ -513,4 +513,8 @@ UserVm importVM(final DataCenter zone, final Host host, final VirtualMachineTemp * @return true if the VM is successfully unmanaged, false if not. */ boolean unmanageUserVM(Long vmId); + + UserVm allocateVMFromBackup(CreateVMFromBackupCmd cmd) throws InsufficientCapacityException, ResourceAllocationException, ResourceUnavailableException; + + UserVm restoreVMFromBackup(CreateVMFromBackupCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException, ResourceAllocationException; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java new file mode 100644 index 000000000000..12959cbc2984 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.vm; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.backup.BackupManager; + +import com.cloud.uservm.UserVm; +import com.cloud.vm.VirtualMachine; + +@APICommand(name = "createVMFromBackup", + description = "Creates and automatically starts a VM from a backup.", + responseObject = UserVmResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {VirtualMachine.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, + since = "4.21.0") +public class CreateVMFromBackupCmd extends DeployVMCmd implements UserCmd { + + @Inject + BackupManager backupManager; + + @Parameter(name = ApiConstants.BACKUP_ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + required = true, + description = "backup ID to create the VM from") + private Long backupId; + + public Long getBackupId() { + return backupId; + } + + @Override + public void create() { + UserVm vm; + try { + vm = _userVmService.allocateVMFromBackup(this); + if (vm != null) { + setEntityId(vm.getId()); + setEntityUuid(vm.getUuid()); + } + } catch (Exception e) { + } + } + + @Override + public void execute () { + UserVm vm = null; + try { + vm = _userVmService.restoreVMFromBackup(this); + } catch (Exception e) { + } + if (vm != null) { + UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", vm).get(0); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to deploy vm uuid:"+getEntityUuid()); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 8b45bb4ee5ef..948279326e1a 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.utils.Pair; import com.cloud.utils.component.Manager; import com.cloud.utils.component.PluggableService; @@ -133,6 +134,11 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer */ boolean restoreBackup(final Long backupId); + /** + * Restore a backup to a new Instance + */ + boolean restoreBackupToVM(Long backupId, Long vmId) throws ResourceUnavailableException; + /** * Restore a backed up volume and attach it to a VM */ @@ -146,5 +152,7 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer */ boolean deleteBackup(final Long backupId, final Boolean forced); + void validateBackupForZone(Long zoneId); + BackupOffering updateBackupOffering(UpdateBackupOfferingCmd updateBackupOfferingCmd); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index d36dfb7360f6..d3aeadca6af7 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -85,6 +85,8 @@ public interface BackupProvider { */ boolean deleteBackup(Backup backup, boolean forced); + boolean restoreBackupToVM(VirtualMachine vm, Backup backup); + /** * Restore VM from backup */ diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index f162c51a703d..ae27090be86e 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -138,4 +138,9 @@ public boolean deleteBackup(Backup backup, boolean forced) { @Override public void syncBackups(VirtualMachine vm, Backup.Metric metric) { } + + @Override + public boolean restoreBackupToVM(VirtualMachine vm, Backup backup) { + return true; + } } diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index 0e87ad338871..7751d5e36e09 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -647,4 +647,9 @@ public void doInTransactionWithoutResult(TransactionStatus status) { @Override public boolean willDeleteBackupsOnOfferingRemoval() { return false; } + + @Override + public boolean restoreBackupToVM(VirtualMachine vm, Backup backup) { + return true; + } } diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index 4750e3264aac..dc176062de29 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -397,6 +397,11 @@ public void doInTransactionWithoutResult(TransactionStatus status) { }); } + @Override + public boolean restoreBackupToVM(VirtualMachine vm, Backup backup) { + return true; + } + @Override public String getConfigComponentName() { return BackupService.class.getSimpleName(); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 04b64dd80a36..3c1b1283e5c9 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -74,6 +74,7 @@ import org.apache.cloudstack.api.command.admin.vm.ExpungeVMCmd; import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd; import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; +import org.apache.cloudstack.api.command.user.vm.CreateVMFromBackupCmd; import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; import org.apache.cloudstack.api.command.user.vm.DeployVnfApplianceCmd; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; @@ -96,6 +97,7 @@ import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.BackupVO; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.cloud.entity.api.VirtualMachineEntity; @@ -8725,6 +8727,172 @@ public boolean unmanageUserVM(Long vmId) { return true; } + @Override + public UserVm allocateVMFromBackup(CreateVMFromBackupCmd cmd) throws InsufficientCapacityException, ResourceAllocationException, ResourceUnavailableException { + //Verify that all objects exist before passing them to the service + Account owner = _accountService.getActiveAccountById(cmd.getEntityOwnerId()); + Long zoneId = cmd.getZoneId(); + DataCenter zone = _entityMgr.findById(DataCenter.class, zoneId); + if (zone == null) { + throw new InvalidParameterValueException("Unable to find zone by id=" + zoneId); + } + + BackupVO backup = backupDao.findById(cmd.getBackupId()); + if (backup == null) { + throw new InvalidParameterValueException("Backup " + cmd.getBackupId() + " does not exist"); + } + if (backup.getZoneId() != cmd.getZoneId()) { + throw new InvalidParameterValueException("Instance should be created in the same zone as the backup"); + } + backupManager.validateBackupForZone(backup.getZoneId()); + + verifyDetails(cmd.getDetails()); + + Long serviceOfferingId = cmd.getServiceOfferingId(); + Long overrideDiskOfferingId = cmd.getOverrideDiskOfferingId(); + + ServiceOffering serviceOffering = _entityMgr.findById(ServiceOffering.class, serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("Unable to find service offering: " + serviceOfferingId); + } + + if (ServiceOffering.State.Inactive.equals(serviceOffering.getState())) { + throw new InvalidParameterValueException(String.format("Service offering is inactive: [%s].", serviceOffering.getUuid())); + } + + if (serviceOffering.getDiskOfferingStrictness() && overrideDiskOfferingId != null) { + throw new InvalidParameterValueException(String.format("Cannot override disk offering id %d since provided service offering is strictly mapped to its disk offering", overrideDiskOfferingId)); + } + + if (!serviceOffering.isDynamic()) { + for(String detail: cmd.getDetails().keySet()) { + if(detail.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || detail.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || detail.equalsIgnoreCase(VmDetailConstants.MEMORY)) { + throw new InvalidParameterValueException("cpuNumber or cpuSpeed or memory should not be specified for static service offering"); + } + } + } + + Long templateId = cmd.getTemplateId(); + VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, templateId); + // todo : check for dummy template + if (template == null) { + throw new InvalidParameterValueException("Unable to use template " + templateId); + } + + if (template.isDeployAsIs()) { + throw new InvalidParameterValueException("Deploy as is template not supported"); + } + + Long diskOfferingId = cmd.getDiskOfferingId(); + DiskOffering diskOffering = null; + if (diskOfferingId != null) { + diskOffering = _entityMgr.findById(DiskOffering.class, diskOfferingId); + if (diskOffering == null) { + throw new InvalidParameterValueException("Unable to find disk offering " + diskOfferingId); + } + if (diskOffering.isComputeOnly()) { + throw new InvalidParameterValueException(String.format("The disk offering id %d provided is directly mapped to a service offering, please provide an individual disk offering", diskOfferingId)); + } + } + + DiskOffering diskOfferingMappedInServiceOffering = _entityMgr.findById(DiskOffering.class, serviceOffering.getDiskOfferingId()); + if (diskOfferingMappedInServiceOffering.isUseLocalStorage()) { + throw new InvalidParameterValueException("Local storage disk offering not supported for instance created from backup"); + } + if (diskOffering != null && diskOffering.isUseLocalStorage()) { + throw new InvalidParameterValueException("Local storage disk offering not supported for instance created from backup"); + } + + List networkIds = cmd.getNetworkIds(); + LinkedHashMap userVmNetworkMap = getVmOvfNetworkMapping(zone, owner, template, cmd.getVmNetworkMap()); + if (MapUtils.isNotEmpty(userVmNetworkMap)) { + networkIds = new ArrayList<>(userVmNetworkMap.values()); + } + + if (cmd.getUserData() != null) { + throw new InvalidParameterValueException("User data not supported for instance created from backup"); + } + + String name = cmd.getName(); + String displayName = cmd.getDisplayName(); + Long size = cmd.getSize(); + Map dataDiskTemplateToDiskOfferingMap = cmd.getDataDiskTemplateToDiskOfferingMap(); + List sshKeyPairs = new ArrayList(); + String ipAddress = cmd.getIpAddress(); + String ip6Address = cmd.getIp6Address(); + String macAddress = cmd.getMacAddress(); + IpAddresses addrs = new IpAddresses(ipAddress, ip6Address, macAddress); + Map userVmOVFProperties = new HashMap<>(); + + UserVm vm = null; + if (zone.getNetworkType() == NetworkType.Basic) { + if (networkIds != null) { + throw new InvalidParameterValueException("Can't specify network Ids in Basic zone"); + } else { + vm = createBasicSecurityGroupVirtualMachine(zone, serviceOffering, template, getSecurityGroupIdList(cmd, zone, template, owner), owner, name, displayName, diskOfferingId, + size , null , cmd.getHypervisor(), cmd.getHttpMethod(), null, null, null, sshKeyPairs, cmd.getIpToNetworkMap(), addrs, null , null , cmd.getAffinityGroupIdList(), + cmd.getDetails(), cmd.getCustomId(), cmd.getDhcpOptionsMap(), + dataDiskTemplateToDiskOfferingMap, userVmOVFProperties, false, overrideDiskOfferingId); + } + } else { + if (_networkModel.checkSecurityGroupSupportForNetwork(owner, zone, networkIds, + cmd.getSecurityGroupIdList())) { + vm = createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, networkIds, getSecurityGroupIdList(cmd, zone, template, owner), owner, name, + displayName, diskOfferingId, size, null, cmd.getHypervisor(), cmd.getHttpMethod(), null, null, null, sshKeyPairs, cmd.getIpToNetworkMap(), addrs, null, null, + cmd.getAffinityGroupIdList(), cmd.getDetails(), cmd.getCustomId(), cmd.getDhcpOptionsMap(), + dataDiskTemplateToDiskOfferingMap, userVmOVFProperties, false, overrideDiskOfferingId, null); + + } else { + if (cmd.getSecurityGroupIdList() != null && !cmd.getSecurityGroupIdList().isEmpty()) { + throw new InvalidParameterValueException("Can't create vm with security groups; security group feature is not enabled per zone"); + } + vm = createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, name, displayName, diskOfferingId, size, null, + cmd.getHypervisor(), cmd.getHttpMethod(), null, null, null, sshKeyPairs, cmd.getIpToNetworkMap(), addrs, null, null, cmd.getAffinityGroupIdList(), cmd.getDetails(), + cmd.getCustomId(), cmd.getDhcpOptionsMap(), dataDiskTemplateToDiskOfferingMap, userVmOVFProperties, false, null, overrideDiskOfferingId); + } + } + + return vm; + } + + @Override + public UserVm restoreVMFromBackup(CreateVMFromBackupCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException, ResourceAllocationException { + long vmId = cmd.getEntityId(); + Map diskOfferingMap = cmd.getDataDiskTemplateToDiskOfferingMap(); + Map additonalParams = new HashMap<>(); + UserVm vm; + try { + vm = startVirtualMachine(vmId, null, null, null, diskOfferingMap, additonalParams, null); + } catch (ResourceUnavailableException e) { + throw new CloudRuntimeException("Unable to start the instance " + e); + } + boolean status = false; + try { + VirtualMachineEntity vmEntity = _orchSrvc.getVirtualMachine(vm.getUuid()); + status = vmEntity.stop(Long.toString(CallContext.current().getCallingUserId())); + if (!status) { + // todo : error handling + } + } catch (ResourceUnavailableException e) { + throw new CloudRuntimeException("Unable to contact the agent to stop the instance before restore " + e); + // todo : error handling + } catch (CloudException e) { + throw new CloudRuntimeException("Unable to contact the agent to stop the instance before restore " + e); + // todo : error handling + } + + backupManager.restoreBackupToVM(cmd.getBackupId(), vmId); + + if (cmd.getStartVm()) { + try { + vm = startVirtualMachine(vmId, null, null, null, diskOfferingMap, additonalParams, null); + } catch (ResourceUnavailableException e) { + throw new CloudRuntimeException("Unable to start the instance " + e); + } + } + return vm; + } + /* Generate usage events related to unmanaging a VM */ diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 6e13ba135df3..01f35d7ff1f0 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -29,6 +29,7 @@ import java.util.stream.Collectors; import com.amazonaws.util.CollectionUtils; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.storage.VolumeApiService; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.VirtualMachineManager; @@ -57,6 +58,7 @@ import org.apache.cloudstack.api.command.user.backup.repository.AddBackupRepositoryCmd; import org.apache.cloudstack.api.command.user.backup.repository.DeleteBackupRepositoryCmd; import org.apache.cloudstack.api.command.user.backup.repository.ListBackupRepositoriesCmd; +import org.apache.cloudstack.api.command.user.vm.CreateVMFromBackupCmd; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.BackupScheduleDao; @@ -185,7 +187,7 @@ public List listBackupProviderOfferings(final Long zoneId) { if (zoneId == null || zoneId < 1) { throw new CloudRuntimeException("Invalid zone ID passed"); } - validateForZone(zoneId); + validateBackupForZone(zoneId); final Account account = CallContext.current().getCallingAccount(); if (!accountService.isRootAdmin(account.getId())) { throw new PermissionDeniedException("Parameter external can only be specified by a Root Admin, permission denied"); @@ -198,7 +200,7 @@ public List listBackupProviderOfferings(final Long zoneId) { @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_IMPORT_OFFERING, eventDescription = "importing backup offering", async = true) public BackupOffering importBackupOffering(final ImportBackupOfferingCmd cmd) { - validateForZone(cmd.getZoneId()); + validateBackupForZone(cmd.getZoneId()); final BackupOffering existingOffering = backupOfferingDao.findByExternalId(cmd.getExternalId(), cmd.getZoneId()); if (existingOffering != null) { throw new CloudRuntimeException("A backup offering with external ID " + cmd.getExternalId() + " already exists"); @@ -273,7 +275,7 @@ public boolean deleteBackupOffering(final Long offeringId) { throw new CloudRuntimeException("Backup offering is assigned to VMs, remove the assignment(s) in order to remove the offering."); } - validateForZone(offering.getZoneId()); + validateBackupForZone(offering.getZoneId()); return backupOfferingDao.remove(offering.getId()); } @@ -294,7 +296,7 @@ public boolean assignVMToBackupOffering(Long vmId, Long offeringId) { throw new CloudRuntimeException("VM is not in running or stopped state"); } - validateForZone(vm.getDataCenterId()); + validateBackupForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); @@ -360,7 +362,7 @@ public boolean removeVMFromBackupOffering(final Long vmId, final boolean forced) throw new CloudRuntimeException(String.format("Can't find any VM with ID: [%s].", vmId)); } - validateForZone(vm.getDataCenterId()); + validateBackupForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); final BackupOfferingVO offering = backupOfferingDao.findById(vm.getBackupOfferingId()); @@ -422,7 +424,7 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { } final VMInstanceVO vm = findVmById(vmId); - validateForZone(vm.getDataCenterId()); + validateBackupForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); if (vm.getBackupOfferingId() == null) { @@ -462,7 +464,7 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { @Override public List listBackupSchedule(final Long vmId) { final VMInstanceVO vm = findVmById(vmId); - validateForZone(vm.getDataCenterId()); + validateBackupForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); return backupScheduleDao.listByVM(vmId).stream().map(BackupSchedule.class::cast).collect(Collectors.toList()); @@ -472,7 +474,7 @@ public List listBackupSchedule(final Long vmId) { @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_SCHEDULE_DELETE, eventDescription = "deleting VM backup schedule") public boolean deleteBackupSchedule(final Long vmId) { final VMInstanceVO vm = findVmById(vmId); - validateForZone(vm.getDataCenterId()); + validateBackupForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); final BackupSchedule schedule = backupScheduleDao.findByVM(vmId); @@ -486,7 +488,7 @@ public boolean deleteBackupSchedule(final Long vmId) { @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_CREATE, eventDescription = "creating VM backup", async = true) public boolean createBackup(final Long vmId) { final VMInstanceVO vm = findVmById(vmId); - validateForZone(vm.getDataCenterId()); + validateBackupForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); if (vm.getBackupOfferingId() == null) { @@ -611,7 +613,7 @@ public boolean restoreBackup(final Long backupId) { if (backup == null) { throw new CloudRuntimeException("Backup " + backupId + " does not exist"); } - validateForZone(backup.getZoneId()); + validateBackupForZone(backup.getZoneId()); final VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); if (vm == null) { @@ -749,6 +751,79 @@ private Backup.VolumeInfo getVolumeInfo(List backedUpVolumes, return null; } + @Override + @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_RESTORE_TO_NEW_VM, eventDescription = "Resoring backup to new vm", async = true) + public boolean restoreBackupToVM(final Long backupId, final Long vmId) throws ResourceUnavailableException { + final BackupVO backup = backupDao.findById(backupId); + if (backup == null) { + throw new CloudRuntimeException("Backup " + backupId + " does not exist"); + } + validateBackupForZone(backup.getZoneId()); + + VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(vmId); + if (vm == null) { + throw new CloudRuntimeException("VM ID " + backup.getVmId() + " couldn't be found on existing or removed VMs"); + } + accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); + + if (vm.getRemoved() != null) { + throw new CloudRuntimeException("VM is removed from CloudStack"); + } + if (vm.getRemoved() != null) { + throw new CloudRuntimeException("VM should be in the stopped state"); + } + if (!vm.getState().equals(VirtualMachine.State.Stopped)) { + throw new CloudRuntimeException("Existing VM should be stopped before being restored from backup"); + } + + List backupVolumes = backup.getBackedUpVolumes(); + if (backupVolumes == null) { + throw new CloudRuntimeException("Backed-up volumes not found"); + } + + List vmVolumes = volumeDao.findByInstance(vmId); + if (vmVolumes.size() != backupVolumes.size()) { + throw new CloudRuntimeException("Unable to restore VM with the current backup as the backup has different number of disks as the VM"); + } + + BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); + if (offering == null) { + String errorMessage = "Failed to find backup offering of the VM backup."; + throw new CloudRuntimeException("Failed to find backup offering of the VM backup."); + } + + String backupDetailsInMessage = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(backup, "uuid", "externalId", "newVMId", "type", "status", "date"); + try { + updateVmState(vm, VirtualMachine.Event.RestoringRequested, VirtualMachine.State.Restoring); + updateVolumeState(vm, Volume.Event.RestoreRequested, Volume.State.Restoring); + ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventTypes.EVENT_VM_BACKUP_RESTORE, + String.format("Restoring VM %s from backup %s", vm.getUuid(), backup.getUuid()), + vm.getId(), ApiCommandResourceType.VirtualMachine.toString(), + true, 0); + + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); + if (!backupProvider.restoreBackupToVM(vm, backup)) { + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_ERROR, EventTypes.EVENT_VM_BACKUP_RESTORE, + String.format("Failed to restore VM %s from backup %s", vm.getInstanceName(), backup.getUuid()), + vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); + throw new CloudRuntimeException("Error restoring VM from backup with uuid " + backup.getUuid()); + } + // The restore process is executed by a backup provider outside of ACS, I am using the catch-all (Exception) to + // ensure that no provider-side exception is missed. Therefore, we have a proper handling of exceptions, and rollbacks if needed. + } catch (Exception e) { + logger.error(String.format("Failed to restore backup [%s] due to: [%s].", backupDetailsInMessage, e.getMessage()), e); + updateVolumeState(vm, Volume.Event.RestoreFailed, Volume.State.Ready); + updateVmState(vm, VirtualMachine.Event.RestoringFailed, VirtualMachine.State.Stopped); + throw new CloudRuntimeException(String.format("Error restoring VM from backup [%s].", backupDetailsInMessage)); + } + updateVolumeState(vm, Volume.Event.RestoreSucceeded, Volume.State.Ready); + updateVmState(vm, VirtualMachine.Event.RestoringSuccess, VirtualMachine.State.Stopped); + + return importRestoredVM(vm.getDataCenterId(), vm.getDomainId(), vm.getAccountId(), vm.getUserId(), + vm.getInstanceName(), vm.getHypervisorType(), backup); + + } + @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_RESTORE, eventDescription = "restoring VM from backup", async = true) public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, final Long backupId, final Long vmId) throws Exception { @@ -759,7 +834,7 @@ public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, if (backup == null) { throw new CloudRuntimeException("Provided backup not found"); } - validateForZone(backup.getZoneId()); + validateBackupForZone(backup.getZoneId()); final VMInstanceVO vm = findVmById(vmId); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); @@ -850,7 +925,7 @@ public boolean deleteBackup(final Long backupId, final Boolean forced) { if (vm == null) { throw new CloudRuntimeException("VM " + vmId + " does not exist"); } - validateForZone(vm.getDataCenterId()); + validateBackupForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); final BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); if (offering == null) { @@ -933,7 +1008,8 @@ public boolean isDisabled(final Long zoneId) { return !(BackupFrameworkEnabled.value() && BackupFrameworkEnabled.valueIn(zoneId)); } - private void validateForZone(final Long zoneId) { + @Override + public void validateBackupForZone(final Long zoneId) { if (zoneId == null || isDisabled(zoneId)) { throw new CloudRuntimeException("Backup and Recovery feature is disabled for the zone"); } @@ -991,6 +1067,7 @@ public List> getCommands() { cmdList.add(AddBackupRepositoryCmd.class); cmdList.add(DeleteBackupRepositoryCmd.class); cmdList.add(ListBackupRepositoriesCmd.class); + cmdList.add(CreateVMFromBackupCmd.class); return cmdList; } From e094e1456c315f368005107852e0182486d2646b Mon Sep 17 00:00:00 2001 From: abh1sar Date: Wed, 25 Dec 2024 10:39:16 +0530 Subject: [PATCH 002/147] Nas backup provider changes --- .../backup/RestoreBackupCommand.java | 19 ++++++++++---- .../cloudstack/backup/NASBackupProvider.java | 17 +++++++++--- .../LibvirtRestoreBackupCommandWrapper.java | 26 +++++++++---------- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java index 7228e35147af..f447fbe3d008 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java @@ -30,7 +30,8 @@ public class RestoreBackupCommand extends Command { private String backupPath; private String backupRepoType; private String backupRepoAddress; - private List volumePaths; + private List backupVolumesUUIDs; + private List restoreVolumePaths; private String diskType; private Boolean vmExists; private String restoreVolumeUUID; @@ -72,12 +73,12 @@ public void setBackupRepoAddress(String backupRepoAddress) { this.backupRepoAddress = backupRepoAddress; } - public List getVolumePaths() { - return volumePaths; + public List getRestoreVolumePaths() { + return restoreVolumePaths; } - public void setVolumePaths(List volumePaths) { - this.volumePaths = volumePaths; + public void setRestoreVolumePaths(List restoreVolumePaths) { + this.restoreVolumePaths = restoreVolumePaths; } public Boolean isVmExists() { @@ -127,4 +128,12 @@ public void setVmState(VirtualMachine.State vmState) { public boolean executeInSequence() { return true; } + + public List getBackupVolumesUUIDs() { + return backupVolumesUUIDs; + } + + public void setBackupVolumesUUIDs(List backupVolumesUUIDs) { + this.backupVolumesUUIDs = backupVolumesUUIDs; + } } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 4a6725abdca5..6c6cfc799b9b 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -208,10 +208,20 @@ private BackupVO createBackupObject(VirtualMachine vm, String backupPath) { return backupDao.persist(backup); } + @Override + public boolean restoreBackupToVM(VirtualMachine vm, Backup backup) { + return restoreVMBackup(vm, backup); + } + @Override public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { + return restoreVMBackup(vm, backup); + } + + private boolean restoreVMBackup(VirtualMachine vm, Backup backup) { List backedVolumes = backup.getBackedUpVolumes(); - List volumes = backedVolumes.stream().map(volume -> volumeDao.findByUuid(volume.getUuid())).collect(Collectors.toList()); + List backedVolumesUUIDs = backedVolumes.stream().map(volume -> volume.getUuid()).collect(Collectors.toList()); + List restoreVolumes = volumeDao.findByInstance(vm.getId()); LOG.debug("Restoring vm {} from backup {} on the NAS Backup Provider", vm.getUuid(), backup.getUuid()); BackupRepository backupRepository = getBackupRepository(vm, backup); @@ -222,7 +232,8 @@ public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { restoreCommand.setBackupRepoType(backupRepository.getType()); restoreCommand.setBackupRepoAddress(backupRepository.getAddress()); restoreCommand.setVmName(vm.getName()); - restoreCommand.setVolumePaths(getVolumePaths(volumes)); + restoreCommand.setBackupVolumesUUIDs(backedVolumesUUIDs); + restoreCommand.setRestoreVolumePaths(getVolumePaths(restoreVolumes)); restoreCommand.setVmExists(vm.getRemoved() == null); restoreCommand.setVmState(vm.getState()); @@ -287,7 +298,7 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU restoreCommand.setBackupRepoType(backupRepository.getType()); restoreCommand.setBackupRepoAddress(backupRepository.getAddress()); restoreCommand.setVmName(vmNameAndState.first()); - restoreCommand.setVolumePaths(Collections.singletonList(String.format("%s/%s", dataStore.getLocalPath(), volumeUUID))); + restoreCommand.setRestoreVolumePaths(Collections.singletonList(String.format("%s/%s", dataStore.getLocalPath(), volumeUUID))); restoreCommand.setDiskType(volume.getVolumeType().name().toLowerCase(Locale.ROOT)); restoreCommand.setVmExists(null); restoreCommand.setVmState(vmNameAndState.second()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 23ead355096d..30e45a70dafe 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -58,36 +58,38 @@ public Answer execute(RestoreBackupCommand command, LibvirtComputingResource ser String mountOptions = command.getMountOptions(); Boolean vmExists = command.isVmExists(); String diskType = command.getDiskType(); - List volumePaths = command.getVolumePaths(); + List backedVolumeUUIDs = command.getBackupVolumesUUIDs(); + List restoreVolumePaths = command.getRestoreVolumePaths(); String restoreVolumeUuid = command.getRestoreVolumeUUID(); String newVolumeId = null; if (Objects.isNull(vmExists)) { - String volumePath = volumePaths.get(0); + String volumePath = restoreVolumePaths.get(0); int lastIndex = volumePath.lastIndexOf("/"); newVolumeId = volumePath.substring(lastIndex + 1); restoreVolume(backupPath, backupRepoType, backupRepoAddress, volumePath, diskType, restoreVolumeUuid, new Pair<>(vmName, command.getVmState())); } else if (Boolean.TRUE.equals(vmExists)) { - restoreVolumesOfExistingVM(volumePaths, backupPath, backupRepoType, backupRepoAddress, mountOptions); + restoreVolumesOfExistingVM(restoreVolumePaths, backedVolumeUUIDs, backupPath, backupRepoType, backupRepoAddress, mountOptions); } else { - restoreVolumesOfDestroyedVMs(volumePaths, vmName, backupPath, backupRepoType, backupRepoAddress, mountOptions); + restoreVolumesOfDestroyedVMs(restoreVolumePaths, vmName, backupPath, backupRepoType, backupRepoAddress, mountOptions); } return new BackupAnswer(command, true, newVolumeId); } - private void restoreVolumesOfExistingVM(List volumePaths, String backupPath, - String backupRepoType, String backupRepoAddress, String mountOptions) { + private void restoreVolumesOfExistingVM(List restoreVolumePaths, List backedVolumesUUIDs, String backupPath, + String backupRepoType, String backupRepoAddress, String mountOptions) { String diskType = "root"; String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType); try { - for (int idx = 0; idx < volumePaths.size(); idx++) { - String volumePath = volumePaths.get(idx); - Pair bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, null); + for (int idx = 0; idx < restoreVolumePaths.size(); idx++) { + String restoreVolumePath = restoreVolumePaths.get(idx); + String backupVolumeUuid = backedVolumesUUIDs.get(idx); + Pair bkpPathAndVolUuid = getBackupPath(mountDirectory, null, backupPath, diskType, backupVolumeUuid); diskType = "datadisk"; try { - replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first()); + replaceVolumeWithBackup(restoreVolumePath, bkpPathAndVolUuid.first()); } catch (IOException e) { throw new CloudRuntimeException(String.format("Unable to revert backup for volume [%s] due to [%s].", bkpPathAndVolUuid.second(), e.getMessage()), e); } @@ -96,7 +98,6 @@ private void restoreVolumesOfExistingVM(List volumePaths, String backupP unmountBackupDirectory(mountDirectory); deleteTemporaryDirectory(mountDirectory); } - } private void restoreVolumesOfDestroyedVMs(List volumePaths, String vmName, String backupPath, @@ -177,8 +178,7 @@ private void deleteTemporaryDirectory(String backupDirectory) { private Pair getBackupPath(String mountDirectory, String volumePath, String backupPath, String diskType, String volumeUuid) { String bkpPath = String.format(FILE_PATH_PLACEHOLDER, mountDirectory, backupPath); - int lastIndex = volumePath.lastIndexOf(File.separator); - String volUuid = Objects.isNull(volumeUuid) ? volumePath.substring(lastIndex + 1) : volumeUuid; + String volUuid = Objects.isNull(volumeUuid) ? volumePath.substring(volumePath.lastIndexOf(File.separator) + 1) : volumeUuid; String backupFileName = String.format("%s.%s.qcow2", diskType.toLowerCase(Locale.ROOT), volUuid); bkpPath = String.format(FILE_PATH_PLACEHOLDER, bkpPath, backupFileName); return new Pair<>(bkpPath, volUuid); From 27feedaabdbf1e9571b82c84f317c28f3b7f4e67 Mon Sep 17 00:00:00 2001 From: abh1sar Date: Mon, 30 Dec 2024 23:29:58 +0530 Subject: [PATCH 003/147] Add hypervisor, service_offering and template to backups table. --- .../org/apache/cloudstack/backup/Backup.java | 4 ++ .../apache/cloudstack/backup/BackupVO.java | 46 ++++++++++++++++++- .../META-INF/db/schema-42010to42100.sql | 7 +++ .../backup/DummyBackupProvider.java | 3 ++ .../cloudstack/backup/NASBackupProvider.java | 3 ++ .../backup/NetworkerBackupProvider.java | 3 ++ .../backup/networker/NetworkerClient.java | 3 ++ .../backup/VeeamBackupProvider.java | 3 ++ 8 files changed, 71 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index f21f20adb33e..e1c9b329c075 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -25,6 +25,7 @@ import org.apache.cloudstack.api.InternalIdentity; import org.apache.commons.lang3.StringUtils; +import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.Volume; public interface Backup extends ControlledEntity, InternalIdentity, Identity { @@ -144,4 +145,7 @@ public String toString() { Long getProtectedSize(); List getBackedUpVolumes(); long getZoneId(); + Hypervisor.HypervisorType getHypervisorType(); + long getServiceOfferingId(); + long getTemplateId(); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index 9b285e66cab9..82c119e3a800 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -17,8 +17,11 @@ package org.apache.cloudstack.backup; +import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.db.GenericDao; import com.google.gson.Gson; + +import org.apache.cloudstack.util.HypervisorTypeConverter; import org.apache.commons.lang3.StringUtils; import java.util.Arrays; @@ -28,6 +31,7 @@ import java.util.UUID; import javax.persistence.Column; +import javax.persistence.Convert; import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; @@ -90,6 +94,16 @@ public class BackupVO implements Backup { @Column(name = "backed_volumes", length = 65535) protected String backedUpVolumes; + @Column(name = "hypervisor_type") + @Convert(converter = HypervisorTypeConverter.class) + protected Hypervisor.HypervisorType hypervisorType; + + @Column(name = "service_offering_id") + protected long serviceOfferingId; + + @Column(name = "vm_template_id", updatable = true, nullable = true, length = 17) + protected Long templateId = new Long(-1); + public BackupVO() { this.uuid = UUID.randomUUID().toString(); } @@ -222,10 +236,40 @@ public void setBackedUpVolumes(String backedUpVolumes) { this.backedUpVolumes = backedUpVolumes; } + @Override + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + public void setHypervisorType(Hypervisor.HypervisorType hypervisorType) { + this.hypervisorType = hypervisorType; + } + + @Override + public long getServiceOfferingId() { + return serviceOfferingId; + } + + public void setServiceOfferingId(long serviceOfferingId) { + this.serviceOfferingId = serviceOfferingId; + } + + @Override + public long getTemplateId() { + if (templateId == null) { + return -1; + } else { + return templateId; + } + } + + public void setTemplateId(Long templateId) { + this.templateId = templateId; + } + public Date getRemoved() { return removed; } - public void setRemoved(Date removed) { this.removed = removed; } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 91223bab798e..1ccf9774fa29 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -24,3 +24,10 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'console_endpoint_ -- Add client_address column to cloud.console_session table CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'client_address', 'VARCHAR(45)'); + +-- Add columns to backup for restoring backups of expunged VMs +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'hypervisor_type', 'CHAR(32)'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'service_offering_id', 'BIGINT UNSIGNED NOT NULL COMMENT \'service offering id\''); +ALTER TABLE `cloud`.`backups` ADD CONSTRAINT `fk_service_offering_id` FOREIGN KEY (`service_offering_id`) REFERENCES `cloud`.`service_offering`(`id`); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'vm_template_id', 'BIGINT UNSIGNED'); +ALTER TABLE `cloud`.`backups` ADD CONSTRAINT `fk_vm_template_id` FOREIGN KEY (`vm_template_id`) REFERENCES `cloud`.`vm_template`(`id`); diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index ae27090be86e..f6466a335738 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -127,6 +127,9 @@ public boolean takeBackup(VirtualMachine vm) { backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); + backup.setHypervisorType(vm.getHypervisorType()); + backup.setServiceOfferingId(vm.getServiceOfferingId()); + backup.setTemplateId(vm.getTemplateId()); return backupDao.persist(backup) != null; } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 6c6cfc799b9b..b9095a982d74 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -205,6 +205,9 @@ private BackupVO createBackupObject(VirtualMachine vm, String backupPath) { backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); + backup.setHypervisorType(vm.getHypervisorType()); + backup.setServiceOfferingId(vm.getServiceOfferingId()); + backup.setTemplateId(vm.getTemplateId()); return backupDao.persist(backup); } diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index 7751d5e36e09..5a6f7892be36 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -627,6 +627,9 @@ public void doInTransactionWithoutResult(TransactionStatus status) { strayBackup.setAccountId(vm.getAccountId()); strayBackup.setDomainId(vm.getDomainId()); strayBackup.setZoneId(vm.getDataCenterId()); + strayBackup.setHypervisorType(vm.getHypervisorType()); + strayBackup.setServiceOfferingId(vm.getServiceOfferingId()); + strayBackup.setTemplateId(vm.getTemplateId()); LOG.debug(String.format("Creating a new entry in backups: [uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " + "domain_id: %s, zone_id: %s].", strayBackup.getUuid(), strayBackup.getVmId(), strayBackup.getExternalId(), strayBackup.getType(), strayBackup.getDate(), strayBackup.getBackupOfferingId(), strayBackup.getAccountId(), diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java index 8aecaa26023e..670d0090790c 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java @@ -267,6 +267,9 @@ public BackupVO registerBackupForVm(VirtualMachine vm, Date backupJobStart, Stri backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); + backup.setHypervisorType(vm.getHypervisorType()); + backup.setServiceOfferingId(vm.getServiceOfferingId()); + backup.setTemplateId(vm.getTemplateId()); return backup; } catch (final IOException e) { LOG.error("Failed to register backup from EMC Networker due to:", e); diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index dc176062de29..5f60d7fd77c5 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -378,6 +378,9 @@ public void doInTransactionWithoutResult(TransactionStatus status) { backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); + backup.setHypervisorType(vm.getHypervisorType()); + backup.setServiceOfferingId(vm.getServiceOfferingId()); + backup.setTemplateId(vm.getTemplateId()); logger.debug(String.format("Creating a new entry in backups: [uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " + "domain_id: %s, zone_id: %s].", backup.getUuid(), backup.getVmId(), backup.getExternalId(), backup.getType(), backup.getDate(), From f98da37c171283d66340d96fe0974d910bb73f6f Mon Sep 17 00:00:00 2001 From: abh1sar Date: Wed, 1 Jan 2025 17:15:59 +0530 Subject: [PATCH 004/147] Create backup_details table and add details related to vm. --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../user/vm/CreateVMFromBackupCmd.java | 5 +- .../api/response/BackupResponse.java | 13 +++ .../org/apache/cloudstack/backup/Backup.java | 6 +- .../cloudstack/backup/BackupDetailVO.java | 98 +++++++++++++++++++ .../apache/cloudstack/backup/BackupVO.java | 52 ++++------ .../cloudstack/backup/dao/BackupDao.java | 2 + .../cloudstack/backup/dao/BackupDaoImpl.java | 49 ++++++++++ .../backup/dao/BackupDetailsDao.java | 26 +++++ .../backup/dao/BackupDetailsDaoImpl.java | 31 ++++++ ...spring-engine-schema-core-daos-context.xml | 1 + .../META-INF/db/schema-42010to42100.sql | 16 +-- .../backup/DummyBackupProvider.java | 19 +++- .../cloudstack/backup/NASBackupProvider.java | 20 +++- .../backup/NetworkerBackupProvider.java | 20 +++- .../backup/networker/NetworkerClient.java | 22 ++++- .../backup/VeeamBackupProvider.java | 13 ++- 17 files changed, 333 insertions(+), 61 deletions(-) create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/backup/BackupDetailVO.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDao.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDaoImpl.java diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index bf8b79b29d0c..89646e6c267b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -551,6 +551,7 @@ public class ApiConstants { public static final String IS_DEDICATED = "isdedicated"; public static final String TAKEN = "taken"; public static final String VM_AVAILABLE = "vmavailable"; + public static final String VM_DETAILS = "vmdetails"; public static final String VM_LIMIT = "vmlimit"; public static final String VM_TOTAL = "vmtotal"; public static final String VM_TYPE = "vmtype"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java index 12959cbc2984..c0a8cf5a9b59 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java @@ -18,6 +18,7 @@ import javax.inject.Inject; +import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; @@ -38,7 +39,9 @@ responseView = ResponseObject.ResponseView.Restricted, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, - since = "4.21.0") + since = "4.21.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) + public class CreateVMFromBackupCmd extends DeployVMCmd implements UserCmd { @Inject diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java index 63419680fea3..06a65b49b53a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java @@ -26,6 +26,7 @@ import com.google.gson.annotations.SerializedName; import java.util.Date; +import java.util.Map; @EntityReference(value = Backup.class) public class BackupResponse extends BaseResponse { @@ -102,6 +103,10 @@ public class BackupResponse extends BaseResponse { @Param(description = "zone name") private String zone; + @SerializedName(ApiConstants.VM_DETAILS) + @Param(description = "Lists the vm specific details for the backup") + private Map vmDetails; + public String getId() { return id; } @@ -245,4 +250,12 @@ public String getZone() { public void setZone(String zone) { this.zone = zone; } + + public Map getVmDetails() { + return vmDetails; + } + + public void setVmDetails(Map vmDetails) { + this.vmDetails = vmDetails; + } } diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index e1c9b329c075..2b57f6343f80 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -19,13 +19,13 @@ import java.util.Date; import java.util.List; +import java.util.Map; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; import org.apache.commons.lang3.StringUtils; -import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.Volume; public interface Backup extends ControlledEntity, InternalIdentity, Identity { @@ -145,7 +145,5 @@ public String toString() { Long getProtectedSize(); List getBackedUpVolumes(); long getZoneId(); - Hypervisor.HypervisorType getHypervisorType(); - long getServiceOfferingId(); - long getTemplateId(); + Map getDetails(); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupDetailVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupDetailVO.java new file mode 100644 index 000000000000..52baca11282e --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupDetailVO.java @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.api.ResourceDetail; + +@Entity +@Table(name = "backup_details") +public class BackupDetailVO implements ResourceDetail { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "backup_id") + private long resourceId; + + @Column(name = "name") + private String name; + + @Column(name = "value", length = 1024) + private String value; + + @Column(name = "display") + private boolean display = true; + + public BackupDetailVO() { + } + + public BackupDetailVO(long templateId, String name, String value, boolean display) { + this.resourceId = templateId; + this.name = name; + this.value = value; + this.display = display; + } + + @Override + public long getId() { + return id; + } + + @Override + public long getResourceId() { + return resourceId; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return display; + } + + public void setId(long id) { + this.id = id; + } + + public void setResourceId(long resourceId) { + this.resourceId = resourceId; + } + + public void setName(String name) { + this.name = name; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index 82c119e3a800..dfae71d73e90 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -17,21 +17,20 @@ package org.apache.cloudstack.backup; -import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.db.GenericDao; import com.google.gson.Gson; -import org.apache.cloudstack.util.HypervisorTypeConverter; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.lang3.StringUtils; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.UUID; import javax.persistence.Column; -import javax.persistence.Convert; import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; @@ -41,6 +40,7 @@ import javax.persistence.Table; import javax.persistence.Temporal; import javax.persistence.TemporalType; +import javax.persistence.Transient; @Entity @Table(name = "backups") @@ -94,15 +94,8 @@ public class BackupVO implements Backup { @Column(name = "backed_volumes", length = 65535) protected String backedUpVolumes; - @Column(name = "hypervisor_type") - @Convert(converter = HypervisorTypeConverter.class) - protected Hypervisor.HypervisorType hypervisorType; - - @Column(name = "service_offering_id") - protected long serviceOfferingId; - - @Column(name = "vm_template_id", updatable = true, nullable = true, length = 17) - protected Long templateId = new Long(-1); + @Transient + Map details; public BackupVO() { this.uuid = UUID.randomUUID().toString(); @@ -237,35 +230,19 @@ public void setBackedUpVolumes(String backedUpVolumes) { } @Override - public Hypervisor.HypervisorType getHypervisorType() { - return hypervisorType; + public Map getDetails() { + return details; } - public void setHypervisorType(Hypervisor.HypervisorType hypervisorType) { - this.hypervisorType = hypervisorType; + public void setDetail(String name, String value) { + assert (details != null) : "Did you forget to load the details?"; + this.details.put(name, value); } - @Override - public long getServiceOfferingId() { - return serviceOfferingId; + public void setDetails(Map details) { + this.details = details; } - public void setServiceOfferingId(long serviceOfferingId) { - this.serviceOfferingId = serviceOfferingId; - } - - @Override - public long getTemplateId() { - if (templateId == null) { - return -1; - } else { - return templateId; - } - } - - public void setTemplateId(Long templateId) { - this.templateId = templateId; - } public Date getRemoved() { return removed; @@ -273,4 +250,9 @@ public Date getRemoved() { public void setRemoved(Date removed) { this.removed = removed; } + + @Override + public String toString() { + return String.format("Backup is %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "uuid", "external_id", "date")); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java index 89a13245b0a0..d29af3fb89d8 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java @@ -35,5 +35,7 @@ public interface BackupDao extends GenericDao { List syncBackups(Long zoneId, Long vmId, List externalBackups); BackupVO getBackupVO(Backup backup); List listByOfferingId(Long backupOfferingId); + void loadDetails(BackupVO backup); + void saveDetails(BackupVO backup); BackupResponse newBackupResponse(Backup backup); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java index 5a9cd0620374..708588ee35ed 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java @@ -18,14 +18,18 @@ package org.apache.cloudstack.backup.dao; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import javax.annotation.PostConstruct; import javax.inject.Inject; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupDetailVO; import org.apache.cloudstack.backup.BackupOffering; import org.apache.cloudstack.backup.BackupVO; @@ -38,6 +42,8 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.VMInstanceDao; import com.google.gson.Gson; @@ -59,6 +65,9 @@ public class BackupDaoImpl extends GenericDaoBase implements Bac @Inject BackupOfferingDao backupOfferingDao; + @Inject + BackupDetailsDao backupDetailsDao; + private SearchBuilder backupSearch; public BackupDaoImpl() { @@ -133,6 +142,16 @@ public void removeExistingBackups(Long zoneId, Long vmId) { expunge(sc); } + @Override + public BackupVO persist(BackupVO backup) { + return Transaction.execute((TransactionCallback) status -> { + BackupVO backupDb = super.persist(backup); + saveDetails(backup); + loadDetails(backupDb); + return backupDb; + }); + } + @Override public List syncBackups(Long zoneId, Long vmId, List externalBackups) { for (Backup backup : externalBackups) { @@ -142,6 +161,26 @@ public List syncBackups(Long zoneId, Long vmId, List externalBac return listByVmId(zoneId, vmId); } + @Override + public void loadDetails(BackupVO backup) { + Map details = backupDetailsDao.listDetailsKeyPairs(backup.getId()); + backup.setDetails(details); + } + + @Override + public void saveDetails(BackupVO backup) { + Map detailsStr = backup.getDetails(); + if (detailsStr == null) { + return; + } + List details = new ArrayList(); + for (String key : detailsStr.keySet()) { + BackupDetailVO detail = new BackupDetailVO(backup.getId(), key, detailsStr.get(key), true); + details.add(detail); + } + backupDetailsDao.saveDetails(details); + } + @Override public BackupResponse newBackupResponse(Backup backup) { VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); @@ -180,6 +219,16 @@ public BackupResponse newBackupResponse(Backup backup) { response.setDomain(domain.getName()); response.setZoneId(zone.getUuid()); response.setZone(zone.getName()); + + Map details = backupDetailsDao.listDetailsKeyPairs(backup.getId(), true); + if (details != null) { + HashMap vmDetails = new HashMap<>(); + vmDetails.put(ApiConstants.HYPERVISOR, details.get(ApiConstants.HYPERVISOR)); + vmDetails.put(ApiConstants.TEMPLATE_ID, details.get(ApiConstants.TEMPLATE_ID)); + vmDetails.put(ApiConstants.SERVICE_OFFERING_ID, details.get(ApiConstants.SERVICE_OFFERING_ID)); + response.setVmDetails(vmDetails); + } + response.setObjectName("backup"); return response; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDao.java new file mode 100644 index 000000000000..664650074bce --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDao.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.dao; + +import org.apache.cloudstack.backup.BackupDetailVO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +import com.cloud.utils.db.GenericDao; + +public interface BackupDetailsDao extends GenericDao, ResourceDetailsDao { + +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDaoImpl.java new file mode 100644 index 000000000000..08c7192af909 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDaoImpl.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.dao; + + +import org.apache.cloudstack.backup.BackupDetailVO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.springframework.stereotype.Component; + +@Component +public class BackupDetailsDaoImpl extends ResourceDetailsDaoBase implements BackupDetailsDao { + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new BackupDetailVO(resourceId, key, value, display)); + } +} diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 4f22234d7bf4..23734b1001a0 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -270,6 +270,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 1ccf9774fa29..1c9b9c49b054 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -25,9 +25,13 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'console_endpoint_ -- Add client_address column to cloud.console_session table CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'client_address', 'VARCHAR(45)'); --- Add columns to backup for restoring backups of expunged VMs -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'hypervisor_type', 'CHAR(32)'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'service_offering_id', 'BIGINT UNSIGNED NOT NULL COMMENT \'service offering id\''); -ALTER TABLE `cloud`.`backups` ADD CONSTRAINT `fk_service_offering_id` FOREIGN KEY (`service_offering_id`) REFERENCES `cloud`.`service_offering`(`id`); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'vm_template_id', 'BIGINT UNSIGNED'); -ALTER TABLE `cloud`.`backups` ADD CONSTRAINT `fk_vm_template_id` FOREIGN KEY (`vm_template_id`) REFERENCES `cloud`.`vm_template`(`id`); +-- Create backup details table +CREATE TABLE `cloud`.`backup_details` ( + `id` bigint unsigned NOT NULL auto_increment, + `backup_id` bigint unsigned NOT NULL COMMENT 'backup id', + `name` varchar(255) NOT NULL, + `value` varchar(1024) NOT NULL, + `display` tinyint(1) NOT NULL DEFAULT 1 COMMENT 'Should detail be displayed to the end user', + PRIMARY KEY (`id`), + CONSTRAINT `fk_backup_details__backup_id` FOREIGN KEY `fk_backup_details__backup_id`(`backup_id`) REFERENCES `backups`(`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index f6466a335738..b4f3602202b8 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -24,11 +24,16 @@ import javax.inject.Inject; +import com.cloud.offering.ServiceOffering; import com.cloud.storage.dao.VolumeDao; + +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.backup.dao.BackupDao; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.db.EntityManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; @@ -40,6 +45,8 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { private BackupDao backupDao; @Inject private VolumeDao volumeDao; + @Inject + private EntityManager entityManager; @Override public String getName() { @@ -127,9 +134,15 @@ public boolean takeBackup(VirtualMachine vm) { backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); - backup.setHypervisorType(vm.getHypervisorType()); - backup.setServiceOfferingId(vm.getServiceOfferingId()); - backup.setTemplateId(vm.getTemplateId()); + + HashMap details = new HashMap<>(); + details.put(ApiConstants.HYPERVISOR, vm.getHypervisorType().toString()); + ServiceOffering serviceOffering = entityManager.findById(ServiceOffering.class, vm.getServiceOfferingId()); + details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); + VirtualMachineTemplate template = entityManager.findById(VirtualMachineTemplate.class, vm.getTemplateId()); + details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + backup.setDetails(details); + return backupDao.persist(backup) != null; } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index b9095a982d74..517a17f80ac4 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -25,17 +25,22 @@ import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; +import com.cloud.offering.ServiceOffering; import com.cloud.storage.ScopeType; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.db.EntityManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; + +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.BackupRepositoryDao; @@ -93,6 +98,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private AgentManager agentManager; + @Inject + private EntityManager entityManager; + protected Host getLastVMHypervisorHost(VirtualMachine vm) { Long hostId = vm.getLastHostId(); if (hostId == null) { @@ -205,9 +213,15 @@ private BackupVO createBackupObject(VirtualMachine vm, String backupPath) { backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); - backup.setHypervisorType(vm.getHypervisorType()); - backup.setServiceOfferingId(vm.getServiceOfferingId()); - backup.setTemplateId(vm.getTemplateId()); + + HashMap details = new HashMap<>(); + details.put(ApiConstants.HYPERVISOR, vm.getHypervisorType().toString()); + ServiceOffering serviceOffering = entityManager.findById(ServiceOffering.class, vm.getServiceOfferingId()); + details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); + VirtualMachineTemplate template = entityManager.findById(VirtualMachineTemplate.class, vm.getTemplateId()); + details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + backup.setDetails(details); + return backupDao.persist(backup); } diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index 5a6f7892be36..934dd62f5620 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -21,14 +21,17 @@ import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; +import com.cloud.offering.ServiceOffering; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; @@ -37,6 +40,8 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; + +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDaoImpl; @@ -117,6 +122,9 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid @Inject private VMInstanceDao vmInstanceDao; + @Inject + private EntityManager entityManager; + private static String getUrlDomain(String url) throws URISyntaxException { URI uri; try { @@ -627,9 +635,15 @@ public void doInTransactionWithoutResult(TransactionStatus status) { strayBackup.setAccountId(vm.getAccountId()); strayBackup.setDomainId(vm.getDomainId()); strayBackup.setZoneId(vm.getDataCenterId()); - strayBackup.setHypervisorType(vm.getHypervisorType()); - strayBackup.setServiceOfferingId(vm.getServiceOfferingId()); - strayBackup.setTemplateId(vm.getTemplateId()); + + HashMap details = new HashMap<>(); + details.put(ApiConstants.HYPERVISOR, vm.getHypervisorType().toString()); + ServiceOffering serviceOffering = entityManager.findById(ServiceOffering.class, vm.getServiceOfferingId()); + details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); + VirtualMachineTemplate template = entityManager.findById(VirtualMachineTemplate.class, vm.getTemplateId()); + details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + strayBackup.setDetails(details); + LOG.debug(String.format("Creating a new entry in backups: [uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " + "domain_id: %s, zone_id: %s].", strayBackup.getUuid(), strayBackup.getVmId(), strayBackup.getExternalId(), strayBackup.getType(), strayBackup.getDate(), strayBackup.getBackupOfferingId(), strayBackup.getAccountId(), diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java index 670d0090790c..1448817356a3 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java @@ -17,11 +17,16 @@ package org.apache.cloudstack.backup.networker; +import com.cloud.offering.ServiceOffering; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.db.EntityManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.nio.TrustAllManager; import com.cloud.vm.VirtualMachine; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; + +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.backup.BackupOffering; @@ -45,6 +50,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import javax.inject.Inject; import javax.net.ssl.SSLContext; import javax.net.ssl.X509TrustManager; import java.io.IOException; @@ -60,6 +66,7 @@ import java.util.ArrayList; import java.util.Base64; import java.util.Date; +import java.util.HashMap; import java.util.List; import static org.apache.cloudstack.backup.NetworkerBackupProvider.BACKUP_IDENTIFIER; @@ -71,6 +78,9 @@ public class NetworkerClient { private final String apiPassword; private final HttpClient httpClient; + @Inject + private EntityManager entityManager; + public NetworkerClient(final String url, final String username, final String password, final boolean validateCertificate, final int timeout) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { apiName = username; @@ -267,9 +277,15 @@ public BackupVO registerBackupForVm(VirtualMachine vm, Date backupJobStart, Stri backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); - backup.setHypervisorType(vm.getHypervisorType()); - backup.setServiceOfferingId(vm.getServiceOfferingId()); - backup.setTemplateId(vm.getTemplateId()); + + HashMap details = new HashMap<>(); + details.put(ApiConstants.HYPERVISOR, vm.getHypervisorType().toString()); + ServiceOffering serviceOffering = entityManager.findById(ServiceOffering.class, vm.getServiceOfferingId()); + details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); + VirtualMachineTemplate template = entityManager.findById(VirtualMachineTemplate.class, vm.getTemplateId()); + details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + backup.setDetails(details); + return backup; } catch (final IOException e) { LOG.error("Failed to register backup from EMC Networker due to:", e); diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index 5f60d7fd77c5..e5e34f2956fb 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -109,6 +109,8 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, private AgentManager agentMgr; @Inject private VirtualMachineManager virtualMachineManager; + @Inject + private EntityManager entityManager; protected VeeamClient getClient(final Long zoneId) { try { @@ -378,9 +380,14 @@ public void doInTransactionWithoutResult(TransactionStatus status) { backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); - backup.setHypervisorType(vm.getHypervisorType()); - backup.setServiceOfferingId(vm.getServiceOfferingId()); - backup.setTemplateId(vm.getTemplateId()); + + HashMap details = new HashMap<>(); + details.put(ApiConstants.HYPERVISOR, vm.getHypervisorType().toString()); + ServiceOffering serviceOffering = entityManager.findById(ServiceOffering.class, vm.getServiceOfferingId()); + details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); + VirtualMachineTemplate template = entityManager.findById(VirtualMachineTemplate.class, vm.getTemplateId()); + details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + backup.setDetails(details); logger.debug(String.format("Creating a new entry in backups: [uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " + "domain_id: %s, zone_id: %s].", backup.getUuid(), backup.getVmId(), backup.getExternalId(), backup.getType(), backup.getDate(), From 04a2deb554ff2b0521f78a3c497aad60211dbe3c Mon Sep 17 00:00:00 2001 From: abh1sar Date: Thu, 2 Jan 2025 13:57:06 +0530 Subject: [PATCH 005/147] Initial UI form --- .../user/vm/CreateVMFromBackupCmd.java | 1 + ui/public/locales/en.json | 1 + ui/src/config/section/storage.js | 11 + ui/src/views/compute/DeployVMFromBackup.vue | 2961 +++++++++++++++++ ui/src/views/storage/CreateVMFromBackup.vue | 71 + 5 files changed, 3045 insertions(+) create mode 100644 ui/src/views/compute/DeployVMFromBackup.vue create mode 100644 ui/src/views/storage/CreateVMFromBackup.vue diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java index c0a8cf5a9b59..b76ff2c5deb9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java @@ -68,6 +68,7 @@ public void create() { setEntityUuid(vm.getUuid()); } } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create vm due to exception: " + e.getMessage()); } } diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index e03aee00599b..e6eb1bccf1ed 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -2620,6 +2620,7 @@ "label.bucket.policy": "Bucket Policy", "label.usersecretkey": "Secret Key", "label.create.bucket": "Create Bucket", +"label.create.instance.from.backup": "Create new instance from backup", "message.acquire.ip.failed": "Failed to acquire IP.", "message.action.acquire.ip": "Please confirm that you want to acquire new IP.", "message.action.cancel.maintenance": "Your host has been successfully canceled for maintenance. This process can take up to several minutes.", diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index e0fa72dff8a0..02626af90250 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -433,6 +433,17 @@ export default { columns: [{ name: (record) => { return record.virtualmachinename } }, 'status', 'size', 'virtualsize', 'type', 'created', 'account', 'domain', 'zone'], details: ['virtualmachinename', 'id', 'type', 'externalid', 'size', 'virtualsize', 'volumes', 'backupofferingname', 'zone', 'account', 'domain', 'created'], actions: [ + { + api: 'createVMFromBackup', + icon: 'plus-outlined', + docHelp: 'adminguide/virtual_machines.html#restoring-vm-backups', + label: 'label.create.instance.from.backup', + message: 'message.backup.restore', + dataView: true, + popup: true, + show: (record) => { return record.state !== 'Destroyed' }, + component: shallowRef(defineAsyncComponent(() => import('@/views/storage/CreateVMFromBackup.vue'))) + }, { api: 'restoreBackup', icon: 'sync-outlined', diff --git a/ui/src/views/compute/DeployVMFromBackup.vue b/ui/src/views/compute/DeployVMFromBackup.vue new file mode 100644 index 000000000000..e45ab7fc72b4 --- /dev/null +++ b/ui/src/views/compute/DeployVMFromBackup.vue @@ -0,0 +1,2961 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + + + diff --git a/ui/src/views/storage/CreateVMFromBackup.vue b/ui/src/views/storage/CreateVMFromBackup.vue new file mode 100644 index 000000000000..9a890e814e31 --- /dev/null +++ b/ui/src/views/storage/CreateVMFromBackup.vue @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + From d3439ad8c47c41286c6b654a46735823f41e5f92 Mon Sep 17 00:00:00 2001 From: abh1sar Date: Fri, 3 Jan 2025 05:33:57 +0530 Subject: [PATCH 006/147] Save networkIds and diskOfferingIds in backup_details. Create required data volumes before restore. --- .../com/cloud/offering/DiskOfferingInfo.java | 7 ++ .../com/cloud/storage/VolumeApiService.java | 3 + .../apache/cloudstack/api/ApiConstants.java | 2 + .../user/vm/CreateVMFromBackupCmd.java | 45 ++++++++ .../cloudstack/backup/BackupManager.java | 9 ++ .../apache/cloudstack/backup/BackupVO.java | 7 ++ .../cloudstack/backup/dao/BackupDaoImpl.java | 16 +++ .../backup/DummyBackupProvider.java | 16 +-- .../cloudstack/backup/NASBackupProvider.java | 16 +-- .../backup/NetworkerBackupProvider.java | 5 + .../backup/networker/NetworkerClient.java | 17 +-- .../backup/VeeamBackupProvider.java | 13 +-- .../cloud/storage/VolumeApiServiceImpl.java | 38 ++++++- .../java/com/cloud/vm/UserVmManagerImpl.java | 7 ++ .../cloudstack/backup/BackupManagerImpl.java | 104 ++++++++++++++++++ 15 files changed, 257 insertions(+), 48 deletions(-) diff --git a/api/src/main/java/com/cloud/offering/DiskOfferingInfo.java b/api/src/main/java/com/cloud/offering/DiskOfferingInfo.java index d83039e15c2b..12dcf423e34f 100644 --- a/api/src/main/java/com/cloud/offering/DiskOfferingInfo.java +++ b/api/src/main/java/com/cloud/offering/DiskOfferingInfo.java @@ -31,6 +31,13 @@ public DiskOfferingInfo(DiskOffering diskOffering) { _diskOffering = diskOffering; } + public DiskOfferingInfo(DiskOffering diskOffering, Long size, Long minIops, Long maxIops) { + _diskOffering = diskOffering; + _size = size; + _minIops = minIops; + _maxIops = maxIops; + } + public void setDiskOffering(DiskOffering diskOffering) { _diskOffering = diskOffering; } diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 6f4c7aa09e26..7b03a7bdd0e4 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; +import com.cloud.offering.DiskOfferingInfo; import com.cloud.utils.Pair; import org.apache.cloudstack.api.command.user.volume.AssignVolumeCmd; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; @@ -106,6 +107,8 @@ public interface VolumeApiService { Volume attachVolumeToVM(AttachVolumeCmd command); + boolean createAndAttachVolumes(Long vmId, List diskOfferings, Long totalSize, Long zoneId, Long ownerId) throws ResourceAllocationException; + Volume attachVolumeToVM(Long vmId, Long volumeId, Long deviceId, Boolean allowAttachForSharedFS); Volume detachVolumeViaDestroyVM(long vmId, long volumeId); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 89646e6c267b..243b543e9167 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -156,7 +156,9 @@ public class ApiConstants { public static final String DISK_IO_WRITE = "diskiowrite"; public static final String DISK_IO_PSTOTAL = "diskiopstotal"; public static final String DISK_SIZE = "disksize"; + public static final String DISK_SIZES = "disksizes"; public static final String DISK_SIZE_STRICTNESS = "disksizestrictness"; + public static final String DISK_OFFERING_IDS = "diskofferingids"; public static final String DISK_OFFERING_STRICTNESS = "diskofferingstrictness"; public static final String DOWNLOAD_DETAILS = "downloaddetails"; public static final String UTILIZATION = "utilization"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java index b76ff2c5deb9..1a937d9c430f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.api.command.user.vm; +import java.util.List; + import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; @@ -27,6 +29,7 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.UserCmd; import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.backup.BackupManager; @@ -54,10 +57,51 @@ public class CreateVMFromBackupCmd extends DeployVMCmd implements UserCmd { description = "backup ID to create the VM from") private Long backupId; + @Parameter(name = ApiConstants.DISK_OFFERING_IDS, + type = CommandType.LIST, + collectionType = CommandType.UUID, + entityType = DiskOfferingResponse.class, + description = "list of disk offering ids to be used by the data volumes of the vm.") + private List diskOfferingIds; + + @Parameter(name = ApiConstants.DISK_SIZES, + type = CommandType.LIST, + collectionType = CommandType.LONG, + description = "list of volume sizes to be used by the data volumes of the vm for custom disk offering.") + private List diskSizes; + + @Parameter(name = ApiConstants.MIN_IOPS, + type = CommandType.LIST, + collectionType = CommandType.LONG, + description = "list of minIops to be used by the data volumes of the vm for custom disk offering.") + private List minIops; + + @Parameter(name = ApiConstants.MAX_IOPS, + type = CommandType.LIST, + collectionType = CommandType.LONG, + description = "list of maxIops to be used by the data volumes of the vm for custom disk offering.") + private List maxIops; + public Long getBackupId() { return backupId; } + public List getDiskOfferingIds() { + return diskOfferingIds; + } + + public List getDiskSizes() { + return diskSizes; + } + + public List getMinIops() { + return minIops; + } + + public List getMaxIops() { + return maxIops; + } + @Override public void create() { UserVm vm; @@ -78,6 +122,7 @@ public void execute () { try { vm = _userVmService.restoreVMFromBackup(this); } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create vm due to exception: " + e.getMessage()); } if (vm != null) { UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", vm).get(0); diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 948279326e1a..6b5e5f582180 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.backup; import java.util.List; +import java.util.Map; import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; @@ -27,10 +28,12 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.utils.Pair; import com.cloud.utils.component.Manager; import com.cloud.utils.component.PluggableService; +import com.cloud.vm.VirtualMachine; /** * Backup and Recover Manager Interface @@ -155,4 +158,10 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer void validateBackupForZone(Long zoneId); BackupOffering updateBackupOffering(UpdateBackupOfferingCmd updateBackupOfferingCmd); + + boolean createDataVolumesForRestore(Long backupId, Long vmId, List diskOfferingIds, List diskSizes, List minIops, List maxIops) throws ResourceAllocationException; + + Map getBackupVmDetails(VirtualMachine vm); + + Map getBackupDiskOfferingDetails(Long vmId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index dfae71d73e90..db0a605280c0 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -239,10 +239,17 @@ public void setDetail(String name, String value) { this.details.put(name, value); } + public String getDetail(String name) { + return this.details.get(name); + } + public void setDetails(Map details) { this.details = details; } + public void addDetails(Map details) { + this.details.putAll(details); + } public Date getRemoved() { return removed; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java index 708588ee35ed..4483ccc7ffaf 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java @@ -152,6 +152,17 @@ public BackupVO persist(BackupVO backup) { }); } + @Override + public boolean update(Long id, BackupVO backup) { + return Transaction.execute((TransactionCallback) status -> { + boolean result = super.update(id, backup); + if (result == true) { + saveDetails(backup); + } + return result; + }); + } + @Override public List syncBackups(Long zoneId, Long vmId, List externalBackups) { for (Backup backup : externalBackups) { @@ -226,6 +237,11 @@ public BackupResponse newBackupResponse(Backup backup) { vmDetails.put(ApiConstants.HYPERVISOR, details.get(ApiConstants.HYPERVISOR)); vmDetails.put(ApiConstants.TEMPLATE_ID, details.get(ApiConstants.TEMPLATE_ID)); vmDetails.put(ApiConstants.SERVICE_OFFERING_ID, details.get(ApiConstants.SERVICE_OFFERING_ID)); + vmDetails.put(ApiConstants.NETWORK_IDS, details.get(ApiConstants.NETWORK_IDS)); + vmDetails.put(ApiConstants.DISK_OFFERING_IDS, details.get(ApiConstants.DISK_OFFERING_IDS)); + vmDetails.put(ApiConstants.DISK_SIZES, details.get(ApiConstants.DISK_SIZES)); + vmDetails.put(ApiConstants.MIN_IOPS, details.get(ApiConstants.MIN_IOPS)); + vmDetails.put(ApiConstants.MAX_IOPS, details.get(ApiConstants.MAX_IOPS)); response.setVmDetails(vmDetails); } diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index b4f3602202b8..bf8fc593fc17 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -24,16 +24,12 @@ import javax.inject.Inject; -import com.cloud.offering.ServiceOffering; import com.cloud.storage.dao.VolumeDao; -import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.backup.dao.BackupDao; -import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; -import com.cloud.utils.db.EntityManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; @@ -46,7 +42,7 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { @Inject private VolumeDao volumeDao; @Inject - private EntityManager entityManager; + private BackupManagerImpl backupManager; @Override public String getName() { @@ -134,14 +130,10 @@ public boolean takeBackup(VirtualMachine vm) { backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); - - HashMap details = new HashMap<>(); - details.put(ApiConstants.HYPERVISOR, vm.getHypervisorType().toString()); - ServiceOffering serviceOffering = entityManager.findById(ServiceOffering.class, vm.getServiceOfferingId()); - details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); - VirtualMachineTemplate template = entityManager.findById(VirtualMachineTemplate.class, vm.getTemplateId()); - details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + Map details = backupManager.getBackupVmDetails(vm); backup.setDetails(details); + Map diskOfferingDetails = backupManager.getBackupDiskOfferingDetails(vm.getId()); + backup.addDetails(diskOfferingDetails); return backupDao.persist(backup) != null; } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 517a17f80ac4..8de6da2b9527 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -25,22 +25,18 @@ import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; -import com.cloud.offering.ServiceOffering; import com.cloud.storage.ScopeType; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VolumeDao; -import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; -import com.cloud.utils.db.EntityManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.BackupRepositoryDao; @@ -99,7 +95,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co private AgentManager agentManager; @Inject - private EntityManager entityManager; + BackupManager backupManager; protected Host getLastVMHypervisorHost(VirtualMachine vm) { Long hostId = vm.getLastHostId(); @@ -187,6 +183,8 @@ public boolean takeBackup(final VirtualMachine vm) { backupVO.setSize(answer.getSize()); backupVO.setStatus(Backup.Status.BackedUp); backupVO.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); + Map details = backupManager.getBackupDiskOfferingDetails(vm.getId()); + backupVO.addDetails(details); return backupDao.update(backupVO.getId(), backupVO); } else { backupVO.setStatus(Backup.Status.Failed); @@ -213,13 +211,7 @@ private BackupVO createBackupObject(VirtualMachine vm, String backupPath) { backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); - - HashMap details = new HashMap<>(); - details.put(ApiConstants.HYPERVISOR, vm.getHypervisorType().toString()); - ServiceOffering serviceOffering = entityManager.findById(ServiceOffering.class, vm.getServiceOfferingId()); - details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); - VirtualMachineTemplate template = entityManager.findById(VirtualMachineTemplate.class, vm.getTemplateId()); - details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + Map details = backupManager.getBackupVmDetails(vm); backup.setDetails(details); return backupDao.persist(backup); diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index 934dd62f5620..de9b1f9a2c27 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -125,6 +125,9 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid @Inject private EntityManager entityManager; + @Inject + BackupManager backupManager; + private static String getUrlDomain(String url) throws URISyntaxException { URI uri; try { @@ -521,6 +524,8 @@ public boolean takeBackup(VirtualMachine vm) { BackupVO backup = getClient(vm.getDataCenterId()).registerBackupForVm(vm, backupJobStart, saveTime); if (backup != null) { backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); + Map details = backupManager.getBackupDiskOfferingDetails(vm.getId()); + backup.addDetails(details); backupDao.persist(backup); return true; } else { diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java index 1448817356a3..f8411afad469 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java @@ -17,18 +17,15 @@ package org.apache.cloudstack.backup.networker; -import com.cloud.offering.ServiceOffering; -import com.cloud.template.VirtualMachineTemplate; -import com.cloud.utils.db.EntityManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.nio.TrustAllManager; import com.cloud.vm.VirtualMachine; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.backup.BackupOffering; import org.apache.cloudstack.backup.BackupVO; import org.apache.cloudstack.backup.networker.api.NetworkerBackup; @@ -66,8 +63,8 @@ import java.util.ArrayList; import java.util.Base64; import java.util.Date; -import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.apache.cloudstack.backup.NetworkerBackupProvider.BACKUP_IDENTIFIER; @@ -79,7 +76,7 @@ public class NetworkerClient { private final HttpClient httpClient; @Inject - private EntityManager entityManager; + BackupManager backupManager; public NetworkerClient(final String url, final String username, final String password, final boolean validateCertificate, final int timeout) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { @@ -277,13 +274,7 @@ public BackupVO registerBackupForVm(VirtualMachine vm, Date backupJobStart, Stri backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); - - HashMap details = new HashMap<>(); - details.put(ApiConstants.HYPERVISOR, vm.getHypervisorType().toString()); - ServiceOffering serviceOffering = entityManager.findById(ServiceOffering.class, vm.getServiceOfferingId()); - details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); - VirtualMachineTemplate template = entityManager.findById(VirtualMachineTemplate.class, vm.getTemplateId()); - details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + Map details = backupManager.getBackupVmDetails(vm); backup.setDetails(details); return backup; diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index e5e34f2956fb..73477abd84f2 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -110,7 +110,7 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, @Inject private VirtualMachineManager virtualMachineManager; @Inject - private EntityManager entityManager; + private BackupManager backupManager; protected VeeamClient getClient(final Long zoneId) { try { @@ -380,14 +380,11 @@ public void doInTransactionWithoutResult(TransactionStatus status) { backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); - - HashMap details = new HashMap<>(); - details.put(ApiConstants.HYPERVISOR, vm.getHypervisorType().toString()); - ServiceOffering serviceOffering = entityManager.findById(ServiceOffering.class, vm.getServiceOfferingId()); - details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); - VirtualMachineTemplate template = entityManager.findById(VirtualMachineTemplate.class, vm.getTemplateId()); - details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); + Map details = backupManager.getBackupVmDetails(vm); backup.setDetails(details); + Map details = backupManager.getBackupDiskOfferingDetails(vm.getId()); + backup.addDetails(details); logger.debug(String.format("Creating a new entry in backups: [uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " + "domain_id: %s, zone_id: %s].", backup.getUuid(), backup.getVmId(), backup.getExternalId(), backup.getType(), backup.getDate(), diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 689d159905fa..fccb3c8be0ee 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -128,6 +128,7 @@ import com.cloud.api.ApiDBUtils; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; +import com.cloud.configuration.Resource; import com.cloud.configuration.Resource.ResourceType; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.DataCenter; @@ -153,6 +154,7 @@ import com.cloud.hypervisor.HypervisorCapabilitiesVO; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.offering.DiskOffering; +import com.cloud.offering.DiskOfferingInfo; import com.cloud.org.Grouping; import com.cloud.projects.Project; import com.cloud.projects.ProjectManager; @@ -933,7 +935,7 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept String userSpecifiedName = getVolumeNameFromCommand(cmd); - return commitVolume(cmd, caller, owner, displayVolume, zoneId, diskOfferingId, provisioningType, size, minIops, maxIops, parentVolume, userSpecifiedName, + return commitVolume(cmd.getSnapshotId(), caller, owner, displayVolume, zoneId, diskOfferingId, provisioningType, size, minIops, maxIops, parentVolume, userSpecifiedName, _uuidMgr.generateUuid(Volume.class, cmd.getCustomId()), details); } @@ -947,7 +949,7 @@ public void validateCustomDiskOfferingSizeRange(Long sizeInGB) { } } - private VolumeVO commitVolume(final CreateVolumeCmd cmd, final Account caller, final Account owner, final Boolean displayVolume, final Long zoneId, final Long diskOfferingId, + private VolumeVO commitVolume(final Long snapshotId, final Account caller, final Account owner, final Boolean displayVolume, final Long zoneId, final Long diskOfferingId, final Storage.ProvisioningType provisioningType, final Long size, final Long minIops, final Long maxIops, final VolumeVO parentVolume, final String userSpecifiedName, final String uuid, final Map details) { return Transaction.execute(new TransactionCallback() { @Override @@ -975,7 +977,7 @@ public VolumeVO doInTransaction(TransactionStatus status) { volume = _volsDao.persist(volume); - if (cmd.getSnapshotId() == null && displayVolume) { + if (snapshotId == null && displayVolume) { // for volume created from snapshot, create usage event after volume creation UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), diskOfferingId, null, size, Volume.class.getName(), volume.getUuid(), displayVolume); @@ -2541,6 +2543,36 @@ private Volume orchestrateAttachVolumeToVM(Long vmId, Long volumeId, Long device return newVol; } + @Override + public boolean createAndAttachVolumes(Long vmId, List diskOfferings, Long totalSize, Long zoneId, Long ownerId) throws ResourceAllocationException { + Account caller = CallContext.current().getCallingAccount(); + Account owner = _accountMgr.getAccount(ownerId); + List volumesId = new ArrayList<>(); + boolean error = false; + + _resourceLimitMgr.checkResourceLimit(owner, Resource.ResourceType.volume, diskOfferings.size()); + _resourceLimitMgr.checkResourceLimit(owner, Resource.ResourceType.primary_storage, totalSize); + + for (DiskOfferingInfo diskOfferingInfo : diskOfferings) { + DiskOffering diskOffering = diskOfferingInfo.getDiskOffering(); + VolumeVO volume = commitVolume(null, caller, owner, true, zoneId, diskOffering.getId(), + diskOffering.getProvisioningType(), diskOfferingInfo.getSize(), diskOfferingInfo.getMinIops(), + diskOfferingInfo.getMaxIops(), null, getRandomVolumeName(), _uuidMgr.generateUuid(Volume.class, null), new HashMap<>()); + if (volume == null) { + error = true; + break; + } + volumesId.add(volume.getId()); + attachVolumeToVM(vmId, volume.getId(), null, false); + } + if (error) { + for (Long volumeId : volumesId) { + destroyVolume(volumeId, caller, true, true); + } + } + return !error; + } + public Volume attachVolumeToVM(Long vmId, Long volumeId, Long deviceId, Boolean allowAttachForSharedFS) { Account caller = CallContext.current().getCallingAccount(); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 3c1b1283e5c9..2c6f29b9c0ef 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -8866,6 +8866,13 @@ public UserVm restoreVMFromBackup(CreateVMFromBackupCmd cmd) throws ResourceUnav } catch (ResourceUnavailableException e) { throw new CloudRuntimeException("Unable to start the instance " + e); } + + try { + backupManager.createDataVolumesForRestore(cmd.getBackupId(), vmId, cmd.getDiskOfferingIds(), cmd.getDiskSizes(), cmd.getMinIops(), cmd.getMaxIops()); + } catch (Exception e) { + throw new CloudRuntimeException("Unable to create data volumes " + e); + } + boolean status = false; try { VirtualMachineEntity vmEntity = _orchSrvc.getVirtualMachine(vm.getUuid()); diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 01f35d7ff1f0..1ee7c051bee8 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -21,16 +21,28 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TimeZone; import java.util.Timer; import java.util.TimerTask; import java.util.stream.Collectors; +import java.util.stream.Stream; import com.amazonaws.util.CollectionUtils; +import com.cloud.api.query.dao.UserVmJoinDao; +import com.cloud.api.query.vo.UserVmJoinVO; +import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.offering.DiskOffering; +import com.cloud.offering.DiskOfferingInfo; +import com.cloud.offering.ServiceOffering; +import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.VolumeApiService; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.db.EntityManager; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.VirtualMachineManager; import javax.inject.Inject; @@ -166,6 +178,10 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { private VolumeApiService volumeApiService; @Inject private VolumeOrchestrationService volumeOrchestrationService; + @Inject + private EntityManager entityManager; + @Inject + private UserVmJoinDao userVmJoinDao; private AsyncJobDispatcher asyncJobDispatcher; private Timer backupTimer; @@ -279,6 +295,50 @@ public boolean deleteBackupOffering(final Long offeringId) { return backupOfferingDao.remove(offering.getId()); } + @Override + public Map getBackupVmDetails(VirtualMachine vm) { + HashMap details = new HashMap<>(); + details.put(ApiConstants.HYPERVISOR, vm.getHypervisorType().toString()); + ServiceOffering serviceOffering = entityManager.findById(ServiceOffering.class, vm.getServiceOfferingId()); + details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); + VirtualMachineTemplate template = entityManager.findById(VirtualMachineTemplate.class, vm.getTemplateId()); + details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + List userVmJoinVOs = userVmJoinDao.searchByIds(vm.getId()); + if (userVmJoinVOs != null && !userVmJoinVOs.isEmpty()) { + Set networkIds = new HashSet<>(); + for (UserVmJoinVO userVmJoinVO : userVmJoinVOs) { + networkIds.add(userVmJoinVO.getNetworkUuid()); + } + details.put(ApiConstants.NETWORK_IDS, String.join(",", networkIds)); + } + return details; + } + + @Override + public Map getBackupDiskOfferingDetails(Long vmId) { + List volumes = volumeDao.findByInstance(vmId); + List diskOfferingIds = new ArrayList<>(); + List diskSizes = new ArrayList<>(); + List minIops = new ArrayList<>(); + List maxIops = new ArrayList<>(); + Map details = new HashMap<>(); + for (Volume vol : volumes) { + if (vol.getVolumeType() != Volume.Type.DATADISK) { + continue; + } + DiskOffering diskOffering = diskOfferingDao.findById(vol.getDiskOfferingId()); + diskOfferingIds.add(diskOffering.getUuid()); + diskSizes.add(vol.getSize()); + minIops.add(vol.getMinIops()); + maxIops.add(vol.getMaxIops()); + } + details.put(ApiConstants.DISK_OFFERING_IDS, String.join(",", diskOfferingIds)); + details.put(ApiConstants.DISK_SIZES, String.join(",", diskSizes.stream().map(String::valueOf).collect(Collectors.toList()))); + details.put(ApiConstants.MIN_IOPS, String.join(",", minIops.stream().map(String::valueOf).collect(Collectors.toList()))); + details.put(ApiConstants.MAX_IOPS, String.join(",", maxIops.stream().map(String::valueOf).collect(Collectors.toList()))); + return details; + } + public static String createVolumeInfoFromVolumes(List vmVolumes) { List list = new ArrayList<>(); for (VolumeVO vol : vmVolumes) { @@ -751,6 +811,50 @@ private Backup.VolumeInfo getVolumeInfo(List backedUpVolumes, return null; } + @Override + public boolean createDataVolumesForRestore(Long backupId, Long vmId, List diskOfferingIds, List diskSizes, List minIopsList, List maxIopsList) throws ResourceAllocationException { + final BackupVO backup = backupDao.findById(backupId); + if (backup == null) { + throw new CloudRuntimeException("Backup " + backupId + " does not exist"); + } + + backupDao.loadDetails(backup); + List diskOfferings; + if (diskOfferingIds != null) { + diskOfferings = diskOfferingIds.stream().map(id -> diskOfferingDao.findById(id)).collect(Collectors.toList()); + } else { + diskOfferings = Stream.of(backup.getDetail(ApiConstants.DISK_OFFERING_IDS).split(",")) + .map(uuid -> diskOfferingDao.findByUuid(uuid)) + .collect(Collectors.toList()); + diskSizes = Stream.of(backup.getDetail(ApiConstants.DISK_SIZES).split(",")) + .map(Long::valueOf) + .collect(Collectors.toList()); + minIopsList = Stream.of(backup.getDetail(ApiConstants.MIN_IOPS).split(",")) + .map(s -> "null".equals(s) ? null : Long.valueOf(s)) + .collect(Collectors.toList()); + maxIopsList = Stream.of(backup.getDetail(ApiConstants.MAX_IOPS).split(",")) + .map(s -> "null".equals(s) ? null : Long.valueOf(s)) + .collect(Collectors.toList()); + } + + List diskOfferingInfos = new ArrayList<>(); + Long totalSize = 0L; + int index = 0; + for (DiskOfferingVO diskOffering : diskOfferings) { + if (diskOffering == null || diskOffering.getRemoved() != null || diskOffering.isComputeOnly()) { + throw new InvalidParameterValueException("Please specify a valid disk offering."); + } + Long size = diskOffering.isCustomized() ? diskSizes.get(index) : diskOffering.getDiskSize(); + Long minIops = (diskOffering.isCustomizedIops() != null && diskOffering.isCustomizedIops()) ? + minIopsList.get(index) : diskOffering.getMinIops(); + Long maxIops = (diskOffering.isCustomizedIops() != null && diskOffering.isCustomizedIops()) ? + maxIopsList.get(index) : diskOffering.getMaxIops(); + diskOfferingInfos.add(new DiskOfferingInfo(diskOffering, size, minIops, maxIops)); + totalSize += size; + } + return volumeApiService.createAndAttachVolumes(vmId, diskOfferingInfos, totalSize, backup.getZoneId(), backup.getAccountId()); + } + @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_RESTORE_TO_NEW_VM, eventDescription = "Resoring backup to new vm", async = true) public boolean restoreBackupToVM(final Long backupId, final Long vmId) throws ResourceUnavailableException { From 753d80faf962beb4f9090dd4c2d218de3c67f148 Mon Sep 17 00:00:00 2001 From: abh1sar Date: Sun, 5 Jan 2025 16:09:00 +0530 Subject: [PATCH 007/147] Restore backup to VM plugin change for VeeamBackupProvider --- .../apache/cloudstack/backup/BackupProvider.java | 2 +- .../cloudstack/backup/DummyBackupProvider.java | 2 +- .../cloudstack/backup/NASBackupProvider.java | 2 +- .../cloudstack/backup/NetworkerBackupProvider.java | 6 ++++-- .../backup/networker/NetworkerClient.java | 8 -------- .../cloudstack/backup/VeeamBackupProvider.java | 14 ++++++++++---- .../cloudstack/backup/veeam/VeeamClient.java | 6 ++++-- .../cloudstack/backup/BackupManagerImpl.java | 9 ++++++++- 8 files changed, 29 insertions(+), 20 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index d3aeadca6af7..774be8426436 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -85,7 +85,7 @@ public interface BackupProvider { */ boolean deleteBackup(Backup backup, boolean forced); - boolean restoreBackupToVM(VirtualMachine vm, Backup backup); + boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid); /** * Restore VM from backup diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index bf8fc593fc17..7936846c49b5 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -148,7 +148,7 @@ public void syncBackups(VirtualMachine vm, Backup.Metric metric) { } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup) { + public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { return true; } } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 8de6da2b9527..0e52dbcbfae8 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -218,7 +218,7 @@ private BackupVO createBackupObject(VirtualMachine vm, String backupPath) { } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup) { + public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { return restoreVMBackup(vm, backup); } diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index de9b1f9a2c27..0558f16151e6 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -524,7 +524,9 @@ public boolean takeBackup(VirtualMachine vm) { BackupVO backup = getClient(vm.getDataCenterId()).registerBackupForVm(vm, backupJobStart, saveTime); if (backup != null) { backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); - Map details = backupManager.getBackupDiskOfferingDetails(vm.getId()); + Map details = backupManager.getBackupVmDetails(vm); + backup.setDetails(details); + details = backupManager.getBackupDiskOfferingDetails(vm.getId()); backup.addDetails(details); backupDao.persist(backup); return true; @@ -671,7 +673,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { public boolean willDeleteBackupsOnOfferingRemoval() { return false; } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup) { + public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { return true; } } diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java index f8411afad469..47243c41655d 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java @@ -25,7 +25,6 @@ import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.backup.BackupOffering; import org.apache.cloudstack.backup.BackupVO; import org.apache.cloudstack.backup.networker.api.NetworkerBackup; @@ -47,7 +46,6 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; -import javax.inject.Inject; import javax.net.ssl.SSLContext; import javax.net.ssl.X509TrustManager; import java.io.IOException; @@ -64,7 +62,6 @@ import java.util.Base64; import java.util.Date; import java.util.List; -import java.util.Map; import static org.apache.cloudstack.backup.NetworkerBackupProvider.BACKUP_IDENTIFIER; @@ -75,9 +72,6 @@ public class NetworkerClient { private final String apiPassword; private final HttpClient httpClient; - @Inject - BackupManager backupManager; - public NetworkerClient(final String url, final String username, final String password, final boolean validateCertificate, final int timeout) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { apiName = username; @@ -274,8 +268,6 @@ public BackupVO registerBackupForVm(VirtualMachine vm, Date backupJobStart, Stri backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); - Map details = backupManager.getBackupVmDetails(vm); - backup.setDetails(details); return backup; } catch (final IOException e) { diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index 73477abd84f2..5a62227b4b9c 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -50,6 +50,7 @@ import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; +import com.cloud.storage.dao.VolumeDao; import com.cloud.user.User; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; @@ -111,6 +112,8 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, private VirtualMachineManager virtualMachineManager; @Inject private BackupManager backupManager; + @Inject + private VolumeDao volumeDao; protected VeeamClient getClient(final Long zoneId) { try { @@ -296,7 +299,7 @@ private void prepareForBackupRestoration(VirtualMachine vm) { public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid, Pair vmNameAndState) { final Long zoneId = backup.getZoneId(); final String restorePointId = backup.getExternalId(); - return getClient(zoneId).restoreVMToDifferentLocation(restorePointId, hostIp, dataStoreUuid); + return getClient(zoneId).restoreVMToDifferentLocation(restorePointId, null, hostIp, dataStoreUuid); } @Override @@ -383,7 +386,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); Map details = backupManager.getBackupVmDetails(vm); backup.setDetails(details); - Map details = backupManager.getBackupDiskOfferingDetails(vm.getId()); + details = backupManager.getBackupDiskOfferingDetails(vm.getId()); backup.addDetails(details); logger.debug(String.format("Creating a new entry in backups: [uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " @@ -405,8 +408,11 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup) { - return true; + public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + final Long zoneId = backup.getZoneId(); + final String restorePointId = backup.getExternalId(); + final String restoreLocation = vm.getInstanceName(); + return getClient(zoneId).restoreVMToDifferentLocation(restorePointId, restoreLocation, hostIp, dataStoreUuid).first(); } @Override diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index d911736090cb..2b41199b1a16 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -925,8 +925,10 @@ private Date formatDate(String date) throws ParseException { return dateFormat.parse(StringUtils.substring(date, 0, 19)); } - public Pair restoreVMToDifferentLocation(String restorePointId, String hostIp, String dataStoreUuid) { - final String restoreLocation = RESTORE_VM_SUFFIX + UUID.randomUUID().toString(); + public Pair restoreVMToDifferentLocation(String restorePointId, String restoreLocation, String hostIp, String dataStoreUuid) { + if (restoreLocation == null) { + restoreLocation = RESTORE_VM_SUFFIX + UUID.randomUUID().toString(); + } final String datastoreId = dataStoreUuid.replace("-",""); final List cmds = Arrays.asList( "$points = Get-VBRRestorePoint", diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 1ee7c051bee8..87dcebea1c03 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -905,8 +905,15 @@ public boolean restoreBackupToVM(final Long backupId, final Long vmId) throws Re vm.getId(), ApiCommandResourceType.VirtualMachine.toString(), true, 0); + String host = null; + String dataStore = null; + if (!"nas".equals(offering.getProvider())) { + Pair restoreInfo = getRestoreVolumeHostAndDatastore(vm); + host = restoreInfo.first().getPrivateIpAddress(); + dataStore = restoreInfo.second().getUuid(); + } final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); - if (!backupProvider.restoreBackupToVM(vm, backup)) { + if (!backupProvider.restoreBackupToVM(vm, backup, host, dataStore)) { ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_ERROR, EventTypes.EVENT_VM_BACKUP_RESTORE, String.format("Failed to restore VM %s from backup %s", vm.getInstanceName(), backup.getUuid()), vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); From 72c943f365d6281c918265396ae7ac2ff7764d20 Mon Sep 17 00:00:00 2001 From: abh1sar Date: Tue, 7 Jan 2025 02:50:23 +0530 Subject: [PATCH 008/147] UI form updates --- ui/src/views/compute/DeployVMFromBackup.vue | 155 ++++------- .../compute/wizard/VolumeDiskOfferingMap.vue | 253 ++++++++++++++++++ ui/src/views/storage/CreateVMFromBackup.vue | 48 +++- 3 files changed, 353 insertions(+), 103 deletions(-) create mode 100644 ui/src/views/compute/wizard/VolumeDiskOfferingMap.vue diff --git a/ui/src/views/compute/DeployVMFromBackup.vue b/ui/src/views/compute/DeployVMFromBackup.vue index e45ab7fc72b4..70aa922b19af 100644 --- a/ui/src/views/compute/DeployVMFromBackup.vue +++ b/ui/src/views/compute/DeployVMFromBackup.vue @@ -44,7 +44,7 @@ @@ -326,7 +261,7 @@ v-if="!template.deployasis && template.childtemplates && template.childtemplates.length > 0" >
-
+
{{ $t(ts[ctype]) }} @@ -377,6 +377,7 @@ export default { MEMORY: 'label.memory', PRIVATE_IP: 'label.management.ips', SECONDARY_STORAGE: 'label.secondary.storage', + OBJECT_STORAGE: 'label.object.storage', STORAGE: 'label.primary.storage.used', STORAGE_ALLOCATED: 'label.primary.storage.allocated', VIRTUAL_NETWORK_PUBLIC_IP: 'label.public.ips', @@ -438,6 +439,7 @@ export default { case 'STORAGE': case 'STORAGE_ALLOCATED': case 'SECONDARY_STORAGE': + case 'OBJECT_STORAGE': case 'LOCAL_STORAGE': value = parseFloat(value / (1024 * 1024 * 1024.0), 10).toFixed(2) if (value >= 1024.0) { diff --git a/ui/src/views/infra/AddObjectStorage.vue b/ui/src/views/infra/AddObjectStorage.vue index 94c4ef0adb98..69f15e4c11dd 100644 --- a/ui/src/views/infra/AddObjectStorage.vue +++ b/ui/src/views/infra/AddObjectStorage.vue @@ -25,10 +25,16 @@ layout="vertical" @finish="handleSubmit" > - + + - + + {{ prov }} - + + @@ -53,6 +62,12 @@ + + + +
{{ $t('label.cancel') }} {{ $t('label.ok') }} @@ -66,6 +81,7 @@ import { ref, reactive, toRaw } from 'vue' import { api } from '@/api' import { mixinForm } from '@/utils/mixin' import ResourceIcon from '@/components/view/ResourceIcon' +import TooltipLabel from '@/components/widgets/TooltipLabel' export default { name: 'AddObjectStorage', @@ -77,7 +93,8 @@ export default { } }, components: { - ResourceIcon + ResourceIcon, + TooltipLabel }, inject: ['parentFetchData'], data () { @@ -87,6 +104,9 @@ export default { loading: false } }, + beforeCreate () { + this.apiParams = this.$getApiParams('addObjectStoragePool') + }, created () { this.initForm() this.fetchData() From beb05aeefe6e7cfbe3175482698fadbe34391f7e Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 4 Mar 2025 23:57:24 +0530 Subject: [PATCH 043/147] Backup storage capacity and alert framework layer. --- .../cloudstack/backup/BackupManager.java | 3 ++ .../cloudstack/backup/BackupProvider.java | 6 +++ .../backup/DummyBackupProvider.java | 41 ++++++++++++++----- .../cloudstack/backup/NASBackupProvider.java | 5 +++ .../backup/NetworkerBackupProvider.java | 5 +++ .../backup/VeeamBackupProvider.java | 5 +++ .../com/cloud/alert/AlertManagerImpl.java | 37 ++++++++++------- .../cloud/server/ManagementServerImpl.java | 24 ++++++++--- .../cloudstack/backup/BackupManagerImpl.java | 8 ++++ ui/src/views/dashboard/CapacityDashboard.vue | 4 +- 10 files changed, 106 insertions(+), 32 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index ace4ab99e056..cf6ae636c6d0 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; +import com.cloud.capacity.Capacity; import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; @@ -251,4 +252,6 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer Map getVmDetailsForBackup(VirtualMachine vm); Map getDiskOfferingDetailsForBackup(Long vmId); + + Capacity getBackupStorageUsedStats(Long zoneId); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index dcf492d67b7b..2456708dc922 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -123,4 +123,10 @@ public interface BackupProvider { */ boolean supportsInstanceFromBackup(); + /** + * Returns the backup storage usage (Used, Total) for a backup provider + * @param zoneId the zone for which to return metrics + * @return a pair of Used size and Total size for the backup storage + */ + Pair getBackupStorageStats(Long zoneId); } diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index aa49aa44de8b..76402edbd4fc 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -25,10 +25,13 @@ import javax.inject.Inject; -import com.cloud.configuration.Resource; +import com.cloud.storage.Volume; import com.cloud.storage.dao.VolumeDao; import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; @@ -37,7 +40,7 @@ import com.cloud.vm.VirtualMachine; public class DummyBackupProvider extends AdapterBase implements BackupProvider { - + private static final Logger LOG = LogManager.getLogger(DummyBackupProvider.class); @Inject private BackupDao backupDao; @@ -92,14 +95,21 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU @Override public Map getBackupMetrics(Long zoneId, List vms) { final Map metrics = new HashMap<>(); - final Backup.Metric metric = new Backup.Metric(1000L, 100L); - if (vms == null || vms.isEmpty()) { + if (CollectionUtils.isEmpty(vms)) { + LOG.warn("Unable to get VM Backup Metrics because the list of VMs is empty."); return metrics; } - for (VirtualMachine vm : vms) { - if (vm != null) { - metrics.put(vm, metric); + + for (final VirtualMachine vm : vms) { + Long vmBackupSize = 0L; + Long vmBackupProtectedSize = 0L; + for (final Backup backup: backupDao.listByVmId(null, vm.getId())) { + vmBackupSize += backup.getSize(); + vmBackupProtectedSize += backup.getProtectedSize(); } + Backup.Metric vmBackupMetric = new Backup.Metric(vmBackupSize,vmBackupProtectedSize); + LOG.debug("Metrics for VM {} is [backup size: {}, data size: {}].", vm, vmBackupMetric.getBackupSize(), vmBackupMetric.getDataSize()); + metrics.put(vm, vmBackupMetric); } return metrics; } @@ -134,8 +144,14 @@ public Pair takeBackup(VirtualMachine vm) { backup.setExternalId("dummy-external-id"); backup.setType("FULL"); backup.setDate(new Date()); - backup.setSize(1024000L); - backup.setProtectedSize(Resource.ResourceType.bytesToGiB); + long virtualSize = 0L; + for (final Volume volume: volumeDao.findByInstance(vm.getId())) { + if (Volume.State.Ready.equals(volume.getState())) { + virtualSize += volume.getSize(); + } + } + backup.setSize(virtualSize); + backup.setProtectedSize(virtualSize); backup.setStatus(Backup.Status.BackedUp); backup.setBackupOfferingId(vm.getBackupOfferingId()); backup.setAccountId(vm.getAccountId()); @@ -154,7 +170,7 @@ public Pair takeBackup(VirtualMachine vm) { @Override public boolean deleteBackup(Backup backup, boolean forced) { - return true; + return backupDao.remove(backup.getId()); } @Override @@ -162,6 +178,11 @@ public boolean supportsInstanceFromBackup() { return true; } + @Override + public Pair getBackupStorageStats(Long zoneId) { + return new Pair<>(8L * 1024 * 1024 * 1024, 10L * 1024 * 1024 * 1024); + } + @Override public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { return true; diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 1e3352b7913e..2913161afe82 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -432,6 +432,11 @@ public boolean supportsInstanceFromBackup() { return true; } + @Override + public Pair getBackupStorageStats(Long zoneId) { + return new Pair<>(0L, 0L); + } + @Override public List listBackupOfferings(Long zoneId) { final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index 194d84ab625e..5e2ac3b7f5e1 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -640,6 +640,11 @@ public boolean supportsInstanceFromBackup() { return false; } + @Override + public Pair getBackupStorageStats(Long zoneId) { + return new Pair<>(0L, 0L); + } + @Override public boolean willDeleteBackupsOnOfferingRemoval() { return false; } diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index ccbb00e5b8fe..a5a8a8c1edfd 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -366,6 +366,11 @@ public boolean supportsInstanceFromBackup() { return true; } + @Override + public Pair getBackupStorageStats(Long zoneId) { + return new Pair<>(0L, 0L); + } + @Override public String getConfigComponentName() { return BackupService.class.getSimpleName(); diff --git a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java index 7f856be398c2..70c77142b848 100644 --- a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java +++ b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java @@ -39,6 +39,8 @@ import com.cloud.dc.DataCenter; import com.cloud.dc.Pod; import com.cloud.org.Cluster; + +import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; @@ -125,6 +127,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi @Inject private ConfigurationManager _configMgr; @Inject + protected BackupManager backupManager; + @Inject protected ConfigDepot _configDepot; @Inject private ObjectStoreDao _objectStoreDao; @@ -543,7 +547,8 @@ public void checkForAlerts() { capacity = _capacityDao.findCapacityBy(capacityType.intValue(), dc.getId(), null, null); if (capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE || - capacityType == Capacity.CAPACITY_TYPE_OBJECT_STORAGE) { + capacityType == Capacity.CAPACITY_TYPE_OBJECT_STORAGE || + capacityType == Capacity.CAPACITY_TYPE_BACKUP_STORAGE) { capacity.add(getUsedStats(capacityType, dc.getId(), null, null)); } if (capacity == null || capacity.size() == 0) { @@ -621,6 +626,8 @@ private SummedCapacity getUsedStats(short capacityType, long zoneId, Long podId, capacity = _storageMgr.getStoragePoolUsedStats(null, clusterId, podId, zoneId); } else if (capacityType == Capacity.CAPACITY_TYPE_OBJECT_STORAGE) { capacity = _storageMgr.getObjectStorageUsedStats(zoneId); + } else if (capacityType == Capacity.CAPACITY_TYPE_BACKUP_STORAGE) { + capacity = (CapacityVO) backupManager.getBackupStorageUsedStats(zoneId); } if (capacity != null) { return new SummedCapacity(capacity.getUsedCapacity(), 0, capacity.getTotalCapacity(), capacityType, clusterId, podId); @@ -726,20 +733,20 @@ private void generateEmailAlert(DataCenterVO dc, HostPodVO pod, ClusterVO cluste msgContent = "Number of unallocated virtual network guest IPv6 subnets is low, total: " + totalStr + ", allocated: " + usedStr + " (" + pctStr + "%)"; alertType = AlertManager.AlertType.ALERT_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET; break; - case Capacity.CAPACITY_TYPE_BACKUP_STORAGE: - msgSubject = "System Alert: Low Available Backup Storage in availability zone " + dc.getName(); - totalStr = Double.toString(totalCapacity); - usedStr = Double.toString(usedCapacity); - msgContent = "Available backup storage space is low, total: " + totalStr + " MB, used: " + usedStr + " MB (" + pctStr + "%)"; - alertType = AlertManager.AlertType.ALERT_TYPE_BACKUP_STORAGE; - break; - case Capacity.CAPACITY_TYPE_OBJECT_STORAGE: - msgSubject = "System Alert: Low Available Object Storage in availability zone " + dc.getName(); - totalStr = Double.toString(totalCapacity); - usedStr = Double.toString(usedCapacity); - msgContent = "Available object storage space is low, total: " + totalStr + " MB, used: " + usedStr + " MB (" + pctStr + "%)"; - alertType = AlertManager.AlertType.ALERT_TYPE_OBJECT_STORAGE; - break; + case Capacity.CAPACITY_TYPE_BACKUP_STORAGE: + msgSubject = "System Alert: Low Available Backup Storage in availability zone " + dc.getName(); + totalStr = Double.toString(totalCapacity); + usedStr = Double.toString(usedCapacity); + msgContent = "Available backup storage space is low, total: " + totalStr + " MB, used: " + usedStr + " MB (" + pctStr + "%)"; + alertType = AlertManager.AlertType.ALERT_TYPE_BACKUP_STORAGE; + break; + case Capacity.CAPACITY_TYPE_OBJECT_STORAGE: + msgSubject = "System Alert: Low Available Object Storage in availability zone " + dc.getName(); + totalStr = Double.toString(totalCapacity); + usedStr = Double.toString(usedCapacity); + msgContent = "Available object storage space is low, total: " + totalStr + " MB, used: " + usedStr + " MB (" + pctStr + "%)"; + alertType = AlertManager.AlertType.ALERT_TYPE_OBJECT_STORAGE; + break; } try { diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index ea3e19161563..b60b0eb08975 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -606,6 +606,7 @@ import org.apache.cloudstack.api.command.user.zone.ListZonesCmd; import org.apache.cloudstack.auth.UserAuthenticator; import org.apache.cloudstack.auth.UserTwoFactorAuthenticator; +import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.config.ConfigurationGroup; @@ -1012,6 +1013,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe UserDataManager userDataManager; @Inject StoragePoolTagsDao storagePoolTagsDao; + @Inject + BackupManager backupManager; @Inject private PublicIpQuarantineDao publicIpQuarantineDao; @@ -3390,6 +3393,9 @@ List getStorageCapacities(Long clusterId, Long podId, Long zoneI if (capacityTypes.contains(Capacity.CAPACITY_TYPE_OBJECT_STORAGE)) { capacities.add(_storageMgr.getObjectStorageUsedStats(dc.getId())); } + if (capacityTypes.contains(Capacity.CAPACITY_TYPE_OBJECT_STORAGE)) { + capacities.add((CapacityVO) backupManager.getBackupStorageUsedStats(dc.getId())); + } for (CapacityVO capacity : capacities) { if (capacity.getTotalCapacity() != 0) { capacity.setUsedPercentage((float)capacity.getUsedCapacity() / capacity.getTotalCapacity()); @@ -3433,12 +3439,18 @@ protected List listCapacitiesWithDetails(final Long zoneId, final Lo // op_host_Capacity contains only allocated stats and the real time // stats are stored "in memory". // List secondary and object storage capacities only when the api is invoked for the zone layer. - if ((capacityType == null || - (capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE || capacityType == Capacity.CAPACITY_TYPE_OBJECT_STORAGE)) && - podId == null && clusterId == null && - StringUtils.isEmpty(t)) { - taggedCapacities.add(_storageMgr.getSecondaryStorageUsedStats(null, zId)); - taggedCapacities.add(_storageMgr.getObjectStorageUsedStats(zId)); + if (podId == null && clusterId == null && StringUtils.isEmpty(t)) { + if (capacityType == null) { + taggedCapacities.add(_storageMgr.getSecondaryStorageUsedStats(null, zId)); + taggedCapacities.add(_storageMgr.getObjectStorageUsedStats(zId)); + taggedCapacities.add((CapacityVO) backupManager.getBackupStorageUsedStats(zId)); + } else if (capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE) { + taggedCapacities.add(_storageMgr.getSecondaryStorageUsedStats(null, zId)); + } else if (capacityType == Capacity.CAPACITY_TYPE_OBJECT_STORAGE) { + taggedCapacities.add(_storageMgr.getObjectStorageUsedStats(zId)); + } else if (capacityType == Capacity.CAPACITY_TYPE_BACKUP_STORAGE) { + taggedCapacities.add((CapacityVO) backupManager.getBackupStorageUsedStats(zId)); + } } if ((capacityType == null || capacityType == Capacity.CAPACITY_TYPE_STORAGE) && storagePoolIdsForCapacity.first()) { taggedCapacities.add(_storageMgr.getStoragePoolUsedStats(zId, podId, clusterId, storagePoolIdsForCapacity.second())); diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 38dda5e10fb7..7e54ff163af1 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -32,6 +32,8 @@ import com.amazonaws.util.CollectionUtils; import com.cloud.alert.AlertManager; +import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityVO; import com.cloud.configuration.Resource; import com.cloud.exception.ResourceAllocationException; import com.cloud.storage.Snapshot; @@ -1776,4 +1778,10 @@ public BackupOffering updateBackupOffering(UpdateBackupOfferingCmd updateBackupO return response; } + @Override + public CapacityVO getBackupStorageUsedStats(Long zoneId) { + final BackupProvider backupProvider = getBackupProvider(zoneId); + Pair backupUsage = backupProvider.getBackupStorageStats(zoneId); + return new CapacityVO(null, zoneId, null, null, backupUsage.first(), backupUsage.second(), Capacity.CAPACITY_TYPE_BACKUP_STORAGE); + } } diff --git a/ui/src/views/dashboard/CapacityDashboard.vue b/ui/src/views/dashboard/CapacityDashboard.vue index 2e5a8dd2d69b..a1a28a2d1d91 100644 --- a/ui/src/views/dashboard/CapacityDashboard.vue +++ b/ui/src/views/dashboard/CapacityDashboard.vue @@ -217,7 +217,7 @@
-
+
{{ $t(ts[ctype]) }} @@ -377,6 +377,7 @@ export default { MEMORY: 'label.memory', PRIVATE_IP: 'label.management.ips', SECONDARY_STORAGE: 'label.secondary.storage', + BACKUP_STORAGE: 'label.backup.storage', OBJECT_STORAGE: 'label.object.storage', STORAGE: 'label.primary.storage.used', STORAGE_ALLOCATED: 'label.primary.storage.allocated', @@ -439,6 +440,7 @@ export default { case 'STORAGE': case 'STORAGE_ALLOCATED': case 'SECONDARY_STORAGE': + case 'BACKUP_STORAGE': case 'OBJECT_STORAGE': case 'LOCAL_STORAGE': value = parseFloat(value / (1024 * 1024 * 1024.0), 10).toFixed(2) From c72a10e7f23c1bad399b20ba8da7a590dbd83e03 Mon Sep 17 00:00:00 2001 From: abh1sar Date: Thu, 6 Mar 2025 12:51:45 +0530 Subject: [PATCH 044/147] Nas provider support for backup storage stats --- .../cloudstack/backup/BackupProvider.java | 7 ++ .../cloudstack/backup/BackupRepository.java | 2 + .../backup/BackupStorageStatsAnswer.java | 50 +++++++++++++ .../backup/GetBackupStorageStatsCommand.java | 66 +++++++++++++++++ .../cloudstack/backup/BackupRepositoryVO.java | 10 +++ .../backup/dao/BackupRepositoryDao.java | 2 + .../backup/dao/BackupRepositoryDaoImpl.java | 8 +++ .../backup/DummyBackupProvider.java | 4 ++ .../cloudstack/backup/NASBackupProvider.java | 56 ++++++++++++--- .../backup/NetworkerBackupProvider.java | 4 ++ .../backup/VeeamBackupProvider.java | 4 ++ .../LibvirtGetBackupStatsCommandWrapper.java | 71 +++++++++++++++++++ scripts/vm/hypervisor/kvm/nasbackup.sh | 11 +++ .../cloud/server/ManagementServerImpl.java | 2 +- .../cloudstack/backup/BackupManagerImpl.java | 2 + 15 files changed, 290 insertions(+), 9 deletions(-) create mode 100644 core/src/main/java/org/apache/cloudstack/backup/BackupStorageStatsAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/GetBackupStorageStatsCommand.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetBackupStatsCommandWrapper.java diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index 2456708dc922..7cb179153b6a 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -129,4 +129,11 @@ public interface BackupProvider { * @return a pair of Used size and Total size for the backup storage */ Pair getBackupStorageStats(Long zoneId); + + /** + * Gets the backup storage usage (Used, Total) from the plugin and stores it in db + * @param zoneId the zone for which to return metrics + */ + void syncBackupStorageStats(Long zoneId); + } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java index 8e5c9740e690..be539a0eb044 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java @@ -28,7 +28,9 @@ public interface BackupRepository extends InternalIdentity, Identity { String getType(); String getAddress(); String getMountOptions(); + void setUsedBytes(Long usedBytes); Long getCapacityBytes(); Long getUsedBytes(); + void setCapacityBytes(Long capacityBytes); Date getCreated(); } diff --git a/core/src/main/java/org/apache/cloudstack/backup/BackupStorageStatsAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/BackupStorageStatsAnswer.java new file mode 100644 index 000000000000..eabf6877ba6e --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/BackupStorageStatsAnswer.java @@ -0,0 +1,50 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; + +public class BackupStorageStatsAnswer extends Answer { + private Long totalSize; + private Long usedSize; + + public BackupStorageStatsAnswer(final Command command, final boolean success, final String details) { + super(command, success, details); + this.totalSize = 0L; + this.usedSize = 0L; + } + + public Long getTotalSize() { + return totalSize; + } + + public void setTotalSize(Long totalSize) { + this.totalSize = totalSize; + } + + public Long getUsedSize() { + return usedSize; + } + + public void setUsedSize(Long usedSize) { + this.usedSize = usedSize; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/GetBackupStorageStatsCommand.java b/core/src/main/java/org/apache/cloudstack/backup/GetBackupStorageStatsCommand.java new file mode 100644 index 000000000000..1ceeac17e52e --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/GetBackupStorageStatsCommand.java @@ -0,0 +1,66 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; + +public class GetBackupStorageStatsCommand extends Command { + private String backupRepoType; + private String backupRepoAddress; + @LogLevel(LogLevel.Log4jLevel.Off) + private String mountOptions; + + public GetBackupStorageStatsCommand(String backupRepoType, String backupRepoAddress, String mountOptions) { + super(); + this.backupRepoType = backupRepoType; + this.backupRepoAddress = backupRepoAddress; + this.mountOptions = mountOptions; + } + + public String getBackupRepoType() { + return backupRepoType; + } + + public void setBackupRepoType(String backupRepoType) { + this.backupRepoType = backupRepoType; + } + + public String getBackupRepoAddress() { + return backupRepoAddress; + } + + public void setBackupRepoAddress(String backupRepoAddress) { + this.backupRepoAddress = backupRepoAddress; + } + + public String getMountOptions() { + return mountOptions == null ? "" : mountOptions; + } + + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java index e8364520ed05..98efa94ceca2 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java @@ -144,11 +144,21 @@ public Long getUsedBytes() { return usedBytes; } + @Override + public void setUsedBytes(Long usedBytes) { + this.usedBytes = usedBytes; + } + @Override public Long getCapacityBytes() { return capacityBytes; } + @Override + public void setCapacityBytes(Long capacityBytes) { + this.capacityBytes = capacityBytes; + } + public Date getCreated() { return created; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java index 0034bfb30ab6..6dec994be0c9 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java @@ -28,4 +28,6 @@ public interface BackupRepositoryDao extends GenericDao listByZoneAndProvider(Long zoneId, String provider); BackupRepository findByBackupOfferingId(Long backupOfferingId); + + boolean updateCapacity(BackupRepository backupRepository, Long capacityBytes, Long usedBytes); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java index 460b6d8aba45..ea969988e2bb 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java @@ -64,4 +64,12 @@ public BackupRepository findByBackupOfferingId(Long backupOfferingId) { } return findByUuid(offering.getExternalId()); } + + @Override + public boolean updateCapacity(BackupRepository backupRepository, Long capacityBytes, Long usedBytes) { + BackupRepositoryVO repository = findById(backupRepository.getId()); + repository.setCapacityBytes(capacityBytes); + repository.setUsedBytes(usedBytes); + return update(repository.getId(), repository); + } } diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index 76402edbd4fc..91226c9f7595 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -183,6 +183,10 @@ public Pair getBackupStorageStats(Long zoneId) { return new Pair<>(8L * 1024 * 1024 * 1024, 10L * 1024 * 1024 * 1024); } + @Override + public void syncBackupStorageStats(Long zoneId) { + } + @Override public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { return true; diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 2913161afe82..91f4f3258830 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -57,6 +57,7 @@ import java.util.HashMap; import java.util.Objects; import java.util.Optional; +import java.util.Random; import java.util.UUID; import java.util.stream.Collectors; @@ -69,6 +70,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private BackupRepositoryDao backupRepositoryDao; + @Inject + private BackupRepositoryService backupRepositoryService; + @Inject private HostDao hostDao; @@ -90,6 +94,19 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject BackupManager backupManager; + private Host getUpHostInZone(Long zoneId) { + List hosts = hostDao.listByDataCenterIdAndHypervisorType(zoneId, Hypervisor.HypervisorType.KVM); + if (hosts.size() == 0) { + return null; + } + + Random random = new Random(); + int randomIndex = random.nextInt(hosts.size()); + HostVO hostInZone = hosts.get(randomIndex); + LOG.debug("Found Host {} in zone {}", zoneId); + return hostInZone; + } + protected Host getLastVMHypervisorHost(VirtualMachine vm) { Long hostId = vm.getLastHostId(); if (hostId == null) { @@ -110,13 +127,7 @@ protected Host getLastVMHypervisorHost(VirtualMachine vm) { } } // Try to find any Host in the zone - for (final HostVO hostInZone : hostDao.listByDataCenterIdAndHypervisorType(host.getDataCenterId(), Hypervisor.HypervisorType.KVM)) { - if (hostInZone.getStatus() == Status.Up) { - LOG.debug("Found Host {} in zone {}", hostInZone, host.getDataCenterId()); - return hostInZone; - } - } - return null; + return getUpHostInZone(host.getDataCenterId()); } protected Host getVMHypervisorHost(VirtualMachine vm) { @@ -434,7 +445,36 @@ public boolean supportsInstanceFromBackup() { @Override public Pair getBackupStorageStats(Long zoneId) { - return new Pair<>(0L, 0L); + final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); + Long totalSize = 0L; + Long usedSize = 0L; + for (final BackupRepository repository : repositories) { + if (repository.getCapacityBytes() != null) { + totalSize += repository.getCapacityBytes(); + } + if (repository.getUsedBytes() != null) { + usedSize += repository.getUsedBytes(); + } + } + return new Pair<>(usedSize, totalSize); + } + + @Override + public void syncBackupStorageStats(Long zoneId) { + final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); + final Host host = getUpHostInZone(zoneId); + for (final BackupRepository repository : repositories) { + GetBackupStorageStatsCommand command = new GetBackupStorageStatsCommand(repository.getType(), repository.getAddress(), repository.getMountOptions()); + BackupStorageStatsAnswer answer; + try { + answer = (BackupStorageStatsAnswer) agentManager.send(host.getId(), command); + backupRepositoryDao.updateCapacity(repository, answer.getTotalSize(), answer.getUsedSize()); + } catch (AgentUnavailableException e) { + logger.warn("Unable to contact backend control plane to get backup stats for repository: {}", repository.getName()); + } catch (OperationTimedoutException e) { + logger.warn("Operation to get backup stats timed out for the repository: " + repository.getName()); + } + } } @Override diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index 5e2ac3b7f5e1..194d15774d10 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -645,6 +645,10 @@ public Pair getBackupStorageStats(Long zoneId) { return new Pair<>(0L, 0L); } + @Override + public void syncBackupStorageStats(Long zoneId) { + } + @Override public boolean willDeleteBackupsOnOfferingRemoval() { return false; } diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index a5a8a8c1edfd..2f031fe7a03e 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -371,6 +371,10 @@ public Pair getBackupStorageStats(Long zoneId) { return new Pair<>(0L, 0L); } + @Override + public void syncBackupStorageStats(Long zoneId) { + } + @Override public String getConfigComponentName() { return BackupService.class.getSimpleName(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetBackupStatsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetBackupStatsCommandWrapper.java new file mode 100644 index 000000000000..1d999b795b3e --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetBackupStatsCommandWrapper.java @@ -0,0 +1,71 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.cloudstack.backup.BackupStorageStatsAnswer; +import org.apache.cloudstack.backup.GetBackupStorageStatsCommand; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Pair; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = GetBackupStorageStatsCommand.class) +public class LibvirtGetBackupStatsCommandWrapper extends CommandWrapper { + @Override + public Answer execute(GetBackupStorageStatsCommand command, LibvirtComputingResource libvirtComputingResource) { + final String backupRepoType = command.getBackupRepoType(); + final String backupRepoAddress = command.getBackupRepoAddress(); + final String mountOptions = command.getMountOptions(); + + List commands = new ArrayList<>(); + commands.add(new String[]{ + libvirtComputingResource.getNasBackupPath(), + "-o", "stats", + "-t", backupRepoType, + "-s", backupRepoAddress, + "-m", mountOptions + }); + + Pair result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); + + logger.debug(String.format("Get backup storage stats result: %s , exit code: %s", result.second(), result.first())); + + if (result.first() != 0) { + logger.debug(String.format("Failed to get backup storage stats: %s", result.second())); + return new BackupStorageStatsAnswer(command, false, result.second()); + } + + BackupStorageStatsAnswer answer = new BackupStorageStatsAnswer(command, false, result.second()); + + String [] stats = result.second().split("\\s+"); + Long total = Long.parseLong(stats[1]); + Long used = Long.parseLong(stats[2]); + answer.setTotalSize(total); + answer.setUsedSize(used); + + return answer; + } +} diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index 5b264321bd8d..7811e6fc46d8 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -96,6 +96,15 @@ delete_backup() { rmdir $mount_point } +get_backup_stats() { + mount_operation + + echo $mount_point + df -P $mount_point 2>/dev/null | awk 'NR==2 {print $2, $3}' + umount $mount_point + rmdir $mount_point +} + mount_operation() { mount_point=$(mktemp -d -t csbackup.XXXXX) dest="$mount_point/${BACKUP_DIR}" @@ -166,4 +175,6 @@ if [ "$OP" = "backup" ]; then fi elif [ "$OP" = "delete" ]; then delete_backup +elif [ "$OP" = "stats" ]; then + get_backup_stats fi diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index b60b0eb08975..1e83581366ae 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -3393,7 +3393,7 @@ List getStorageCapacities(Long clusterId, Long podId, Long zoneI if (capacityTypes.contains(Capacity.CAPACITY_TYPE_OBJECT_STORAGE)) { capacities.add(_storageMgr.getObjectStorageUsedStats(dc.getId())); } - if (capacityTypes.contains(Capacity.CAPACITY_TYPE_OBJECT_STORAGE)) { + if (capacityTypes.contains(Capacity.CAPACITY_TYPE_BACKUP_STORAGE)) { capacities.add((CapacityVO) backupManager.getBackupStorageUsedStats(dc.getId())); } for (CapacityVO capacity : capacities) { diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 7e54ff163af1..ba26db384a66 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -1619,6 +1619,8 @@ protected void runInContext() { continue; } + backupProvider.syncBackupStorageStats(dataCenter.getId()); + List vms = vmInstanceDao.listByZoneWithBackups(dataCenter.getId(), null); if (vms == null || vms.isEmpty()) { logger.debug("Can't find any VM to sync backups in zone {}", dataCenter); From 7791ff3c2356728730755a91be0b0463a217a70a Mon Sep 17 00:00:00 2001 From: abh1sar Date: Fri, 7 Mar 2025 05:17:07 +0530 Subject: [PATCH 045/147] Don't show object store and backup capacity if total capacity is 0 --- .../api/command/admin/resource/ListCapacityCmd.java | 2 ++ .../main/java/com/cloud/api/ApiResponseHelper.java | 5 +++++ .../java/com/cloud/storage/StorageManagerImpl.java | 11 ++++++----- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java index 6b31c4cc43cd..e0a38b6aca52 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java @@ -134,6 +134,8 @@ public int compare(CapacityResponse resp1, CapacityResponse resp2) { int res = resp1.getZoneName().compareTo(resp2.getZoneName()); if (res != 0) { return res; + } else if (getSortBy() != null) { + return 0; } else { return resp1.getCapacityType().compareTo(resp2.getCapacityType()); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index c089427203d6..fabe327b2352 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -2073,6 +2073,11 @@ public List createCapacityResponse(List re List capacityResponses = new ArrayList(); for (Capacity summedCapacity : result) { + if (summedCapacity.getTotalCapacity() == 0 && + (summedCapacity.getCapacityType() == Capacity.CAPACITY_TYPE_BACKUP_STORAGE || + summedCapacity.getCapacityType() == Capacity.CAPACITY_TYPE_OBJECT_STORAGE)) { + continue; + } CapacityResponse capacityResponse = new CapacityResponse(); capacityResponse.setCapacityTotal(summedCapacity.getTotalCapacity()); if (summedCapacity.getAllocatedCapacity() != null) { diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index f96dbadf2aaa..c4f749b17c11 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -4248,7 +4248,7 @@ public ObjectStore discoverObjectStore(String name, String url, Long size, Strin params.put("url", url); params.put("name", name); if (size == null) { - params.put("size", 0); + params.put("size", 0L); } else { params.put("size", size); } @@ -4347,11 +4347,12 @@ public CapacityVO getObjectStorageUsedStats(Long zoneId) { Long allocated = 0L; Long total = 0L; for (ObjectStoreVO objectStore: objectStores) { - if (objectStore.getAllocatedSize() == null || objectStore.getTotalSize() == null) { - continue; + if (objectStore.getAllocatedSize() != null) { + allocated += objectStore.getAllocatedSize(); + } + if (objectStore.getTotalSize() != null) { + total += objectStore.getTotalSize(); } - allocated += objectStore.getAllocatedSize(); - total += objectStore.getTotalSize(); } CapacityVO capacity = new CapacityVO(null, zoneId, null, null, allocated, total, Capacity.CAPACITY_TYPE_OBJECT_STORAGE); return capacity; From 051d52c1a9770ecb9b3b4cc57965b24d65493cc9 Mon Sep 17 00:00:00 2001 From: abh1sar Date: Mon, 10 Mar 2025 17:16:59 +0530 Subject: [PATCH 046/147] Handle backup listing and restore for expunged and purged Instances --- .../java/com/cloud/vm/VirtualMachine.java | 1 - .../apache/cloudstack/api/ApiConstants.java | 1 + .../api/response/BackupResponse.java | 12 ++++++ .../org/apache/cloudstack/backup/Backup.java | 6 ++- .../cloudstack/backup/BackupManager.java | 2 + .../cloud/vm/VirtualMachineManagerImpl.java | 5 +++ .../apache/cloudstack/backup/BackupVO.java | 25 +++++++++--- .../cloudstack/backup/dao/BackupDaoImpl.java | 33 ++++++++-------- .../META-INF/db/schema-42010to42100.sql | 38 +++++++++++-------- .../cloudstack/backup/NASBackupProvider.java | 9 ++++- .../cloudstack/backup/BackupManagerImpl.java | 32 ++++++++++++---- ui/src/components/view/DetailsTab.vue | 3 +- ui/src/components/view/ListView.vue | 3 +- ui/src/config/section/compute.js | 2 +- .../compute/wizard/TemplateIsoSelection.vue | 1 + 15 files changed, 122 insertions(+), 51 deletions(-) diff --git a/api/src/main/java/com/cloud/vm/VirtualMachine.java b/api/src/main/java/com/cloud/vm/VirtualMachine.java index e2ea408e7b8c..d244de7115e8 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachine.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachine.java @@ -128,7 +128,6 @@ public static StateMachine2 getStat s_fsm.addTransition(new Transition(State.Error, VirtualMachine.Event.DestroyRequested, State.Expunging, null)); s_fsm.addTransition(new Transition(State.Error, VirtualMachine.Event.ExpungeOperation, State.Expunging, null)); s_fsm.addTransition(new Transition(State.Stopped, Event.RestoringRequested, State.Restoring, null)); - s_fsm.addTransition(new Transition(State.Expunging, Event.RestoringRequested, State.Restoring, null)); s_fsm.addTransition(new Transition(State.Destroyed, Event.RestoringRequested, State.Restoring, null)); s_fsm.addTransition(new Transition(State.Restoring, Event.RestoringSuccess, State.Stopped, null)); s_fsm.addTransition(new Transition(State.Restoring, Event.RestoringFailed, State.Stopped, null)); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 696ebfd0de45..cde0e0dc89e2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -368,6 +368,7 @@ public class ApiConstants { public static final String OP = "op"; public static final String OPTION = "option"; public static final String OPTIONS = "options"; + public static final String IS_ORPHAN = "isorphan"; public static final String OS_CATEGORY_ID = "oscategoryid"; public static final String OS_CATEGORY_NAME = "oscategoryname"; public static final String OS_NAME = "osname"; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java index bac7fabf22c2..79d9e3ce1e72 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java @@ -119,6 +119,10 @@ public class BackupResponse extends BaseResponse { @Param(description = "the interval type of the backup schedule") private String intervalType; + @SerializedName(ApiConstants.IS_ORPHAN) + @Param(description = "The backups is orphaned and no associated with an existing VM. A new VM can still be created using the backup") + private Boolean isOrphan; + public String getId() { return id; } @@ -294,4 +298,12 @@ public String getIntervalType() { public void setIntervalType(String intervalType) { this.intervalType = intervalType; } + + public Boolean getIsOrphan() { + return isOrphan; + } + + public void setIsOrphan(Boolean isOrphan) { + this.isOrphan = isOrphan; + } } diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index 37dadb5967b3..376e53cb06f5 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -157,8 +157,10 @@ public String toString() { } } - long getVmId(); + Long getVmId(); + String getVmName(); long getBackupOfferingId(); + void setVmName(String vmName); String getExternalId(); String getType(); Date getDate(); @@ -168,7 +170,7 @@ public String toString() { void setName(String name); String getDescription(); void setDescription(String description); - short getBackupIntervalType(); + Short getBackupIntervalType(); List getBackedUpVolumes(); long getZoneId(); Map getDetails(); diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index e75c97b75be4..456f81cc1934 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -254,5 +254,7 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer Map getDiskOfferingDetailsForBackup(Long vmId); + void updateOrphanedBackups(VirtualMachine vm); + Capacity getBackupStorageUsedStats(Long zoneId); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 8e0815e77296..305833f09138 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -57,6 +57,7 @@ import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd; import org.apache.cloudstack.api.command.admin.volume.MigrateVolumeCmdByAdmin; import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd; +import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; @@ -409,6 +410,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac ResourceCleanupService resourceCleanupService; @Inject VmWorkJobDao vmWorkJobDao; + @Inject + BackupManager backupManager; private SingleCache> vmIdsInProgressCache; @@ -650,6 +653,8 @@ protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableExcepti throw new CloudRuntimeException("Unable to expunge " + vm, e); } + backupManager.updateOrphanedBackups(vm); + logger.debug("Expunging vm " + vm); final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index 17974bef347b..ca5c7ab21b29 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -60,7 +60,10 @@ public class BackupVO implements Backup { private String uuid; @Column(name = "vm_id") - private long vmId; + private Long vmId; + + @Column(name = "vm_name") + private String vmName; @Column(name = "external_id") private String externalId; @@ -98,7 +101,7 @@ public class BackupVO implements Backup { private long zoneId; @Column(name = "backup_interval_type") - private short backupIntervalType; + private Short backupIntervalType; @Column(name = "backed_volumes", length = 65535) protected String backedUpVolumes; @@ -113,7 +116,7 @@ public BackupVO() { @Override public String toString() { return String.format("Backup %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields( - this, "id", "uuid", "vmId", "backupType", "externalId")); + this, "id", "uuid", "vmId", "vmName", "backupType", "externalId")); } @Override @@ -127,14 +130,24 @@ public String getUuid() { } @Override - public long getVmId() { + public Long getVmId() { return vmId; } - public void setVmId(long vmId) { + public void setVmId(Long vmId) { this.vmId = vmId; } + @Override + public String getVmName() { + return vmName; + } + + @Override + public void setVmName(String vmName) { + this.vmName = vmName; + } + @Override public String getExternalId() { return externalId; @@ -224,7 +237,7 @@ public void setZoneId(long zoneId) { } @Override - public short getBackupIntervalType() { + public Short getBackupIntervalType() { return backupIntervalType; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java index 6ba782fabd84..b9a45b407970 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java @@ -244,36 +244,39 @@ public void saveDetails(BackupVO backup) { @Override public BackupResponse newBackupResponse(Backup backup, Boolean listVmDetails) { - VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); - AccountVO account = accountDao.findByIdIncludingRemoved(vm.getAccountId()); - DomainVO domain = domainDao.findByIdIncludingRemoved(vm.getDomainId()); - DataCenterVO zone = dataCenterDao.findByIdIncludingRemoved(vm.getDataCenterId()); - Long offeringId = backup.getBackupOfferingId(); - if (offeringId == null) { - offeringId = vm.getBackupOfferingId(); + VMInstanceVO vm = null; + if (backup.getVmId() != null) { + vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); } + AccountVO account = accountDao.findByIdIncludingRemoved(backup.getAccountId()); + DomainVO domain = domainDao.findByIdIncludingRemoved(backup.getDomainId()); + DataCenterVO zone = dataCenterDao.findByIdIncludingRemoved(backup.getZoneId()); + Long offeringId = backup.getBackupOfferingId(); BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(offeringId); BackupResponse response = new BackupResponse(); response.setId(backup.getUuid()); - if (backup.getName() != null) { - response.setName(backup.getName()); + response.setName(backup.getName()); + response.setDescription(backup.getDescription()); + if (vm != null) { + response.setVmId(vm.getUuid()); + response.setVmName(vm.getHostName()); } else { - response.setName(vm.getHostName()); + response.setVmName(backup.getVmName()); + response.setIsOrphan(true); } - response.setDescription(backup.getDescription()); - response.setVmId(vm.getUuid()); - response.setVmName(vm.getHostName()); response.setExternalId(backup.getExternalId()); response.setType(backup.getType()); response.setDate(backup.getDate()); response.setSize(backup.getSize()); response.setProtectedSize(backup.getProtectedSize()); response.setStatus(backup.getStatus()); - response.setIntervalType(Backup.Type.values()[backup.getBackupIntervalType()].toString()); + if (backup.getBackupIntervalType() != null) { + response.setIntervalType(Backup.Type.values()[backup.getBackupIntervalType()].toString()); + } // ACS 4.20: For backups taken prior this release the backup.backed_volumes column would be empty hence use vm_instance.backup_volumes String backedUpVolumes; - if (Objects.isNull(backup.getBackedUpVolumes())) { + if (Objects.isNull(backup.getBackedUpVolumes()) && vm != null) { backedUpVolumes = new Gson().toJson(vm.getBackupVolumeList().toArray(), Backup.VolumeInfo[].class); } else { backedUpVolumes = new Gson().toJson(backup.getBackedUpVolumes().toArray(), Backup.VolumeInfo[].class); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index bdba38c05de2..2fbad24f3ed7 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -19,19 +19,36 @@ -- Schema upgrade from 4.20.1.0 to 4.21.0.0 --; +-- Add console_endpoint_creator_address column to cloud.console_session table +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'console_endpoint_creator_address', 'VARCHAR(45)'); + +-- Add client_address column to cloud.console_session table +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'client_address', 'VARCHAR(45)'); + +-- Allow default roles to use quotaCreditsList +INSERT INTO `cloud`.`role_permissions` (uuid, role_id, rule, permission, sort_order) +SELECT uuid(), role_id, 'quotaCreditsList', permission, sort_order +FROM `cloud`.`role_permissions` rp +WHERE rp.rule = 'quotaStatement' +AND NOT EXISTS(SELECT 1 FROM cloud.role_permissions rp_ WHERE rp.role_id = rp_.role_id AND rp_.rule = 'quotaCreditsList'); + +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.host', 'last_mgmt_server_id', 'bigint unsigned DEFAULT NULL COMMENT "last management server this host is connected to" AFTER `mgmt_server_id`'); + -- Add column max_backup to backup_schedule table ALTER TABLE `cloud`.`backup_schedule` ADD COLUMN `max_backups` int(8) default NULL COMMENT 'maximum number of backups to maintain'; -- Add columns name, description and backup_interval_type to backup table ALTER TABLE `cloud`.`backups` ADD COLUMN `backup_interval_type` int(5) COMMENT 'type of backup, e.g. manual, recurring - hourly, daily, weekly or monthly'; -ALTER TABLE `cloud`.`backups` ADD COLUMN `name` varchar(255) COMMENT 'name of the backup'; +ALTER TABLE `cloud`.`backups` ADD COLUMN `name` varchar(255) NOT NULL COMMENT 'name of the backup'; +ALTER TABLE `cloud`.`backups` ADD COLUMN `vm_name` varchar(255) COMMENT 'name of the vm for which backup is taken, only set for orphaned backups'; ALTER TABLE `cloud`.`backups` ADD COLUMN `description` varchar(1024) COMMENT 'description for the backup'; +UPDATE `cloud`.`backups` JOIN `cloud`.`vm_instance` ON `backups`.`vm_id` = `vm_instance`.`id` SET `backups`.`name` = `vm_instance`.`name`; --- Add console_endpoint_creator_address column to cloud.console_session table -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'console_endpoint_creator_address', 'VARCHAR(45)'); - --- Add client_address column to cloud.console_session table -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'client_address', 'VARCHAR(45)'); +-- Make the column vm_id in backups table nullable and set it to null if foreign key is deleted +ALTER TABLE `cloud`.`backups` DROP FOREIGN KEY `fk_backup__vm_id``; +ALTER TABLE `cloud`.`backups` MODIFY COLUMN `vm_id` BIGINT UNSIGNED NULL; +ALTER TABLE `cloud`.`backups` ADD CONSTRAINT `fk_backup__vm_id`` FOREIGN KEY (`vm_id``) REFERENCES `cloud`.`vm_instance`(`id`) ON DELETE SET NULL; +ALTER TABLE `cloud`.`backups` varchar(255) NOT NULL COMMENT 'name of the backup' -- Create backup details table CREATE TABLE `cloud`.`backup_details` ( @@ -44,15 +61,6 @@ CREATE TABLE `cloud`.`backup_details` ( CONSTRAINT `fk_backup_details__backup_id` FOREIGN KEY `fk_backup_details__backup_id`(`backup_id`) REFERENCES `backups`(`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8; --- Allow default roles to use quotaCreditsList -INSERT INTO `cloud`.`role_permissions` (uuid, role_id, rule, permission, sort_order) -SELECT uuid(), role_id, 'quotaCreditsList', permission, sort_order -FROM `cloud`.`role_permissions` rp -WHERE rp.rule = 'quotaStatement' -AND NOT EXISTS(SELECT 1 FROM cloud.role_permissions rp_ WHERE rp.role_id = rp_.role_id AND rp_.rule = 'quotaCreditsList'); - -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.host', 'last_mgmt_server_id', 'bigint unsigned DEFAULT NULL COMMENT "last management server this host is connected to" AFTER `mgmt_server_id`'); - -- Add column allocated_size to object_store table. Rename column 'used_bytes' to 'used_size' ALTER TABLE `cloud`.`object_store` ADD COLUMN `allocated_size` bigint unsigned COMMENT 'allocated size in bytes'; ALTER TABLE `cloud`.`object_store` CHANGE COLUMN `used_bytes` `used_size` BIGINT UNSIGNED COMMENT 'used size in bytes'; diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index d30a89f7852a..4075d0b81303 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -370,8 +370,13 @@ public boolean deleteBackup(Backup backup, boolean forced) { throw new CloudRuntimeException("No valid backup repository found for the VM, please check the attached backup offering"); } - final VirtualMachine vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); - final Host host = getLastVMHypervisorHost(vm); + final Host host; + final VirtualMachine vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); + if (vm != null) { + host = getLastVMHypervisorHost(vm); + } else { + host = getUpHostInZone(backup.getZoneId()); + } DeleteBackupCommand command = new DeleteBackupCommand(backup.getExternalId(), backupRepository.getType(), backupRepository.getAddress(), backupRepository.getMountOptions()); diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 8185b295ccaf..2d1d2d7b644a 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -1227,13 +1227,17 @@ public boolean deleteBackup(final Long backupId, final Boolean forced) { if (backup == null) { throw new CloudRuntimeException("Backup " + backupId + " does not exist"); } + final Long vmId = backup.getVmId(); - final VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(vmId); - if (vm == null) { - throw new CloudRuntimeException("VM " + vmId + " does not exist"); + if (vmId == null) { + logger.debug("Deleting orphaned backup with ID {}", backupId); + } else { + logger.debug("Deleting backup {} belonging to instance {}", backupId, vmId); } - validateBackupForZone(vm.getDataCenterId()); - accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); + final VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(vmId); + + validateBackupForZone(backup.getZoneId()); + accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm == null ? backup : vm); final BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); if (offering == null) { throw new CloudRuntimeException(String.format("Backup offering with ID [%s] does not exist.", backup.getBackupOfferingId())); @@ -1241,8 +1245,8 @@ public boolean deleteBackup(final Long backupId, final Boolean forced) { final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); boolean result = backupProvider.deleteBackup(backup, forced); if (result) { - resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); - resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); return backupDao.remove(backup.getId()); } throw new CloudRuntimeException("Failed to delete the backup"); @@ -1793,6 +1797,20 @@ public BackupOffering updateBackupOffering(UpdateBackupOfferingCmd updateBackupO return response; } + @Override + public void updateOrphanedBackups(VirtualMachine vm) { + List backups = backupDao.listByVmId(vm.getDataCenterId(), vm.getId()); + if (backups.size() == 0) { + return; + } + for (Backup backup : backups) { + BackupVO backupVO = backupDao.findById(backup.getId()); + backupVO.setVmId(null); + backupVO.setVmName(vm.getHostName()); + backupDao.update(backup.getId(), backupVO); + } + } + @Override public CapacityVO getBackupStorageUsedStats(Long zoneId) { final BackupProvider backupProvider = getBackupProvider(zoneId); diff --git a/ui/src/components/view/DetailsTab.vue b/ui/src/components/view/DetailsTab.vue index 3622f87e67d8..2b74d5c2f44c 100644 --- a/ui/src/components/view/DetailsTab.vue +++ b/ui/src/components/view/DetailsTab.vue @@ -62,7 +62,8 @@
- {{ volume.type }} - {{ volume.path }} ({{ parseFloat(volume.size / (1024.0 * 1024.0 * 1024.0)).toFixed(1) }} GB) + {{ volume.type }} - {{ volume.path }} + {{ volume.type }} - {{ volume.path }} ({{ parseFloat(volume.size / (1024.0 * 1024.0 * 1024.0)).toFixed(1) }} GB)
diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 30b0d4bf671c..4c2d5b40c950 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -179,7 +179,8 @@ {{ text }} +
{{ $t('label.cancel') }} {{ $t('label.ok') }} From 9894effe4cb2381389a3b823daa21ad1c304b280 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 11 Mar 2025 09:28:21 +0530 Subject: [PATCH 048/147] Added UT to AlertManager, BackupDao, NasBackupProvider and BackupManager. --- .../backup/dao/BackupDaoImplTest.java | 150 +++++++++++ .../backup/NASBackupProviderTest.java | 255 ++++++++++++++++++ .../com/cloud/alert/AlertManagerImplTest.java | 49 ++++ .../cloudstack/backup/BackupManagerTest.java | 60 +++++ 4 files changed, 514 insertions(+) create mode 100644 engine/schema/src/test/java/org/apache/cloudstack/backup/dao/BackupDaoImplTest.java create mode 100644 plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java diff --git a/engine/schema/src/test/java/org/apache/cloudstack/backup/dao/BackupDaoImplTest.java b/engine/schema/src/test/java/org/apache/cloudstack/backup/dao/BackupDaoImplTest.java new file mode 100644 index 000000000000..d0760e254c07 --- /dev/null +++ b/engine/schema/src/test/java/org/apache/cloudstack/backup/dao/BackupDaoImplTest.java @@ -0,0 +1,150 @@ +package org.apache.cloudstack.backup.dao; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupOfferingVO; +import org.apache.cloudstack.backup.BackupVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; + +@RunWith(MockitoJUnitRunner.class) +public class BackupDaoImplTest { + @Spy + @InjectMocks + private BackupDaoImpl backupDao; + + @Mock + BackupDetailsDao backupDetailsDao; + + @Mock + SearchBuilder backupSearch; + + @Mock + VMInstanceDao vmInstanceDao; + + @Mock + AccountDao accountDao; + + @Mock + DomainDao domainDao; + + @Mock + DataCenterDao dataCenterDao; + + @Mock + BackupOfferingDao backupOfferingDao; + + @Test + public void testLoadDetails() { + Long backupId = 1L; + BackupVO backup = new BackupVO(); + ReflectionTestUtils.setField(backup, "id", backupId); + Map details = new HashMap<>(); + details.put("key1", "value1"); + details.put("key2", "value2"); + + Mockito.when(backupDetailsDao.listDetailsKeyPairs(backupId)).thenReturn(details); + + backupDao.loadDetails(backup); + + Assert.assertEquals(details, backup.getDetails()); + Mockito.verify(backupDetailsDao).listDetailsKeyPairs(backupId); + } + + @Test + public void testSaveDetails() { + Long backupId = 1L; + BackupVO backup = new BackupVO(); + ReflectionTestUtils.setField(backup, "id", backupId); + Map details = new HashMap<>(); + details.put("key1", "value1"); + details.put("key2", "value2"); + backup.setDetails(details); + + backupDao.saveDetails(backup); + + Mockito.verify(backupDetailsDao).saveDetails(Mockito.anyList()); + } + + @Test + public void testNewBackupResponse() { + Long vmId = 1L; + Long accountId = 2L; + Long domainId = 3L; + Long zoneId = 4L; + Long offeringId = 5L; + Long backupId = 6L; + BackupVO backup = new BackupVO(); + ReflectionTestUtils.setField(backup, "id", backupId); + ReflectionTestUtils.setField(backup, "uuid", "backup-uuid"); + backup.setVmId(vmId); + backup.setAccountId(accountId); + backup.setDomainId(domainId); + backup.setZoneId(zoneId); + backup.setBackupOfferingId(offeringId); + backup.setType("Full"); + backup.setBackupIntervalType((short) Backup.Type.MANUAL.ordinal()); + + VMInstanceVO vm = new VMInstanceVO(vmId, 0L, "test-vm", "test-vm", VirtualMachine.Type.User, + 0L, Hypervisor.HypervisorType.Simulator, 0L, domainId, accountId, 0L, false); + vm.setDataCenterId(zoneId); + + AccountVO account = new AccountVO(); + account.setUuid("account-uuid"); + account.setAccountName("test-account"); + + DomainVO domain = new DomainVO(); + domain.setUuid("domain-uuid"); + domain.setName("test-domain"); + + DataCenterVO zone = new DataCenterVO(1L, "test-zone", null, null, null, null, null, null, null, null, DataCenter.NetworkType.Advanced, null, null); + zone.setUuid("zone-uuid"); + + BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); + Mockito.when(offering.getUuid()).thenReturn("offering-uuid"); + Mockito.when(offering.getName()).thenReturn("test-offering"); + + Mockito.when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + Mockito.when(accountDao.findByIdIncludingRemoved(accountId)).thenReturn(account); + Mockito.when(domainDao.findByIdIncludingRemoved(domainId)).thenReturn(domain); + Mockito.when(dataCenterDao.findByIdIncludingRemoved(zoneId)).thenReturn(zone); + Mockito.when(backupOfferingDao.findByIdIncludingRemoved(offeringId)).thenReturn(offering); + + BackupResponse response = backupDao.newBackupResponse(backup, false); + + Assert.assertEquals("backup-uuid", response.getId()); + Assert.assertEquals("test-vm", response.getVmName()); + Assert.assertEquals("account-uuid", response.getAccountId()); + Assert.assertEquals("test-account", response.getAccount()); + Assert.assertEquals("domain-uuid", response.getDomainId()); + Assert.assertEquals("test-domain", response.getDomain()); + Assert.assertEquals("zone-uuid", response.getZoneId()); + Assert.assertEquals("test-zone", response.getZone()); + Assert.assertEquals("offering-uuid", response.getBackupOfferingId()); + Assert.assertEquals("test-offering", response.getBackupOffering()); + Assert.assertEquals("MANUAL", response.getIntervalType()); + } +} \ No newline at end of file diff --git a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java new file mode 100644 index 000000000000..cc0677f75b01 --- /dev/null +++ b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup; + +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupRepositoryDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.agent.AgentManager; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.Pair; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; + +@RunWith(MockitoJUnitRunner.class) +public class NASBackupProviderTest { + @Spy + @InjectMocks + private NASBackupProvider nasBackupProvider; + + @Mock + private BackupDao backupDao; + + @Mock + private BackupRepositoryDao backupRepositoryDao; + + @Mock + private BackupOfferingDao backupOfferingDao; + + @Mock + private VMInstanceDao vmInstanceDao; + + @Mock + private AgentManager agentManager; + + @Mock + private VolumeDao volumeDao; + + @Mock + private HostDao hostDao; + + @Mock + private BackupManager backupManager; + + @Test + public void testDeleteBackup() throws OperationTimedoutException, AgentUnavailableException { + Long hostId = 1L; + BackupVO backup = new BackupVO(); + backup.setBackupOfferingId(1L); + backup.setVmId(1L); + backup.setExternalId("externalId"); + ReflectionTestUtils.setField(backup, "id", 1L); + + BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1024L); + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + HostVO host = mock(HostVO.class); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + Mockito.when(backupRepositoryDao.findByBackupOfferingId(1L)).thenReturn(backupRepository); + Mockito.when(vmInstanceDao.findByIdIncludingRemoved(1L)).thenReturn(vm); + Mockito.when(agentManager.send(anyLong(), Mockito.any(DeleteBackupCommand.class))).thenReturn(new BackupAnswer(new DeleteBackupCommand(null, null, null, null), true, "details")); + Mockito.when(backupDao.remove(1L)).thenReturn(true); + + boolean result = nasBackupProvider.deleteBackup(backup, true); + Assert.assertTrue(result); + } + + @Test + public void testGetBackupMetrics() { + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + + Backup backup1 = mock(Backup.class); + Mockito.when(backup1.getSize()).thenReturn(100L); + Mockito.when(backup1.getProtectedSize()).thenReturn(50L); + Backup backup2 = mock(Backup.class); + Mockito.when(backup2.getSize()).thenReturn(200L); + Mockito.when(backup2.getProtectedSize()).thenReturn(50L); + + Mockito.when(backupDao.listByVmId(null, 1L)).thenReturn(List.of(backup1, backup2)); + + Map result = nasBackupProvider.getBackupMetrics(1L, Collections.singletonList(vm)); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(300L, result.get(vm).getBackupSize().longValue()); + Assert.assertEquals(100L, result.get(vm).getDataSize().longValue()); + } + + @Test + public void testSyncBackupStorageStats() throws AgentUnavailableException, OperationTimedoutException { + BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1024L); + + HostVO host = mock(HostVO.class); + Mockito.when(hostDao.listByDataCenterIdAndHypervisorType(1L, Hypervisor.HypervisorType.KVM)).thenReturn(Collections.singletonList(host)); + + Mockito.when(backupRepositoryDao.listByZoneAndProvider(1L, "nas")).thenReturn(Collections.singletonList(backupRepository)); + GetBackupStorageStatsCommand command = new GetBackupStorageStatsCommand("nfs", "address", "sync"); + BackupStorageStatsAnswer answer = new BackupStorageStatsAnswer(command, true, null); + answer.setTotalSize(100L); + answer.setUsedSize(50L); + Mockito.when(agentManager.send(anyLong(), Mockito.any(GetBackupStorageStatsCommand.class))).thenReturn(answer); + + nasBackupProvider.syncBackupStorageStats(1L); + Mockito.verify(backupRepositoryDao, Mockito.times(1)).updateCapacity(backupRepository, 100L, 50L); + } + + @Test + public void testListBackupOfferings() { + BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1024L); + ReflectionTestUtils.setField(backupRepository, "uuid", "uuid"); + + Mockito.when(backupRepositoryDao.listByZoneAndProvider(1L, "nas")).thenReturn(Collections.singletonList(backupRepository)); + + List result = nasBackupProvider.listBackupOfferings(1L); + Assert.assertEquals(1, result.size()); + Assert.assertEquals("test-repo", result.get(0).getName()); + Assert.assertEquals("uuid", result.get(0).getUuid()); + } + + @Test + public void testGetBackupStorageStats() { + BackupRepositoryVO backupRepository1 = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1000L); + backupRepository1.setUsedBytes(500L); + + BackupRepositoryVO backupRepository2 = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 2000L); + backupRepository2.setUsedBytes(600L); + + Mockito.when(backupRepositoryDao.listByZoneAndProvider(1L, "nas")) + .thenReturn(List.of(backupRepository1, backupRepository2)); + + Pair result = nasBackupProvider.getBackupStorageStats(1L); + Assert.assertEquals(Long.valueOf(1100L), result.first()); + Assert.assertEquals(Long.valueOf(3000L), result.second()); + } + + @Test + public void takeBackupSuccessfully() throws AgentUnavailableException, OperationTimedoutException { + Long vmId = 1L; + Long hostId = 2L; + Long backupOfferingId = 3L; + Long accountId = 4L; + Long domainId = 5L; + Long zoneId = 6L; + Long backupId = 7L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(vmId); + Mockito.when(vm.getHostId()).thenReturn(hostId); + Mockito.when(vm.getInstanceName()).thenReturn("test-vm"); + Mockito.when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); + Mockito.when(vm.getAccountId()).thenReturn(accountId); + Mockito.when(vm.getDomainId()).thenReturn(domainId); + Mockito.when(vm.getDataCenterId()).thenReturn(zoneId); + Mockito.when(vm.getHostName()).thenReturn("test-host"); + Mockito.when(vm.getState()).thenReturn(VMInstanceVO.State.Running); + + BackupRepository backupRepository = mock(BackupRepository.class); + Mockito.when(backupRepository.getType()).thenReturn("nfs"); + Mockito.when(backupRepository.getAddress()).thenReturn("address"); + Mockito.when(backupRepository.getMountOptions()).thenReturn("sync"); + Mockito.when(backupRepositoryDao.findByBackupOfferingId(backupOfferingId)).thenReturn(backupRepository); + + HostVO host = mock(HostVO.class); + Mockito.when(host.getId()).thenReturn(hostId); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(host.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + + VolumeVO volume1 = mock(VolumeVO.class); + Mockito.when(volume1.getState()).thenReturn(Volume.State.Ready); + Mockito.when(volume1.getSize()).thenReturn(100L); + VolumeVO volume2 = mock(VolumeVO.class); + Mockito.when(volume2.getState()).thenReturn(Volume.State.Ready); + Mockito.when(volume2.getSize()).thenReturn(200L); + Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of(volume1, volume2)); + + BackupAnswer answer = mock(BackupAnswer.class); + Mockito.when(answer.getResult()).thenReturn(true); + Mockito.when(answer.getSize()).thenReturn(100L); + Mockito.when(agentManager.send(anyLong(), Mockito.any(TakeBackupCommand.class))).thenReturn(answer); + + Map details = new HashMap<>(); + details.put("key1", "value1"); + Mockito.when(backupManager.getDiskOfferingDetailsForBackup(vmId)).thenReturn(details); + + Mockito.when(backupDao.persist(Mockito.any(BackupVO.class))).thenAnswer(invocation -> invocation.getArgument(0)); + Mockito.when(backupDao.update(Mockito.anyLong(), Mockito.any(BackupVO.class))).thenReturn(true); + + Pair result = nasBackupProvider.takeBackup(vm); + + Assert.assertTrue(result.first()); + Assert.assertNotNull(result.second()); + BackupVO backup = (BackupVO) result.second(); + Assert.assertEquals(Optional.ofNullable(100L), Optional.ofNullable(backup.getSize())); + Assert.assertEquals(Backup.Status.BackedUp, backup.getStatus()); + Assert.assertEquals("value1", backup.getDetail("key1")); + Assert.assertEquals("FULL", backup.getType()); + Assert.assertEquals(Optional.of(300L), Optional.of(backup.getProtectedSize())); + Assert.assertEquals(Optional.of(backupOfferingId), Optional.of(backup.getBackupOfferingId())); + Assert.assertEquals(Optional.of(accountId), Optional.of(backup.getAccountId())); + Assert.assertEquals(Optional.of(domainId), Optional.of(backup.getDomainId())); + Assert.assertEquals(Optional.of(zoneId), Optional.of(backup.getZoneId())); + + Mockito.verify(backupDao).persist(Mockito.any(BackupVO.class)); + Mockito.verify(backupDao).update(Mockito.anyLong(), Mockito.any(BackupVO.class)); + Mockito.verify(agentManager).send(anyLong(), Mockito.any(TakeBackupCommand.class)); + } +} diff --git a/server/src/test/java/com/cloud/alert/AlertManagerImplTest.java b/server/src/test/java/com/cloud/alert/AlertManagerImplTest.java index d34d0b5873f2..170fceae9861 100644 --- a/server/src/test/java/com/cloud/alert/AlertManagerImplTest.java +++ b/server/src/test/java/com/cloud/alert/AlertManagerImplTest.java @@ -17,10 +17,15 @@ package com.cloud.alert; import java.io.UnsupportedEncodingException; +import java.util.HashMap; import java.util.List; +import java.util.Optional; import javax.mail.MessagingException; +import javax.naming.ConfigurationException; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.mailing.SMTPMailSender; @@ -39,6 +44,8 @@ import com.cloud.alert.dao.AlertDao; import com.cloud.capacity.Capacity; import com.cloud.capacity.CapacityManager; +import com.cloud.capacity.CapacityVO; +import com.cloud.capacity.dao.CapacityDao; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; @@ -96,6 +103,15 @@ public class AlertManagerImplTest { @Mock SMTPMailSender mailSenderMock; + @Mock + CapacityDao capacityDao; + + @Mock + BackupManager backupManager; + + @Mock + ConfigurationDao configDao; + private final String[] recipients = new String[]{"test@test.com"}; private final String senderAddress = "sender@test.com"; @@ -219,4 +235,37 @@ public void testRecalculateStorageCapacities() { Mockito.verify(storageManager, Mockito.times(2)).createCapacityEntry(sharedPool, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, 10L); Mockito.verify(storageManager, Mockito.times(1)).createCapacityEntry(nonSharedPool, Capacity.CAPACITY_TYPE_LOCAL_STORAGE, 20L); } + + @Test + public void testCheckForAlerts() throws ConfigurationException { + Long zoneId = 1L; + Mockito.doNothing().when(alertManagerImplMock).recalculateCapacity(); + DataCenterVO dc = Mockito.mock(DataCenterVO.class); + Mockito.when(dc.getId()).thenReturn(zoneId); + Mockito.when(dc.getName()).thenReturn("zone1"); + Mockito.when(_dcDao.listAll()).thenReturn(List.of(dc)); + Mockito.when(_dcDao.findById(zoneId)).thenReturn(dc); + Mockito.when(configDao.getConfiguration("management-server", null)).thenReturn(new HashMap<>()); + + alertManagerImplMock.configure(null, null); + CapacityVO secondaryStorageCapacity = new CapacityVO(null, zoneId, null, null, 100L, 200L, Capacity.CAPACITY_TYPE_SECONDARY_STORAGE); + CapacityVO storagePoolCapacity = new CapacityVO(null, zoneId, null, null, 200L, 300L, Capacity.CAPACITY_TYPE_STORAGE); + CapacityVO objectStoreCapacity = new CapacityVO(null, zoneId, null, null, 200L, 300L, Capacity.CAPACITY_TYPE_OBJECT_STORAGE); + CapacityVO backupCapacity = new CapacityVO(null, zoneId, null, null, 180L, 200L, Capacity.CAPACITY_TYPE_BACKUP_STORAGE); + Mockito.when(storageManager.getSecondaryStorageUsedStats(null, zoneId)).thenReturn(secondaryStorageCapacity); + Mockito.when(storageManager.getObjectStorageUsedStats(zoneId)).thenReturn(objectStoreCapacity); + Mockito.when(backupManager.getBackupStorageUsedStats(zoneId)).thenReturn(backupCapacity); + alertManagerImplMock.checkForAlerts(); + + Mockito.verify(alertManagerImplMock).recalculateCapacity(); + + ArgumentCaptor alertCaptor = ArgumentCaptor.forClass(AlertVO.class); + verify(_alertDao).persist(alertCaptor.capture()); + AlertVO capturedAlert = alertCaptor.getValue(); + assertNotNull("Captured alert should not be null", capturedAlert); + assertEquals(Optional.of(zoneId), Optional.ofNullable(capturedAlert.getDataCenterId())); + assertEquals("System Alert: Low Available Backup Storage in availability zone zone1", capturedAlert.getSubject()); + assertEquals("Available backup storage space is low, total: 200.0 MB, used: 180.0 MB (90%)", capturedAlert.getContent()); + assertEquals(AlertManager.AlertType.ALERT_TYPE_BACKUP_STORAGE.getType(), capturedAlert.getType()); + } } diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index edb9a31f8b31..3d2304e11cdb 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -19,6 +19,7 @@ import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.api.query.vo.UserVmJoinVO; import com.cloud.alert.AlertManager; +import com.cloud.capacity.CapacityVO; import com.cloud.configuration.Resource; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; @@ -92,6 +93,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.TimeZone; import static org.junit.Assert.assertEquals; @@ -986,6 +988,11 @@ public void testRemoveVMFromBackupOffering() { } } + private + void setupRestoreBackupToVMMocks() { + + } + @Test public void testRestoreBackupToVM() throws NoTransitionException { Long backupId = 1L; @@ -994,6 +1001,8 @@ public void testRestoreBackupToVM() throws NoTransitionException { Long offeringId = 4L; Long poolId = 5L; + setupRestoreBackupToVMMocks(); + BackupVO backup = mock(BackupVO.class); when(backup.getBackupOfferingId()).thenReturn(offeringId); @@ -1040,4 +1049,55 @@ public void testRestoreBackupToVM() throws NoTransitionException { fail("Test failed due to exception" + e); } } + + @Test + public void testUpdateOrphanedBackups() { + Long vmId = 1L; + Long zoneId = 2L; + Long backupId1 = 3L; + Long backupId2 = 4L; + + VirtualMachine vm = mock(VirtualMachine.class); + when(vm.getDataCenterId()).thenReturn(1L); + when(vm.getId()).thenReturn(vmId); + when(vm.getHostName()).thenReturn("test-vm"); + when(vm.getDataCenterId()).thenReturn(zoneId); + + Backup backup1 = mock(Backup.class); + Backup backup2 = mock(Backup.class); + when(backup1.getId()).thenReturn(backupId1); + when(backup2.getId()).thenReturn(backupId2); + List backups = List.of(backup1, backup2); + when(backupDao.listByVmId(zoneId, vmId)).thenReturn(backups); + + BackupVO backupVO1 = new BackupVO(); + BackupVO backupVO2 = new BackupVO(); + when(backupDao.findById(backupId1)).thenReturn(backupVO1); + when(backupDao.findById(backupId2)).thenReturn(backupVO2); + + backupManager.updateOrphanedBackups(vm); + + Assert.assertEquals(null, backupVO1.getVmId()); + Assert.assertEquals("test-vm", backupVO1.getVmName()); + Assert.assertEquals(null, backupVO2.getVmId()); + Assert.assertEquals("test-vm", backupVO2.getVmName()); + for (Backup backup : backups) { + verify(backupDao).update(Mockito.eq(backup.getId()), any(BackupVO.class)); + } + } + + @Test + public void testGetBackupStorageUsedStats() { + Long zoneId = 1L; + overrideBackupFrameworkConfigValue(); + when(backupManager.getBackupProvider(zoneId)).thenReturn(backupProvider); + when(backupProvider.getBackupStorageStats(zoneId)).thenReturn(new Pair<>(100L, 200L)); + + CapacityVO capacity = backupManager.getBackupStorageUsedStats(zoneId); + + Assert.assertNotNull(capacity); + Assert.assertEquals(Optional.ofNullable(Long.valueOf(100)), Optional.ofNullable(capacity.getUsedCapacity())); + Assert.assertEquals(Optional.ofNullable(Long.valueOf(200)), Optional.ofNullable(capacity.getTotalCapacity())); + Assert.assertEquals(CapacityVO.CAPACITY_TYPE_BACKUP_STORAGE, capacity.getCapacityType()); + } } From 47b6cd1fd667d1d0db1b888b2e7f47f545750e35 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 11 Mar 2025 09:28:34 +0530 Subject: [PATCH 049/147] make capacity dasboard scrollable --- ui/src/views/dashboard/CapacityDashboard.vue | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ui/src/views/dashboard/CapacityDashboard.vue b/ui/src/views/dashboard/CapacityDashboard.vue index a1a28a2d1d91..b9b095a0a815 100644 --- a/ui/src/views/dashboard/CapacityDashboard.vue +++ b/ui/src/views/dashboard/CapacityDashboard.vue @@ -210,7 +210,7 @@ - + + + + +
+ + {{ $t('label.fetch.from.backup') }} + + + {{ $t('label.clear') }} + +
@@ -218,28 +226,31 @@ export default { }, updateNetworkData (name, key, value) { this.formRef.value.validate().then(() => { - this.$emit('handler-error', false) - const index = this.networks.findIndex(item => item.key === key) - if (index === -1) { - const networkItem = {} - networkItem.key = key - networkItem[name] = value - this.networks.push(networkItem) - this.$emit('update-network-config', this.networks) - return - } - - this.networks.filter((item, index) => { - if (item.key === key) { - this.networks[index][name] = value - } - }) + this.updateNetworkDataWithoutValidation(name, key, value) this.$emit('update-network-config', this.networks) }).catch((error) => { this.formRef.value.scrollToField(error.errorFields[0].name) this.$emit('handler-error', true) }) }, + updateNetworkDataWithoutValidation (name, key, value) { + this.$emit('handler-error', false) + const index = this.networks.findIndex(item => item.key === key) + if (index === -1) { + const networkItem = {} + networkItem.key = key + networkItem[name] = value + this.networks.push(networkItem) + this.$emit('update-network-config', this.networks) + return + } + + this.networks.filter((item, index) => { + if (item.key === key) { + this.networks[index][name] = value + } + }) + }, removeItem (id) { this.dataItems = this.dataItems.filter(item => item.id !== id) if (this.selectedRowKeys.includes(id)) { @@ -249,6 +260,59 @@ export default { } } }, + handleFetchIpAddresses () { + if (!this.preFillContent.networkids) { + return + } + if (!this.preFillContent.ipAddresses && !this.preFillContent.macAddresses) { + return + } + + const networkIds = this.dataItems.map(item => item.id) + this.dataItems.forEach(record => { + const ipAddressKey = 'ipAddress' + record.id + const macAddressKey = 'macAddress' + record.id + this.form[ipAddressKey] = '' + this.form[macAddressKey] = '' + }) + + networkIds.forEach((networkId) => { + const backupIndex = this.preFillContent.networkids.findIndex(id => id === networkId) + if (backupIndex !== -1) { + if (this.preFillContent.ipAddresses && backupIndex < this.preFillContent.ipAddresses.length) { + const ipAddress = this.preFillContent.ipAddresses[backupIndex] + if (ipAddress) { + const ipAddressKey = 'ipAddress' + networkId + this.form[ipAddressKey] = ipAddress + this.updateNetworkDataWithoutValidation('ipAddress', networkId, ipAddress) + } + } + + if (this.preFillContent.macAddresses && backupIndex < this.preFillContent.macAddresses.length) { + const macAddress = this.preFillContent.macAddresses[backupIndex] + if (macAddress) { + const macAddressKey = 'macAddress' + networkId + this.form[macAddressKey] = macAddress + this.updateNetworkDataWithoutValidation('macAddress', networkId, macAddress) + } + } + } + }) + }, + handleClearIpAddresses () { + this.dataItems.forEach(record => { + const ipAddressKey = 'ipAddress' + record.id + const macAddressKey = 'macAddress' + record.id + this.form[ipAddressKey] = '' + this.form[macAddressKey] = '' + + this.updateNetworkDataWithoutValidation('ipAddress', record.id, '') + this.updateNetworkDataWithoutValidation('macAddress', record.id, '') + }) + + this.networks = [] + this.$emit('update-network-config', this.networks) + }, async validatorMacAddress (rule, value) { if (!value || value === '') { return Promise.resolve() diff --git a/ui/src/views/storage/CreateVMFromBackup.vue b/ui/src/views/storage/CreateVMFromBackup.vue index 84c5938131d1..c82726facf79 100644 --- a/ui/src/views/storage/CreateVMFromBackup.vue +++ b/ui/src/views/storage/CreateVMFromBackup.vue @@ -25,6 +25,17 @@
+ + + + + + + + +
{{ $t('label.cancel') }} @@ -50,6 +61,7 @@ import { h } from 'vue' import { api } from '@/api' import { Button } from 'ant-design-vue' +import TooltipLabel from '@/components/widgets/TooltipLabel' import eventBus from '@/config/eventBus' import DeployVMFromBackup from '@/components/view/DeployVMFromBackup' @@ -57,7 +69,8 @@ import DeployVMFromBackup from '@/components/view/DeployVMFromBackup' export default { name: 'CreateVMFromBackup', components: { - DeployVMFromBackup + DeployVMFromBackup, + TooltipLabel }, data () { return { @@ -65,7 +78,11 @@ export default { dataPreFill: {}, vmdetails: {}, serviceOffering: {}, - loading: true + loading: true, + form: { + name: '', + preserveIpAddresses: false + } } }, props: { @@ -108,7 +125,10 @@ export default { this.dataPreFill.backupid = this.resource.id this.dataPreFill.computeofferingid = this.vmdetails.serviceofferingid this.dataPreFill.templateid = this.vmdetails.templateid + this.dataPreFill.allowIpAddressesFetch = !this.resource.virtualmachineid this.dataPreFill.networkids = (this.vmdetails.networkids || '').split(',') + this.dataPreFill.ipAddresses = (this.vmdetails.ipaddresses || '').split(',') + this.dataPreFill.macAddresses = (this.vmdetails.macaddresses || '').split(',') this.diskofferingids = (this.vmdetails.diskofferingids || '').split(',') this.miniops = (this.vmdetails.miniops || '').split(',').map(item => item === 'null' ? '' : item) this.maxiops = (this.vmdetails.maxiops || '').split(',').map(item => item === 'null' ? '' : item) @@ -156,6 +176,15 @@ export default { args.zoneid = this.resource.zoneid args.backupid = this.resource.id + if (this.form.name) { + args.name = this.form.name + args.displayname = this.form.name + } + + if (this.form.preserveIpAddresses) { + args.preserveip = this.form.preserveIpAddresses + } + const title = this.$t('label.create.instance.from.backup') const description = '' const password = this.$t('label.password') From 555eed8c59d3ef92769e631fe6e4d2476167e145 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 29 Apr 2025 14:26:20 +0530 Subject: [PATCH 077/147] fix pre-commit failure in CreateVMFromBackupCmd --- .../cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java index 1068f800d2ac..cec0cfafa63b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java @@ -72,7 +72,7 @@ public class CreateVMFromBackupCmd extends BaseDeployVMCmd { @Parameter(name = ApiConstants.TEMPLATE_ID, type = CommandType.UUID, entityType = TemplateResponse.class, description = "the ID of the template for the virtual machine") private Long templateId; - @Parameter(name = ApiConstants.PRESERVE_IP, type = CommandType.BOOLEAN, description = "Use the same IP/MAC addresses as stored in the backup metadata. Works only if the original Instance is deleted and the IP/MAC address is avaialable.") + @Parameter(name = ApiConstants.PRESERVE_IP, type = CommandType.BOOLEAN, description = "Use the same IP/MAC addresses as stored in the backup metadata. Works only if the original Instance is deleted and the IP/MAC address is available.") private Boolean preserveIp; ///////////////////////////////////////////////////// From a34c78a7a4e4bc33437bb000e3876f8493fea2b1 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 1 May 2025 16:14:14 +0530 Subject: [PATCH 078/147] Added more UT to UserVmManagerImplTest --- .../com/cloud/vm/UserVmManagerImplTest.java | 235 +++++++++++++++++- 1 file changed, 232 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index 66218c3cb56f..dd120b289717 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -23,11 +23,13 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -45,11 +47,14 @@ import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; +import org.apache.cloudstack.api.command.admin.vm.ExpungeVMCmd; import org.apache.cloudstack.api.command.user.vm.CreateVMFromBackupCmd; import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; import org.apache.cloudstack.api.command.user.vm.DeployVnfApplianceCmd; +import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; import org.apache.cloudstack.api.command.user.vm.ResetVMSSHKeyCmd; import org.apache.cloudstack.api.command.user.vm.ResetVMUserDataCmd; import org.apache.cloudstack.api.command.user.vm.RestoreVMCmd; @@ -3150,7 +3155,6 @@ public void executeStepsToChangeOwnershipOfVmTestResourceCountRunningVmsOnlyEnab public void testAllocateVMFromBackupUsingCmdValues() throws InsufficientCapacityException, ResourceAllocationException, ResourceUnavailableException { Long backupId = 4L; Long rootDiskOfferingId = 5L; - Long networkId = 6L; CreateVMFromBackupCmd cmd = new CreateVMFromBackupCmd(); cmd._accountService = accountService; @@ -3212,8 +3216,6 @@ public void testAllocateVMFromBackupUsingCmdValues() throws InsufficientCapacity public void testAllocateVMFromBackupUsingBackupValues() throws InsufficientCapacityException, ResourceAllocationException, ResourceUnavailableException { Long backupId = 5L; Long rootDiskOfferingId = 6L; - Long network1Id = 7L; - Long network2Id = 8L; CreateVMFromBackupCmd cmd = mock(CreateVMFromBackupCmd.class); when(cmd.getZoneId()).thenReturn(zoneId); @@ -3316,4 +3318,231 @@ public void testResetVMSSHKey() throws ResourceUnavailableException, Insufficien "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAr..."); Assert.assertEquals(details.get(VmDetailConstants.SSH_KEY_PAIR_NAMES), "keypair1,keypair2"); } + + @Test + public void testAllocateVMFromBackupUsingCmdValuesWithISO() throws InsufficientCapacityException, ResourceAllocationException, ResourceUnavailableException { + Long backupId = 4L; + Long rootDiskOfferingId = 5L; + Long isoId = 7L; + + CreateVMFromBackupCmd cmd = new CreateVMFromBackupCmd(); + cmd._accountService = accountService; + cmd._entityMgr = entityManager; + when(accountService.finalyzeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); + when(accountService.getActiveAccountById(accountId)).thenReturn(account); + + ReflectionTestUtils.setField(cmd, "serviceOfferingId", serviceOfferingId); + ReflectionTestUtils.setField(cmd, "templateId", isoId); + ReflectionTestUtils.setField(cmd, "backupId", backupId); + ReflectionTestUtils.setField(cmd, "zoneId", zoneId); + ReflectionTestUtils.setField(cmd, "diskOfferingId", rootDiskOfferingId); + + ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); + when(serviceOffering.getDiskOfferingId()).thenReturn(rootDiskOfferingId); + DiskOfferingVO rootDiskOffering = mock(DiskOfferingVO.class); + when(_serviceOfferingDao.findById(serviceOfferingId)).thenReturn(serviceOffering); + when(diskOfferingDao.findById(rootDiskOfferingId)).thenReturn(rootDiskOffering); + when(entityManager.findById(DiskOffering.class, rootDiskOfferingId)).thenReturn(rootDiskOffering); + + Map diskDetails = new HashMap<>(); + diskDetails.put(ApiConstants.DISK_OFFERING_ID, "disk-offering-uuid"); + diskDetails.put(ApiConstants.DEVICE_ID, "1"); + diskDetails.put(ApiConstants.SIZE, "5"); + diskDetails.put(ApiConstants.MIN_IOPS, "1000"); + diskDetails.put(ApiConstants.MAX_IOPS, "5000"); + Map> disksDetails = new HashMap<>(); + disksDetails.put(0, diskDetails); + ReflectionTestUtils.setField(cmd, "dataDisksDetails", disksDetails); + DiskOffering diskOffering = mock(DiskOffering.class); + when(diskOffering.isCustomized()).thenReturn(true); + when(diskOffering.isCustomizedIops()).thenReturn(true); + when(entityManager.findByUuid(DiskOffering.class, "disk-offering-uuid")).thenReturn(diskOffering); + + BackupVO backup = mock(BackupVO.class); + when(backup.getZoneId()).thenReturn(zoneId); + when(backup.getVmId()).thenReturn(vmId); + when(backupDao.findById(backupId)).thenReturn(backup); + + UserVmVO userVmVO = new UserVmVO(); + userVmVO.setTemplateId(isoId); + when(userVmDao.findByIdIncludingRemoved(vmId)).thenReturn(userVmVO); + VMTemplateVO iso = mock(VMTemplateVO.class); + when(iso.getFormat()).thenReturn(Storage.ImageFormat.ISO); + when(templateDao.findById(isoId)).thenReturn(iso); + + Mockito.doReturn(userVmVoMock).when(userVmManagerImpl).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), + any(), any(), any(), any(), eq(false), any(), any()); + + UserVm result = userVmManagerImpl.allocateVMFromBackup(cmd); + + assertNotNull(result); + Mockito.verify(backupDao).findById(backupId); + Mockito.verify(userVmManagerImpl).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), + any(), any(), any(), any(), eq(false), any(), any()); + } + + @Test + public void testAllocateVMFromBackupUsingBackupValuesWithISO() throws InsufficientCapacityException, ResourceAllocationException, ResourceUnavailableException { + Long backupId = 5L; + Long rootDiskOfferingId = 6L; + Long isoId = 9L; + + CreateVMFromBackupCmd cmd = mock(CreateVMFromBackupCmd.class); + when(cmd.getZoneId()).thenReturn(zoneId); + when(cmd.getBackupId()).thenReturn(backupId); + when(cmd.getEntityOwnerId()).thenReturn(accountId); + when(cmd.getServiceOfferingId()).thenReturn(null); + when(cmd.getDiskOfferingId()).thenReturn(null); + when(cmd.getTemplateId()).thenReturn(null); + when(cmd.getNetworkIds()).thenReturn(null); + when(cmd.getIpToNetworkMap()).thenReturn(null); + when(cmd.getDataDiskOfferingsInfo()).thenReturn(null); + + Account owner = mock(Account.class); + when(accountService.getActiveAccountById(accountId)).thenReturn(owner); + + DataCenterVO zone = mock(DataCenterVO.class); + when(_dcDao.findById(zoneId)).thenReturn(zone); + + BackupVO backup = mock(BackupVO.class); + when(backup.getZoneId()).thenReturn(zoneId); + when(backup.getVmId()).thenReturn(vmId); + when(backup.getDetail(ApiConstants.SERVICE_OFFERING_ID)).thenReturn("service-offering-uuid"); + when(backupDao.findById(backupId)).thenReturn(backup); + + UserVmVO userVmVO = new UserVmVO(); + userVmVO.setTemplateId(isoId); + when(userVmDao.findByIdIncludingRemoved(vmId)).thenReturn(userVmVO); + VMTemplateVO iso = mock(VMTemplateVO.class); + when(iso.getFormat()).thenReturn(Storage.ImageFormat.ISO); + when(templateDao.findById(isoId)).thenReturn(iso); + + ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); + when(serviceOffering.getDiskOfferingId()).thenReturn(rootDiskOfferingId); + DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + when(backup.getDetail(ApiConstants.SERVICE_OFFERING_ID)).thenReturn("service-offering-uuid"); + when(_serviceOfferingDao.findByUuid("service-offering-uuid")).thenReturn(serviceOffering); + when(serviceOffering.getDiskOfferingId()).thenReturn(rootDiskOfferingId); + when(diskOfferingDao.findById(rootDiskOfferingId)).thenReturn(diskOffering); + + DiskOfferingInfo rootDiskOfferingInfo = new DiskOfferingInfo(diskOffering, 10L, 1000L, 2000L); + when(backupManager.getRootDiskOfferingInfoFromBackup(backup)).thenReturn(rootDiskOfferingInfo); + + NetworkVO network1 = mock(NetworkVO.class); + NetworkVO network2 = mock(NetworkVO.class); + when(backupManager.getDataDiskOfferingListFromBackup(backup)).thenReturn(List.of(new DiskOfferingInfo(diskOffering, 10L, 1000L, 2000L))); + + Mockito.doReturn(userVmVoMock).when(userVmManagerImpl).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), + any(), any(), any(), any(), eq(false), any(), any()); + + UserVm result = userVmManagerImpl.allocateVMFromBackup(cmd); + + assertNotNull(result); + Mockito.verify(userVmManagerImpl).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), + any(), any(), any(), any(), eq(false), any(), any()); + } + + @Test + public void testRestoreVMFromBackup() throws ResourceUnavailableException, InsufficientCapacityException, ResourceAllocationException { + Long backupId = 5L; + Long templateId = 6L; + + CreateVMFromBackupCmd cmd = mock(CreateVMFromBackupCmd.class); + when(cmd.getBackupId()).thenReturn(backupId); + when(cmd.getStartVm()).thenReturn(true); + when(cmd.getEntityId()).thenReturn(vmId); + + UserVmVO vm = mock(UserVmVO.class); + when(vm.getId()).thenReturn(vmId); + when(vm.getState()).thenReturn(VirtualMachine.State.Running); + when(vm.getTemplateId()).thenReturn(templateId); + + when(backupManager.restoreBackupToVM(backupId, vmId)).thenReturn(true); + + Map params = new HashMap<>(); + Pair> vmPair = new Pair<>(vm, params); + doReturn(vmPair).when(userVmManagerImpl).startVirtualMachine(anyLong(), isNull(), isNull(), isNull(), anyMap(), isNull()); + doReturn(vmPair).when(userVmManagerImpl).startVirtualMachine(anyLong(), isNull(), isNull(), anyLong(), anyMap(), isNull()); + when(userVmDao.findById(vmId)).thenReturn(vm); + when(templateDao.findByIdIncludingRemoved(templateId)).thenReturn(mock(VMTemplateVO.class)); + when(userVmManagerImpl.stopVirtualMachine(anyLong(), anyLong())).thenReturn(true); + + UserVm result = userVmManagerImpl.restoreVMFromBackup(cmd); + + assertNotNull(result); + assertEquals(vm, result); + Mockito.verify(backupManager).restoreBackupToVM(backupId, vmId); + } + + @Test + public void testDestroyVm() throws ResourceUnavailableException { + Long volumeId = 4L; + Long accountId = 5L; + Long userId = 6L; + boolean expunge = true; + + CallContext callContext = mock(CallContext.class); + Account callingAccount = mock(Account.class); + when(callingAccount.getId()).thenReturn(accountId); + when(callContext.getCallingAccount()).thenReturn(callingAccount); + when(callContext.getCallingUserId()).thenReturn(userId); + when(accountManager.isAdmin(callingAccount.getId())).thenReturn(true); + doNothing().when(accountManager).checkApiAccess(callingAccount, BaseCmd.getCommandNameByClass(ExpungeVMCmd.class)); + try (MockedStatic mockedCallContext = Mockito.mockStatic(CallContext.class)) { + mockedCallContext.when(CallContext::current).thenReturn(callContext); + + DestroyVMCmd cmd = mock(DestroyVMCmd.class); + when(cmd.getId()).thenReturn(vmId); + when(cmd.getExpunge()).thenReturn(expunge); + List volumeIds = List.of(volumeId); + when(cmd.getVolumeIds()).thenReturn(volumeIds); + + UserVmVO vm = mock(UserVmVO.class); + when(vm.getId()).thenReturn(vmId); + when(vm.getState()).thenReturn(VirtualMachine.State.Running); + when(vm.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(vm.getServiceOfferingId()).thenReturn(serviceOfferingId); + when(vm.getTemplateId()).thenReturn(templateId); + when(vm.getAccountId()).thenReturn(accountId); + when(vm.getUuid()).thenReturn("vm-uuid"); + when(vm.getUserVmType()).thenReturn("User"); + when(userVmDao.findById(vmId)).thenReturn(vm); + + when(accountDao.findById(accountId)).thenReturn(account); + + ServiceOfferingVO offering = mock(ServiceOfferingVO.class); + when(offering.getCpu()).thenReturn(2); + when(offering.getRamSize()).thenReturn(2048); + when(_serviceOfferingDao.findByIdIncludingRemoved(serviceOfferingId)).thenReturn(offering); + + VMTemplateVO template = mock(VMTemplateVO.class); + when(templateDao.findByIdIncludingRemoved(templateId)).thenReturn(template); + + VolumeVO vol = Mockito.mock(VolumeVO.class); + when(vol.getInstanceId()).thenReturn(vmId); + when(vol.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeDaoMock.findById(volumeId)).thenReturn(vol); + + List dataVolumes = new ArrayList<>(); + when(volumeDaoMock.findByInstanceAndType(vmId, Volume.Type.DATADISK)).thenReturn(dataVolumes); + + doReturn(vm).when(userVmManagerImpl).stopVirtualMachine(anyLong(), anyBoolean()); + doReturn(vm).when(userVmManagerImpl).destroyVm(vmId, expunge); + doReturn(true).when(userVmManagerImpl).expunge(vm); + + try (MockedStatic mockedUsageEventUtils = Mockito.mockStatic(UsageEventUtils.class)) { + + UserVm result = userVmManagerImpl.destroyVm(cmd); + + assertNotNull(result); + assertEquals(vm, result); + Mockito.verify(userVmManagerImpl).stopVirtualMachine(vmId, false); + Mockito.verify(backupManager).checkAndRemoveBackupOfferingBeforeExpunge(vm); + } + } + } } From a298c33d155380d2b36739474a0f4787325971b1 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 1 May 2025 17:10:53 +0530 Subject: [PATCH 079/147] Fix errors in testDestroyVM in UserVmManagerImplTest --- .../com/cloud/vm/UserVmManagerImplTest.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index dd120b289717..60922e308aa7 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -46,6 +46,7 @@ import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; @@ -103,6 +104,7 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.network.NetworkModel; +import com.cloud.network.as.AutoScaleManager; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.element.UserDataServiceProvider; @@ -144,6 +146,7 @@ import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.db.EntityManager; +import com.cloud.utils.db.UUIDManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.ExceptionProxyObject; import com.cloud.vm.dao.NicDao; @@ -391,6 +394,12 @@ public class UserVmManagerImplTest { @Mock private VMInstanceVO vmInstanceMock; + @Mock + private AutoScaleManager autoScaleManager; + + @Mock + private UUIDManager uuidMgr; + private static final long vmId = 1l; private static final long zoneId = 2L; private static final long accountId = 3L; @@ -3485,15 +3494,16 @@ public void testDestroyVm() throws ResourceUnavailableException { Long userId = 6L; boolean expunge = true; + ReflectionTestUtils.setField(userVmManagerImpl, "_uuidMgr", uuidMgr); CallContext callContext = mock(CallContext.class); Account callingAccount = mock(Account.class); when(callingAccount.getId()).thenReturn(accountId); when(callContext.getCallingAccount()).thenReturn(callingAccount); - when(callContext.getCallingUserId()).thenReturn(userId); when(accountManager.isAdmin(callingAccount.getId())).thenReturn(true); doNothing().when(accountManager).checkApiAccess(callingAccount, BaseCmd.getCommandNameByClass(ExpungeVMCmd.class)); try (MockedStatic mockedCallContext = Mockito.mockStatic(CallContext.class)) { mockedCallContext.when(CallContext::current).thenReturn(callContext); + mockedCallContext.when(() -> CallContext.register(callContext, ApiCommandResourceType.Volume)).thenReturn(callContext); DestroyVMCmd cmd = mock(DestroyVMCmd.class); when(cmd.getId()).thenReturn(vmId); @@ -3504,32 +3514,21 @@ public void testDestroyVm() throws ResourceUnavailableException { UserVmVO vm = mock(UserVmVO.class); when(vm.getId()).thenReturn(vmId); when(vm.getState()).thenReturn(VirtualMachine.State.Running); - when(vm.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); - when(vm.getServiceOfferingId()).thenReturn(serviceOfferingId); - when(vm.getTemplateId()).thenReturn(templateId); - when(vm.getAccountId()).thenReturn(accountId); when(vm.getUuid()).thenReturn("vm-uuid"); when(vm.getUserVmType()).thenReturn("User"); when(userVmDao.findById(vmId)).thenReturn(vm); - when(accountDao.findById(accountId)).thenReturn(account); - - ServiceOfferingVO offering = mock(ServiceOfferingVO.class); - when(offering.getCpu()).thenReturn(2); - when(offering.getRamSize()).thenReturn(2048); - when(_serviceOfferingDao.findByIdIncludingRemoved(serviceOfferingId)).thenReturn(offering); - - VMTemplateVO template = mock(VMTemplateVO.class); - when(templateDao.findByIdIncludingRemoved(templateId)).thenReturn(template); - VolumeVO vol = Mockito.mock(VolumeVO.class); when(vol.getInstanceId()).thenReturn(vmId); + when(vol.getId()).thenReturn(volumeId); when(vol.getVolumeType()).thenReturn(Volume.Type.DATADISK); when(volumeDaoMock.findById(volumeId)).thenReturn(vol); List dataVolumes = new ArrayList<>(); when(volumeDaoMock.findByInstanceAndType(vmId, Volume.Type.DATADISK)).thenReturn(dataVolumes); + when(volumeApiService.destroyVolume(volumeId, CallContext.current().getCallingAccount(), expunge, false)).thenReturn(vol); + doReturn(vm).when(userVmManagerImpl).stopVirtualMachine(anyLong(), anyBoolean()); doReturn(vm).when(userVmManagerImpl).destroyVm(vmId, expunge); doReturn(true).when(userVmManagerImpl).expunge(vm); From 062e343e0a3526a4c3fd9f2d1ce27daf822cdd5a Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 1 May 2025 18:08:37 +0530 Subject: [PATCH 080/147] Add more UT to BackupManagerTest --- .../cloudstack/backup/BackupManagerTest.java | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index 237abc903b9e..caf642b8e2a6 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -31,6 +31,10 @@ import com.cloud.exception.ResourceUnavailableException; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; +import com.cloud.network.Network; +import com.cloud.network.NetworkService; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; import com.cloud.offering.DiskOffering; import com.cloud.offering.DiskOfferingInfo; import com.cloud.service.ServiceOfferingVO; @@ -169,6 +173,12 @@ public class BackupManagerTest { @Mock HostDao hostDao; + @Mock + private NetworkDao networkDao; + + @Mock + private NetworkService networkService; + private AccountVO account; private UserVO user; @@ -1235,4 +1245,85 @@ public void testCheckAndRemoveBackupOfferingBeforeExpunge() { + "Backup Offering [id: 3, external id: backup-external-id] with 1 backups. Please, remove the backup offering " + "before proceeding to VM exclusion!", exception.getMessage()); } + + @Test + public void testGetIpToNetworkMapFromBackup() { + Long networkId1 = 1L; + Long networkId2 = 2L; + String networkUuid1 = "network-uuid-1"; + String networkUuid2 = "network-uuid-2"; + String ip1 = "10.1.1.1"; + String ip2 = "10.1.1.2"; + String ipv61 = "2001:db8::1"; + String ipv62 = "2001:db8::2"; + String mac1 = "00:11:22:33:44:55"; + String mac2 = "00:11:22:33:44:56"; + + // Test case 1: Missing network information + Backup backup1 = mock(Backup.class); + when(backup1.getDetail(ApiConstants.NETWORK_IDS)).thenReturn(null); + List networkIds1 = new ArrayList<>(); + try { + backupManager.getIpToNetworkMapFromBackup(backup1, true, networkIds1); + fail("Expected CloudRuntimeException for missing network information"); + } catch (CloudRuntimeException e) { + assertEquals("Backup doesn't contain network information. Please specify atleast one valid network while creating instance", e.getMessage()); + } + + // Test case 2: IP preservation enabled with IP information + Backup backup2 = mock(Backup.class); + when(backup2.getDetail(ApiConstants.NETWORK_IDS)).thenReturn(networkUuid1 + "," + networkUuid2); + when(backup2.getDetail(ApiConstants.IP_ADDRESSES)).thenReturn(ip1 + "," + ip2); + when(backup2.getDetail(ApiConstants.IP6_ADDRESSES)).thenReturn(ipv61 + "," + ipv62); + when(backup2.getDetail(ApiConstants.MAC_ADDRESSES)).thenReturn(mac1 + "," + mac2); + + NetworkVO network1 = mock(NetworkVO.class); + NetworkVO network2 = mock(NetworkVO.class); + when(networkDao.findByUuid(networkUuid1)).thenReturn(network1); + when(networkDao.findByUuid(networkUuid2)).thenReturn(network2); + when(network1.getId()).thenReturn(networkId1); + when(network2.getId()).thenReturn(networkId2); + + Network.IpAddresses ipAddresses1 = mock(Network.IpAddresses.class); + Network.IpAddresses ipAddresses2 = mock(Network.IpAddresses.class); + when(networkService.getIpAddressesFromIps(ip1, ipv61, mac1)).thenReturn(ipAddresses1); + when(networkService.getIpAddressesFromIps(ip2, ipv62, mac2)).thenReturn(ipAddresses2); + + List networkIds2 = new ArrayList<>(); + Map result2 = backupManager.getIpToNetworkMapFromBackup(backup2, true, networkIds2); + + assertEquals(2, result2.size()); + assertEquals(ipAddresses1, result2.get(networkId1)); + assertEquals(ipAddresses2, result2.get(networkId2)); + assertEquals(2, networkIds2.size()); + assertTrue(networkIds2.contains(networkId1)); + assertTrue(networkIds2.contains(networkId2)); + + // Test case 3: IP preservation enabled but missing IP information + Backup backup3 = mock(Backup.class); + when(backup3.getDetail(ApiConstants.NETWORK_IDS)).thenReturn(networkUuid1); + when(backup3.getDetail(ApiConstants.IP_ADDRESSES)).thenReturn(null); + when(backup3.getDetail(ApiConstants.IP6_ADDRESSES)).thenReturn(null); + when(backup3.getDetail(ApiConstants.MAC_ADDRESSES)).thenReturn(null); + + List networkIds3 = new ArrayList<>(); + Map result3 = backupManager.getIpToNetworkMapFromBackup(backup3, true, networkIds3); + + assertEquals(1, result3.size()); + assertNull(result3.get(networkId1)); + assertEquals(1, networkIds3.size()); + assertTrue(networkIds3.contains(networkId1)); + + // Test case 4: IP preservation disabled + Backup backup4 = mock(Backup.class); + when(backup4.getDetail(ApiConstants.NETWORK_IDS)).thenReturn(networkUuid1); + + List networkIds4 = new ArrayList<>(); + Map result4 = backupManager.getIpToNetworkMapFromBackup(backup4, false, networkIds4); + + assertEquals(1, result4.size()); + assertNull(result4.get(networkId1)); + assertEquals(1, networkIds4.size()); + assertTrue(networkIds4.contains(networkId1)); + } } From 359af7bd8c0d34b4b3d94062fdeec495fe7a8c3a Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Fri, 2 May 2025 10:15:56 +0530 Subject: [PATCH 081/147] Added UT to BackupManager, NetworkService and StorageManager --- .../cloudstack/backup/BackupManagerImpl.java | 18 +- .../cloud/network/NetworkServiceImplTest.java | 31 +++ .../cloud/storage/StorageManagerImplTest.java | 191 +++++++++++++++++- .../cloudstack/backup/BackupManagerTest.java | 52 +++++ 4 files changed, 280 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 7a5dea9aa32f..462d0c743e63 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -1326,9 +1326,13 @@ public boolean deleteBackup(final Long backupId, final Boolean forced) { if (result) { resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup); resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); - return backupDao.remove(backup.getId()); + if (backupDao.remove(backup.getId())) { + checkAndGenerateUsageForLastBackupDeletedAfterOfferingRemove(vm, backup); + return true; + } else { + return false; + } } - checkAndGenerateUsageForLastBackupDeletedAfterOfferingRemove(vm, backup); throw new CloudRuntimeException("Failed to delete the backup"); } @@ -1391,7 +1395,7 @@ private boolean attachVolumeToVM(Long zoneId, String restoredVolumeLocation, Lis } private void checkAndGenerateUsageForLastBackupDeletedAfterOfferingRemove(VirtualMachine vm, Backup backup) { - if (vm.getBackupOfferingId() != backup.getBackupOfferingId()) { + if (vm != null && vm.getBackupOfferingId() != backup.getBackupOfferingId()) { List backups = backupDao.listByVmIdAndOffering(vm.getDataCenterId(), vm.getId(), backup.getBackupOfferingId()); if (backups.size() == 0) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_DELETE_LAST_POST_OFFERING_REMOVE, vm.getAccountId(), @@ -1826,8 +1830,12 @@ private void processRemoveList(List removeList, VirtualMachine vm) { Backup backup = backupDao.findById(backupIdToRemove); resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup); resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); - backupDao.remove(backupIdToRemove); - checkAndGenerateUsageForLastBackupDeletedAfterOfferingRemove(vm, backup); + boolean result = backupDao.remove(backupIdToRemove); + if (result) { + checkAndGenerateUsageForLastBackupDeletedAfterOfferingRemove(vm, backup); + } else { + logger.error("Failed to remove backup db wentry ith ID: {} during sync backups", backupIdToRemove); + } } } diff --git a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java index 376b0bc767d2..46520d000273 100644 --- a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java @@ -1215,4 +1215,35 @@ public void checkAndSetSourceNatIp() { Assert.fail(e.getMessage()); } } + + @Test + public void testGetIpAddressesFromIps() { + // Test with valid IPv4, IPv6 and MAC address + Network.IpAddresses result = service.getIpAddressesFromIps("192.168.1.1", "2001:db8::1", "00:11:22:33:44:55"); + Assert.assertEquals("192.168.1.1", result.getIp4Address()); + Assert.assertEquals("2001:db8::1", result.getIp6Address()); + Assert.assertEquals("00:11:22:33:44:55", result.getMacAddress()); + + // Test with all null values + result = service.getIpAddressesFromIps(null, null, null); + Assert.assertNull(result.getIp4Address()); + Assert.assertNull(result.getIp6Address()); + Assert.assertNull(result.getMacAddress()); + + // Test with invalid MAC address (non-unicast) + try { + service.getIpAddressesFromIps(null, null, "ff:ff:ff:ff:ff:ff"); + Assert.fail("Expected InvalidParameterValueException for non-unicast MAC address"); + } catch (InvalidParameterValueException e) { + Assert.assertEquals("Mac address is not unicast: ff:ff:ff:ff:ff:ff", e.getMessage()); + } + + // Test with invalid MAC address (invalid format) + try { + service.getIpAddressesFromIps(null, null, "invalid-mac"); + Assert.fail("Expected InvalidParameterValueException for invalid MAC address format"); + } catch (InvalidParameterValueException e) { + Assert.assertEquals("Mac address is not valid: invalid-mac", e.getMessage()); + } + } } diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index 999bf85907be..ab74bbc6a080 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -22,19 +22,28 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.storage.ChangeStoragePoolScopeCmd; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.commons.collections.MapUtils; import org.junit.Assert; import org.junit.Test; @@ -50,7 +59,9 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.Command; import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.capacity.Capacity; import com.cloud.capacity.CapacityManager; +import com.cloud.capacity.CapacityVO; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; @@ -125,6 +136,15 @@ public class StorageManagerImplTest { @Mock private VMInstanceVO vmInstanceVOMock; + @Mock + protected ObjectStoreDao objectStoreDao; + + @Mock + DataStoreProviderManager dataStoreProviderMgr; + + @Mock + DataStoreManager dataStoreMgr; + @Test public void createLocalStoragePoolName() { String hostMockName = "host1"; @@ -239,7 +259,7 @@ public void testExtractUriParamsAsMapWithSolidFireUrl() { String sfUrl = "MVIP=1.2.3.4;SVIP=6.7.8.9;clusterAdminUsername=admin;" + "clusterAdminPassword=password;clusterDefaultMinIops=1000;" + "clusterDefaultMaxIops=2000;clusterDefaultBurstIopsPercentOfMaxIops=2"; - Map uriParams = storageManagerImpl.extractUriParamsAsMap(sfUrl); + Map uriParams = storageManagerImpl.extractUriParamsAsMap(sfUrl); Assert.assertTrue(MapUtils.isEmpty(uriParams)); } @@ -249,7 +269,7 @@ public void testExtractUriParamsAsMapWithNFSUrl() { String host = "HOST"; String path = "/PATH"; String sfUrl = String.format("%s://%s%s", scheme, host, path); - Map uriParams = storageManagerImpl.extractUriParamsAsMap(sfUrl); + Map uriParams = storageManagerImpl.extractUriParamsAsMap(sfUrl); Assert.assertTrue(MapUtils.isNotEmpty(uriParams)); Assert.assertEquals(scheme, uriParams.get("scheme")); Assert.assertEquals(host, uriParams.get("host")); @@ -408,7 +428,7 @@ public void testIsStoragePoolCompliantWithStoragePolicy() { } try { Mockito.doReturn(new com.cloud.agent.api.Answer( - Mockito.mock(CheckDataStoreStoragePolicyComplainceCommand.class))) + Mockito.mock(CheckDataStoreStoragePolicyComplainceCommand.class))) .when(storageManagerImpl).getCheckDatastorePolicyComplianceAnswer("policy", pool); Assert.assertTrue(storageManagerImpl.isStoragePoolCompliantWithStoragePolicy(1L, pool)); } catch (StorageUnavailableException e) { @@ -455,8 +475,7 @@ public void testGetCheckDatastorePolicyComplianceAnswerAgentException() throws S Mockito.when(policy.getPolicyId()).thenReturn("some"); Mockito.when(vsphereStoragePolicyDao.findById(Mockito.anyLong())) .thenReturn(policy); - Mockito.doReturn(new ArrayList<>(List.of(1L, 2L))) - .when(storageManagerImpl).getUpHostsInPool(Mockito.anyLong()); + Mockito.doReturn(new ArrayList<>(List.of(1L, 2L))).when(storageManagerImpl).getUpHostsInPool(Mockito.anyLong()); Mockito.when(hvGuruMgr.getGuruProcessedCommandTargetHost(Mockito.anyLong(), Mockito.any(CheckDataStoreStoragePolicyComplainceCommand.class))).thenReturn(1L); try { @@ -527,7 +546,7 @@ public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallation .thenReturn(new ArrayList<>()); //new installation storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations(); Mockito.verify(configurationDao, Mockito.never()) - .update(StorageManager.DataStoreDownloadFollowRedirects.key(),StorageManager.DataStoreDownloadFollowRedirects.defaultValue()); + .update(StorageManager.DataStoreDownloadFollowRedirects.key(), StorageManager.DataStoreDownloadFollowRedirects.defaultValue()); } @Test @@ -771,7 +790,7 @@ private Long testCheckPoolforSpaceForResizeSetup(StoragePoolVO pool, Long alloca Long zoneId = 2L; Long capacityBytes = (long) (allocatedSizeWithTemplate / Double.valueOf(CapacityManager.StorageAllocatedCapacityDisableThreshold.defaultValue()) - / Double.valueOf(CapacityManager.StorageOverprovisioningFactor.defaultValue())); + / Double.valueOf(CapacityManager.StorageOverprovisioningFactor.defaultValue())); Long maxAllocatedSizeForResize = (long) (capacityBytes * Double.valueOf(CapacityManager.StorageOverprovisioningFactor.defaultValue()) * Double.valueOf(CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.defaultValue())); @@ -894,4 +913,162 @@ public void testGetStoragePoolIopsStats_UsedIopsNegative() { Assert.assertEquals("Capacity IOPS should match pool's capacity IOPS", 1000L, result.first().longValue()); Assert.assertNull("Used IOPS should be null when usedIops <= 0", result.second()); } + + @Test + public void testGetObjectStorageUsedStats() { + Long zoneId = 1L; + List objectStores = new ArrayList<>(); + + ObjectStoreVO store1 = new ObjectStoreVO(); + store1.setAllocatedSize(1000L); + store1.setTotalSize(2000L); + objectStores.add(store1); + + ObjectStoreVO store2 = new ObjectStoreVO(); + store2.setAllocatedSize(2000L); + store2.setTotalSize(4000L); + objectStores.add(store2); + + ObjectStoreVO store3 = new ObjectStoreVO(); + store3.setAllocatedSize(null); + store3.setTotalSize(null); + objectStores.add(store3); + + Mockito.when(objectStoreDao.listObjectStores()).thenReturn(objectStores); + + CapacityVO result = storageManagerImpl.getObjectStorageUsedStats(zoneId); + + Assert.assertEquals(zoneId, result.getDataCenterId()); + Assert.assertEquals(Optional.of(3000L), Optional.of(result.getUsedCapacity())); // 1000 + 2000 + Assert.assertEquals(6000L, result.getTotalCapacity()); // 2000 + 4000 + Assert.assertEquals(Capacity.CAPACITY_TYPE_OBJECT_STORAGE, result.getCapacityType()); + Assert.assertNull(result.getPodId()); + Assert.assertNull(result.getClusterId()); + } + + @Test + public void testGetObjectStorageUsedStatsWithNullSizes() { + Long zoneId = 1L; + List objectStores = new ArrayList<>(); + + ObjectStoreVO store1 = new ObjectStoreVO(); + store1.setAllocatedSize(null); + store1.setTotalSize(null); + objectStores.add(store1); + + ObjectStoreVO store2 = new ObjectStoreVO(); + store2.setAllocatedSize(null); + store2.setTotalSize(null); + objectStores.add(store2); + + Mockito.when(objectStoreDao.listObjectStores()).thenReturn(objectStores); + + CapacityVO result = storageManagerImpl.getObjectStorageUsedStats(zoneId); + + Assert.assertEquals(zoneId, result.getDataCenterId()); + Assert.assertEquals(Optional.of(0L), Optional.of(result.getUsedCapacity())); + Assert.assertEquals(0L, result.getTotalCapacity()); + Assert.assertEquals(Capacity.CAPACITY_TYPE_OBJECT_STORAGE, result.getCapacityType()); + Assert.assertNull(result.getPodId()); + Assert.assertNull(result.getClusterId()); + } + + @Test + public void testDiscoverObjectStore() { + Long objectStoreId = 1L; + + String name = "test-store"; + String url = "http://10.1.1.33:80"; + Long size = 1000L; + String providerName = "test-provider"; + Map details = new HashMap<>(); + details.put("key1", "value1"); + + ObjectStoreVO objectStoreVO = new ObjectStoreVO(); + ReflectionTestUtils.setField(objectStoreVO, "id", objectStoreId); + objectStoreVO.setName(name); + objectStoreVO.setUrl(url); + objectStoreVO.setProviderName(providerName); + objectStoreVO.setTotalSize(size); + + DataStoreProvider storeProvider = Mockito.mock(DataStoreProvider.class); + DataStoreLifeCycle lifeCycle = Mockito.mock(DataStoreLifeCycle.class); + DataStore store = Mockito.mock(DataStore.class); + ObjectStore objectStore = Mockito.mock(ObjectStore.class); + + Mockito.when(dataStoreProviderMgr.getDataStoreProvider(providerName)).thenReturn(storeProvider); + Mockito.when(storeProvider.getDataStoreLifeCycle()).thenReturn(lifeCycle); + Mockito.when(lifeCycle.initialize(Mockito.any())).thenReturn(store); + Mockito.when(store.getId()).thenReturn(1L); + Mockito.when(dataStoreMgr.getDataStore(1L, DataStoreRole.Object)).thenReturn(null); + + ObjectStore result = storageManagerImpl.discoverObjectStore(name, url, size, providerName, details); + + Mockito.verify(dataStoreProviderMgr).getDataStoreProvider(providerName); + Mockito.verify(lifeCycle).initialize(Mockito.any()); + Mockito.verify(dataStoreMgr).getDataStore(1L, DataStoreRole.Object); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDiscoverObjectStoreInvalidProvider() { + // Setup + String name = "test-store"; + String url = "http://10.1.1.33:80"; + Long size = 1000L; + String providerName = "invalid-provider"; + Map details = new HashMap<>(); + + Mockito.when(dataStoreProviderMgr.getDataStoreProvider(providerName)).thenReturn(null); + + storageManagerImpl.discoverObjectStore(name, url, size, providerName, details); + } + + @Test(expected = IllegalArgumentException.class) + public void testDiscoverObjectStoreInvalidUrl() { + String name = "test-store"; + String url = "invalid-url"; + Long size = 1000L; + String providerName = "test-provider"; + Map details = new HashMap<>(); + + DataStoreProvider storeProvider = Mockito.mock(DataStoreProvider.class); + Mockito.when(dataStoreProviderMgr.getDataStoreProvider(providerName)).thenReturn(storeProvider); + + storageManagerImpl.discoverObjectStore(name, url, size, providerName, details); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDiscoverObjectStoreDuplicateUrl() { + String name = "test-store"; + String url = "http://10.1.1.33:80"; + Long size = 1000L; + String providerName = "test-provider"; + Map details = new HashMap<>(); + + DataStoreProvider storeProvider = Mockito.mock(DataStoreProvider.class); + ObjectStoreVO existingStore = new ObjectStoreVO(); + + Mockito.when(dataStoreProviderMgr.getDataStoreProvider(providerName)).thenReturn(storeProvider); + Mockito.when(objectStoreDao.findByUrl(url)).thenReturn(existingStore); + + storageManagerImpl.discoverObjectStore(name, url, size, providerName, details); + } + + @Test(expected = CloudRuntimeException.class) + public void testDiscoverObjectStoreInitializationFailure() { + String name = "test-store"; + String url = "http://10.1.1.33:80"; + Long size = 1000L; + String providerName = "test-provider"; + Map details = new HashMap<>(); + + DataStoreProvider storeProvider = Mockito.mock(DataStoreProvider.class); + DataStoreLifeCycle lifeCycle = Mockito.mock(DataStoreLifeCycle.class); + + Mockito.when(dataStoreProviderMgr.getDataStoreProvider(providerName)).thenReturn(storeProvider); + Mockito.when(storeProvider.getDataStoreLifeCycle()).thenReturn(lifeCycle); + Mockito.when(lifeCycle.initialize(Mockito.any())).thenThrow(new RuntimeException("Initialization failed")); + + storageManagerImpl.discoverObjectStore(name, url, size, providerName, details); + } } diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index caf642b8e2a6..c4a71829819b 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -1326,4 +1326,56 @@ public void testGetIpToNetworkMapFromBackup() { assertEquals(1, networkIds4.size()); assertTrue(networkIds4.contains(networkId1)); } + + @Test + public void testDeleteBackupVmNotFound() { + Long backupId = 1L; + Long vmId = 2L; + Long zoneId = 3L; + Long accountId = 4L; + Long backupOfferingId = 5L; + String vmHostName = "vm1"; + String vmUuid = "uuid1"; + String resourceName = "Backup-" + vmHostName + "-" + vmUuid; + + BackupVO backup = mock(BackupVO.class); + when(backup.getId()).thenReturn(backupId); + when(backup.getVmId()).thenReturn(vmId); + when(backup.getZoneId()).thenReturn(zoneId); + when(backup.getAccountId()).thenReturn(accountId); + when(backup.getBackupOfferingId()).thenReturn(backupOfferingId); + when(backup.getSize()).thenReturn(100L); + + overrideBackupFrameworkConfigValue(); + + VMInstanceVO vm = mock(VMInstanceVO.class); + when(vm.getId()).thenReturn(vmId); + when(vm.getAccountId()).thenReturn(accountId); + when(vm.getBackupOfferingId()).thenReturn(10L); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getHostName()).thenReturn(vmHostName); + when(vm.getUuid()).thenReturn(vmUuid); + when(backupDao.findByIdIncludingRemoved(backupId)).thenReturn(backup); + when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + when(backupDao.listByVmIdAndOffering(zoneId, vmId, backupOfferingId)).thenReturn(new ArrayList<>()); + + BackupOfferingVO offering = mock(BackupOfferingVO.class); + when(backupOfferingDao.findByIdIncludingRemoved(backupOfferingId)).thenReturn(offering); + + when(backupProvider.deleteBackup(backup, false)).thenReturn(true); + + when(backupDao.remove(backupId)).thenReturn(true); + + try (MockedStatic usageEventUtilsMocked = Mockito.mockStatic(UsageEventUtils.class)) { + boolean result = backupManager.deleteBackup(backupId, false); + + assertTrue(result); + verify(backupProvider).deleteBackup(backup, false); + verify(resourceLimitMgr).decrementResourceCount(accountId, Resource.ResourceType.backup); + verify(resourceLimitMgr).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, backup.getSize()); + verify(backupDao).remove(backupId); + usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_DELETE_LAST_POST_OFFERING_REMOVE, accountId, zoneId, vmId, resourceName, + backupOfferingId, null, null, Backup.class.getSimpleName(), vmUuid)); + } + } } From 5ff2e362c42ecf1a73c7d8d1ab52a9823ae1ad40 Mon Sep 17 00:00:00 2001 From: abh1sar Date: Mon, 5 May 2025 14:22:49 +0530 Subject: [PATCH 082/147] Fixed a bug in publishUsageEvent --- api/src/main/java/com/cloud/event/EventTypes.java | 2 +- .../java/org/apache/cloudstack/backup/BackupManagerImpl.java | 5 +++-- usage/src/main/java/com/cloud/usage/UsageManagerImpl.java | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index d397dd1b57ff..c719ea3df7b4 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -604,11 +604,11 @@ public class EventTypes { public static final String EVENT_VM_BACKUP_IMPORT_OFFERING = "BACKUP.IMPORT.OFFERING"; public static final String EVENT_VM_BACKUP_OFFERING_ASSIGN = "BACKUP.OFFERING.ASSIGN"; public static final String EVENT_VM_BACKUP_OFFERING_REMOVE = "BACKUP.OFFERING.REMOVE"; - public static final String EVENT_VM_BACKUP_DELETE_LAST_POST_OFFERING_REMOVE = "BACKUP.DELETE.LAST.POST.OFFERING.REMOVE"; public static final String EVENT_VM_BACKUP_CREATE = "BACKUP.CREATE"; public static final String EVENT_VM_BACKUP_RESTORE = "BACKUP.RESTORE"; public static final String EVENT_VM_BACKUP_RESTORE_TO_VM = "BACKUP.RESTORE.TO.VM"; public static final String EVENT_VM_BACKUP_DELETE = "BACKUP.DELETE"; + public static final String EVENT_VM_BACKUP_OFFERING_BACKUPS_DELETED = "BACKUP.OFFERING.BACKUPS.DEL"; public static final String EVENT_VM_BACKUP_RESTORE_VOLUME_TO_VM = "BACKUP.RESTORE.VOLUME.TO.VM"; public static final String EVENT_VM_BACKUP_SCHEDULE_CONFIGURE = "BACKUP.SCHEDULE.CONFIGURE"; public static final String EVENT_VM_BACKUP_SCHEDULE_DELETE = "BACKUP.SCHEDULE.DELETE"; diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 462d0c743e63..3b1e0e2bdb12 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -1395,10 +1395,11 @@ private boolean attachVolumeToVM(Long zoneId, String restoredVolumeLocation, Lis } private void checkAndGenerateUsageForLastBackupDeletedAfterOfferingRemove(VirtualMachine vm, Backup backup) { - if (vm != null && vm.getBackupOfferingId() != backup.getBackupOfferingId()) { + if (vm != null && + (vm.getBackupOfferingId() == null || vm.getBackupOfferingId() != backup.getBackupOfferingId())) { List backups = backupDao.listByVmIdAndOffering(vm.getDataCenterId(), vm.getId(), backup.getBackupOfferingId()); if (backups.size() == 0) { - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_DELETE_LAST_POST_OFFERING_REMOVE, vm.getAccountId(), + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_BACKUPS_DELETED, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(), backup.getBackupOfferingId(), null, null, Backup.class.getSimpleName(), vm.getUuid()); } diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 28d87ca9a06e..33b05f519c8c 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -1187,7 +1187,7 @@ private boolean isVmSnapshotOnPrimaryEvent(String eventType) { private boolean isBackupEvent(String eventType) { return eventType != null && ( eventType.equals(EventTypes.EVENT_VM_BACKUP_OFFERING_ASSIGN) || - eventType.equals(EventTypes.EVENT_VM_BACKUP_DELETE_LAST_POST_OFFERING_REMOVE) || + eventType.equals(EventTypes.EVENT_VM_BACKUP_OFFERING_BACKUPS_DELETED) || eventType.equals(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC)); } @@ -2142,7 +2142,7 @@ private void createBackupEvent(final UsageEventVO event) { if (EventTypes.EVENT_VM_BACKUP_OFFERING_ASSIGN.equals(event.getType())) { final UsageBackupVO backupVO = new UsageBackupVO(zoneId, accountId, domainId, vmId, backupOfferingId, created); usageBackupDao.persist(backupVO); - } else if (EventTypes.EVENT_VM_BACKUP_DELETE_LAST_POST_OFFERING_REMOVE.equals(event.getType())) { + } else if (EventTypes.EVENT_VM_BACKUP_OFFERING_BACKUPS_DELETED.equals(event.getType())) { usageBackupDao.removeUsage(accountId, vmId, backupOfferingId, event.getCreateDate()); } else if (EventTypes.EVENT_VM_BACKUP_USAGE_METRIC.equals(event.getType())) { usageBackupDao.updateMetrics(vmId, backupOfferingId, event.getSize(), event.getVirtualSize()); From 756af19c167a82b8dd6417c2d673ede263004083 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 5 May 2025 15:43:21 +0530 Subject: [PATCH 083/147] Fixed compilation error in BackupManagerTest --- .../java/org/apache/cloudstack/backup/BackupManagerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index c4a71829819b..415235c3e1c7 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -1374,7 +1374,7 @@ public void testDeleteBackupVmNotFound() { verify(resourceLimitMgr).decrementResourceCount(accountId, Resource.ResourceType.backup); verify(resourceLimitMgr).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, backup.getSize()); verify(backupDao).remove(backupId); - usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_DELETE_LAST_POST_OFFERING_REMOVE, accountId, zoneId, vmId, resourceName, + usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_BACKUPS_DELETED, accountId, zoneId, vmId, resourceName, backupOfferingId, null, null, Backup.class.getSimpleName(), vmUuid)); } } From 58d1067f80097a46d5869ac114969e15fdfae469 Mon Sep 17 00:00:00 2001 From: abh1sar Date: Tue, 6 May 2025 00:18:07 +0530 Subject: [PATCH 084/147] Add Nas backup and recovery integration test --- .../smoke/test_backup_recovery_dummy.py | 16 +- .../smoke/test_backup_recovery_nas.py | 219 ++++++++++++++++++ tools/marvin/marvin/lib/base.py | 123 +++++++--- 3 files changed, 317 insertions(+), 41 deletions(-) create mode 100644 test/integration/smoke/test_backup_recovery_nas.py diff --git a/test/integration/smoke/test_backup_recovery_dummy.py b/test/integration/smoke/test_backup_recovery_dummy.py index 6482bd97dfc1..3e2f16f8c686 100644 --- a/test/integration/smoke/test_backup_recovery_dummy.py +++ b/test/integration/smoke/test_backup_recovery_dummy.py @@ -18,7 +18,7 @@ from marvin.cloudstackTestCase import cloudstackTestCase from marvin.lib.utils import (cleanup_resources) -from marvin.lib.base import (Account, ServiceOffering, DiskOffering, VirtualMachine, BackupOffering, Configurations, Backup, Volume, createVMFromBackup) +from marvin.lib.base import (Account, ServiceOffering, DiskOffering, VirtualMachine, BackupOffering, Configurations, Backup, Volume) from marvin.lib.common import (get_domain, get_zone, get_template) from nose.plugins.attrib import attr from marvin.codes import FAILED @@ -45,15 +45,15 @@ def setUpClass(cls): cls._cleanup = [] # Check backup configuration values, set them to enable the dummy provider - backup_enabled_cfg = Configurations.list(cls.api_client, name='backup.framework.enabled', zoneid=cls.zone.id) - backup_provider_cfg = Configurations.list(cls.api_client, name='backup.framework.provider.plugin', zoneid=cls.zone.id) + backup_enabled_cfg = Configurations.list(cls.api_client, name='backup.framework.enabled') + backup_provider_cfg = Configurations.list(cls.api_client, name='backup.framework.provider.plugin') cls.backup_enabled = backup_enabled_cfg[0].value cls.backup_provider = backup_provider_cfg[0].value if cls.backup_enabled == "false": - Configurations.update(cls.api_client, 'backup.framework.enabled', value='true', zoneid=cls.zone.id) + cls.skipTest(cls, reason="Test can be run only if the config backup.framework.enabled is true") if cls.backup_provider != "dummy": - Configurations.update(cls.api_client, 'backup.framework.provider.plugin', value='dummy', zoneid=cls.zone.id) + Configurations.update(cls.api_client, 'backup.framework.provider.plugin', value='dummy') if cls.hypervisor.lower() != 'simulator': return @@ -81,10 +81,8 @@ def tearDownClass(cls): cleanup_resources(cls.api_client, cls._cleanup) # Restore original backup framework values values - if cls.backup_enabled == "false": - Configurations.update(cls.api_client, 'backup.framework.enabled', value=cls.backup_enabled, zoneid=cls.zone.id) if cls.backup_provider != "dummy": - Configurations.update(cls.api_client, 'backup.framework.provider.plugin', value=cls.backup_provider, zoneid=cls.zone.id) + Configurations.update(cls.api_client, 'backup.framework.provider.plugin', value=cls.backup_provider) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) @@ -186,6 +184,8 @@ def test_vm_backup_create_vm_from_backup(self): new_vm_name = "vm-from-backup1-" + str(int(time.time())) new_vm = Backup.createVMFromBackup( self.apiclient, + self.services["small"], + mode=self.services["mode"], backupid=backups[0].id, vmname=new_vm_name, accountname=self.account.name, diff --git a/test/integration/smoke/test_backup_recovery_nas.py b/test/integration/smoke/test_backup_recovery_nas.py new file mode 100644 index 000000000000..ea7f1112cbe5 --- /dev/null +++ b/test/integration/smoke/test_backup_recovery_nas.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.utils import (cleanup_resources) +from marvin.lib.base import (Account, ServiceOffering, DiskOffering, VirtualMachine, BackupOffering, + BackupRepository, Backup, Configurations, Volume, StoragePool) +from marvin.lib.common import (get_domain, get_zone, get_template) +from nose.plugins.attrib import attr +from marvin.codes import FAILED +import time + +class TestNASBackupAndRecovery(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + # Setup + + cls.testClient = super(TestNASBackupAndRecovery, cls).getClsTestClient() + cls.api_client = cls.testClient.getApiClient() + cls.services = cls.testClient.getParsedTestDataConfig() + cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests()) + cls.services["mode"] = cls.zone.networktype + cls.hypervisor = cls.testClient.getHypervisorInfo() + cls.domain = get_domain(cls.api_client) + cls.template = get_template(cls.api_client, cls.zone.id, cls.services["ostype"]) + if cls.template == FAILED: + assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = cls.template.id + cls._cleanup = [] + + if cls.hypervisor.lower() != 'kvm': + cls.skipTest(cls, reason="Test can be run only on KVM hypervisor") + + cls.storage_pool = StoragePool.list(cls.api_client)[0] + if cls.storage_pool.type.lower() != 'networkfilesystem': + cls.skipTest(cls, reason="Test can be run only if the primary storage is of type NFS") + + # Check backup configuration values, set them to enable the nas provider + backup_enabled_cfg = Configurations.list(cls.api_client, name='backup.framework.enabled') + backup_provider_cfg = Configurations.list(cls.api_client, name='backup.framework.provider.plugin') + cls.backup_enabled = backup_enabled_cfg[0].value + cls.backup_provider = backup_provider_cfg[0].value + + if cls.backup_enabled == "false": + cls.skipTest(cls, reason="Test can be run only if the config backup.framework.enabled is true") + if cls.backup_provider != "nas": + Configurations.update(cls.api_client, 'backup.framework.provider.plugin', value='nas') + + cls.account = Account.create(cls.api_client, cls.services["account"], domainid=cls.domain.id) + + cls._cleanup = [cls.account] + + # Create NAS backup repository and offering. Use the same directory as the storage pool + cls.backup_repository = BackupRepository.add(cls.api_client, zoneid=cls.zone.id, name="Nas", + address=cls.storage_pool.ipaddress + ":" + cls.storage_pool.path, + provider="nas", type="nfs",) + cls._cleanup.append(cls.backup_repository) + cls.provider_offerings = BackupOffering.listExternal(cls.api_client, cls.zone.id) + cls.backup_offering = BackupOffering.importExisting(cls.api_client, cls.zone.id, cls.provider_offerings[0].externalid, + cls.provider_offerings[0].name, cls.provider_offerings[0].description) + cls._cleanup.append(cls.backup_offering) + + cls.offering = ServiceOffering.create(cls.api_client,cls.services["service_offerings"]["small"]) + cls.diskoffering = DiskOffering.create(cls.api_client, cls.services["disk_offering"]) + cls._cleanup.extend([cls.offering, cls.diskoffering, cls.account]) + cls.vm = VirtualMachine.create(cls.api_client, cls.services["small"], accountid=cls.account.name, + domainid=cls.account.domainid, serviceofferingid=cls.offering.id, + diskofferingid=cls.diskoffering.id, mode=cls.services["mode"]) + + + @classmethod + def tearDownClass(cls): + try: + # Cleanup resources used + cleanup_resources(cls.api_client, cls._cleanup) + + if cls.backup_provider != "nas": + Configurations.update(cls.api_client, 'backup.framework.provider.plugin', value=cls.backup_provider) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + if self.hypervisor.lower() != 'kvm': + raise self.skipTest("Skipping test cases which must only run for Simulator") + self.cleanup = [] + + def tearDown(self): + try: + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + @attr(tags=["advanced", "backup"], required_hardware="true") + def test_vm_backup_lifecycle(self): + """ + Test VM backup lifecycle + """ + + # Verify there are no backups for the VM + backups = Backup.list(self.apiclient, self.vm.id) + self.assertEqual(backups, None, "There should not exist any backup for the VM") + + # Assign VM to offering and create ad-hoc backup + self.backup_offering.assignOffering(self.apiclient, self.vm.id) + Backup.create(self.apiclient, self.vm.id) + + # Verify backup is created for the VM + backups = Backup.list(self.apiclient, self.vm.id) + self.assertEqual(len(backups), 1, "There should exist only one backup for the VM") + backup = backups[0] + + # Delete backup + Backup.delete(self.apiclient, backup.id) + + # Verify backup is deleted + backups = Backup.list(self.apiclient, self.vm.id) + self.assertEqual(backups, None, "There should not exist any backup for the VM") + + # Remove VM from offering + self.backup_offering.removeOffering(self.apiclient, self.vm.id) + + @attr(tags=["advanced", "backup"], required_hardware="true") + def test_vm_backup_create_vm_from_backup(self): + """ + Test creating a new VM from a backup + """ + self.backup_offering.assignOffering(self.apiclient, self.vm.id) + + # Create a file and take backup + try: + ssh_client_vm = self.vm.get_ssh_client(reconnect=True) + ssh_client_vm.execute("touch test_backup_and_recovery.txt") + except Exception as err: + self.fail("SSH failed for Virtual machine: %s due to %s" % (self.vm.ipaddress, err)) + + time.sleep(5) + + Backup.create(self.apiclient, self.vm.id, "backup1") + Backup.create(self.apiclient, self.vm.id, "backup2") + + # Verify backup is created for the VM + backups = Backup.list(self.apiclient, self.vm.id) + self.assertEqual(len(backups), 2, "There should exist two backups for the VM") + + # Remove VM from offering + self.backup_offering.removeOffering(self.apiclient, self.vm.id) + + # Verify no. of backups after removing the backup offering + backups = Backup.list(self.apiclient, self.vm.id) + self.assertEqual(len(backups), 2, "There should exist two backups for the VM") + + # Create a new VM from first backup + new_vm_name = "vm-from-backup1-" + str(int(time.time())) + new_vm = Backup.createVMFromBackup( + self.apiclient, + self.services["small"], + mode=self.services["mode"], + backupid=backups[0].id, + vmname=new_vm_name, + accountname=self.account.name, + domainid=self.account.domainid, + zoneid=self.zone.id + ) + self.cleanup.append(new_vm) + + # Verify the new VM was created successfully + self.assertIsNotNone(new_vm, "Failed to create VM from backup") + self.assertEqual(new_vm.name, new_vm_name, "VM name does not match the requested name") + + # Verify the new VM is running + self.assertEqual(new_vm.state, "Running", "New VM should be in Running state") + + # Verify the new VM has the correct service offering + self.assertEqual(new_vm.serviceofferingid, self.offering.id, + "New VM should have the correct service offering") + + # Verify the new VM has the correct zone + self.assertEqual(new_vm.zoneid, self.zone.id, "New VM should be in the correct zone") + + # Verify the new VM has the correct number of volumes (ROOT + DATADISK) + volumes = Volume.list( + self.apiclient, + virtualmachineid=new_vm.id, + listall=True + ) + self.assertTrue(isinstance(volumes, list), "List volumes should return a valid list") + self.assertEqual(2, len(volumes), "The new VM should have 2 volumes (ROOT + DATADISK)") + + # Verify that the file is present in the Instance created from backup + try: + ssh_client_new_vm = new_vm.get_ssh_client(reconnect=True) + result = ssh_client_new_vm.execute("ls test_backup_and_recovery.txt") + self.assertEqual(result[0], "test_backup_and_recovery.txt", + "Instance created from Backup should have the same file as the backup.") + except Exception as err: + self.fail("SSH failed for Virtual machine: %s due to %s" % (self.vm.ipaddress, err)) + + # Delete backups + Backup.delete(self.apiclient, backups[0].id) + Backup.delete(self.apiclient, backups[1].id) diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 557434ea2ee3..348adafe58a3 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -516,6 +516,43 @@ def access_ssh_over_nat( virtual_machine.ssh_ip = nat_rule.ipaddress virtual_machine.public_ip = nat_rule.ipaddress + @classmethod + def program_ssh_access( + cls, apiclient, services, mode, networkids, virtual_machine, allow_egress=False, vpcid=None): + """ + Program SSH access to the VM + """ + # program ssh access over NAT via PF + retries = 5 + interval = 30 + while retries > 0: + try: + if mode.lower() == 'advanced': + cls.access_ssh_over_nat( + apiclient, + services, + virtual_machine, + allow_egress=allow_egress, + networkid=networkids[0] if networkids else None, + vpcid=vpcid) + elif mode.lower() == 'basic': + if virtual_machine.publicip is not None: + # EIP/ELB (netscaler) enabled zone + vm_ssh_ip = virtual_machine.publicip + else: + # regular basic zone with security group + vm_ssh_ip = virtual_machine.nic[0].ipaddress + virtual_machine.ssh_ip = vm_ssh_ip + virtual_machine.public_ip = vm_ssh_ip + break + except Exception as e: + if retries >= 0: + retries = retries - 1 + time.sleep(interval) + continue + raise Exception( + "The following exception appeared while programming ssh access - %s" % e) + @classmethod def create(cls, apiclient, services, templateid=None, accountid=None, domainid=None, zoneid=None, networkids=None, @@ -702,36 +739,7 @@ def create(cls, apiclient, services, templateid=None, accountid=None, virtual_machine.public_ip = virtual_machine.nic[0].ipaddress return VirtualMachine(virtual_machine.__dict__, services) - # program ssh access over NAT via PF - retries = 5 - interval = 30 - while retries > 0: - time.sleep(interval) - try: - if mode.lower() == 'advanced': - cls.access_ssh_over_nat( - apiclient, - services, - virtual_machine, - allow_egress=allow_egress, - networkid=cmd.networkids[0] if cmd.networkids else None, - vpcid=vpcid) - elif mode.lower() == 'basic': - if virtual_machine.publicip is not None: - # EIP/ELB (netscaler) enabled zone - vm_ssh_ip = virtual_machine.publicip - else: - # regular basic zone with security group - vm_ssh_ip = virtual_machine.nic[0].ipaddress - virtual_machine.ssh_ip = vm_ssh_ip - virtual_machine.public_ip = vm_ssh_ip - break - except Exception as e: - if retries >= 0: - retries = retries - 1 - continue - raise Exception( - "The following exception appeared while programming ssh access - %s" % e) + cls.program_ssh_access(apiclient, services, mode, cmd.networkids, virtual_machine, allow_egress, vpcid) return VirtualMachine(virtual_machine.__dict__, services) @@ -6113,11 +6121,13 @@ def __init__(self, items): self.__dict__.update(items) @classmethod - def create(self, apiclient, vmid): + def create(cls, apiclient, vmid, name=None): """Create VM backup""" cmd = createBackup.createBackupCmd() cmd.virtualmachineid = vmid + if name: + cmd.name = name return Backup(apiclient.createBackup(cmd).__dict__) @classmethod @@ -6131,11 +6141,12 @@ def delete(self, apiclient, id, forced=None): return (apiclient.deleteBackup(cmd)) @classmethod - def list(self, apiclient, vmid): + def list(self, apiclient, vmid=None): """List VM backups""" cmd = listBackups.listBackupsCmd() - cmd.virtualmachineid = vmid + if vmid: + cmd.virtualmachineid = vmid cmd.listall = True return (apiclient.listBackups(cmd)) @@ -6157,6 +6168,21 @@ def restoreVolumeFromBackupAndAttachToVM(self, apiclient, backupid, volumeid, vi cmd.virtualmachineid = virtualmachineid return (apiclient.restoreVolumeFromBackupAndAttachToVM(cmd)) + @classmethod + def createVMFromBackup(cls, apiclient, services, mode, backupid, accountname, domainid, zoneid, vmname=None): + """Create new VM from backup + """ + cmd = createVMFromBackup.createVMFromBackupCmd() + cmd.backupid = backupid + cmd.account = accountname + cmd.domainid = domainid + cmd.zoneid = zoneid + if vmname: + cmd.name = vmname + response = apiclient.createVMFromBackup(cmd) + virtual_machine = VirtualMachine(response.__dict__, []) + VirtualMachine.program_ssh_access(apiclient, services, mode, cmd.networkids, virtual_machine) + return virtual_machine class BackupSchedule: @@ -6199,6 +6225,37 @@ def update(self, apiclient, vmid, **kwargs): return (apiclient.updateBackupSchedule(cmd)) +class BackupRepository: + + def __init__(self, items): + self.__dict__.update(items) + + @classmethod + def add(cls, apiclient, zoneid, name, address, provider, type): + """Add backup repository""" + + cmd = addBackupRepository.addBackupRepositoryCmd() + cmd.zoneid = zoneid + cmd.name = name + cmd.address = address + cmd.provider = provider + cmd.type = type + response = apiclient.addBackupRepository(cmd) + return BackupRepository(response.__dict__) + + def delete(self, apiclient): + """Delete backup repository""" + + cmd = deleteBackupRepository.deleteBackupRepositoryCmd() + cmd.id = self.id + return (apiclient.deleteBackupRepository(cmd)) + + def list(self, apiclient): + """List backup repository""" + + cmd = listBackupRepositories.listBackupRepositoriesCmd() + return (apiclient.listBackupRepository(cmd)) + class ProjectRole: def __init__(self, items): From 4ef398b6900adea08f990072650a24670aaf560b Mon Sep 17 00:00:00 2001 From: abh1sar Date: Tue, 6 May 2025 01:31:50 +0530 Subject: [PATCH 085/147] Disallow offering delete if it is not assigned to any vm, but it has backups --- .../java/org/apache/cloudstack/backup/BackupManagerImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 3b1e0e2bdb12..2684b4362828 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -311,6 +311,10 @@ public boolean deleteBackupOffering(final Long offeringId) { throw new CloudRuntimeException("Could not find a backup offering with id: " + offeringId); } + if (backupDao.listByOfferingId(offering.getId()).size() > 0) { + throw new CloudRuntimeException("Backup Offering cannot be removed as it has backups associated with it."); + } + if (vmInstanceDao.listByZoneAndBackupOffering(offering.getZoneId(), offering.getId()).size() > 0) { throw new CloudRuntimeException("Backup offering is assigned to VMs, remove the assignment(s) in order to remove the offering."); } From 8c881ef3da032ed12d5d99fec30e4af2514b24aa Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 6 May 2025 10:08:53 +0530 Subject: [PATCH 086/147] Add backup offering in the list view and search filter in list backups --- .../command/user/backup/ListBackupsCmd.java | 12 +++++++ .../cloudstack/backup/BackupManagerImpl.java | 6 ++++ ui/src/components/view/SearchView.vue | 35 ++++++++++++++++++- ui/src/config/section/storage.js | 6 +++- 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupsCmd.java index ec0f5e28f433..f65f579fca7e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupsCmd.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupOfferingResponse; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UserVmResponse; @@ -75,6 +76,13 @@ public class ListBackupsCmd extends BaseListProjectAndAccountResourcesCmd { description = "list backups by zone id") private Long zoneId; + @Parameter(name = ApiConstants.BACKUP_OFFERING_ID, + type = CommandType.UUID, + entityType = BackupOfferingResponse.class, + since = "4.21.0", + description = "list backups by backup offering") + private Long backupOfferingId; + @Parameter(name = ApiConstants.LIST_VM_DETAILS, type = CommandType.BOOLEAN, since = "4.21.0", @@ -93,6 +101,10 @@ public Long getVmId() { return vmId; } + public Long getBackupOfferingId() { + return backupOfferingId; + } + public Long getZoneId() { return zoneId; } diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 3b1e0e2bdb12..6e3fca4df8b3 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -756,6 +756,7 @@ public Pair, Integer> listBackups(final ListBackupsCmd cmd) { final Long id = cmd.getId(); final Long vmId = cmd.getVmId(); final Long zoneId = cmd.getZoneId(); + final Long backupOfferingId = cmd.getBackupOfferingId(); final Account caller = CallContext.current().getCallingAccount(); final String keyword = cmd.getKeyword(); List permittedAccounts = new ArrayList(); @@ -782,6 +783,7 @@ public Pair, Integer> listBackups(final ListBackupsCmd cmd) { sb.and("idIN", sb.entity().getId(), SearchCriteria.Op.IN); sb.and("vmId", sb.entity().getVmId(), SearchCriteria.Op.EQ); sb.and("zoneId", sb.entity().getZoneId(), SearchCriteria.Op.EQ); + sb.and("backupOfferingId", sb.entity().getBackupOfferingId(), SearchCriteria.Op.EQ); if (keyword != null) { SearchBuilder vmSearch = vmInstanceDao.createSearchBuilder(); @@ -805,6 +807,10 @@ public Pair, Integer> listBackups(final ListBackupsCmd cmd) { sc.setParameters("zoneId", zoneId); } + if (backupOfferingId != null) { + sc.setParameters("backupOfferingId", backupOfferingId); + } + if (keyword != null) { sc.setJoinParameters("vmSearch", "name", "%" + keyword + "%"); } diff --git a/ui/src/components/view/SearchView.vue b/ui/src/components/view/SearchView.vue index f47c9e783bb8..e5ec87b5103c 100644 --- a/ui/src/components/view/SearchView.vue +++ b/ui/src/components/view/SearchView.vue @@ -305,11 +305,14 @@ export default { if (item === 'isencrypted' && !('listVolumes' in this.$store.getters.apis)) { return true } + if (item === 'backupofferingid' && !('listBackupOfferings' in this.$store.getters.apis)) { + return true + } if (['zoneid', 'domainid', 'imagestoreid', 'storageid', 'state', 'account', 'hypervisor', 'level', 'clusterid', 'podid', 'groupid', 'entitytype', 'accounttype', 'systemvmtype', 'scope', 'provider', 'type', 'scope', 'managementserverid', 'serviceofferingid', 'diskofferingid', 'networkid', 'usagetype', 'restartrequired', - 'displaynetwork', 'guestiptype', 'usersource', 'arch'].includes(item) + 'displaynetwork', 'guestiptype', 'usersource', 'arch', 'backupofferingid'].includes(item) ) { type = 'list' } else if (item === 'tags') { @@ -470,6 +473,7 @@ export default { let networkIndex = -1 let usageTypeIndex = -1 let volumeIndex = -1 + let backupOfferingIndex = -1 if (arrayField.includes('type')) { if (this.$route.path === '/alert') { @@ -573,6 +577,12 @@ export default { promises.push(await this.fetchVolumes(searchKeyword)) } + if (arrayField.includes('backupofferingid')) { + backupOfferingIndex = this.fields.findIndex(item => item.name === 'backupofferingid') + this.fields[backupOfferingIndex].loading = true + promises.push(await this.fetchBackupOfferings(searchKeyword)) + } + Promise.all(promises).then(response => { if (typeIndex > -1) { const types = response.filter(item => item.type === 'type') @@ -669,6 +679,13 @@ export default { this.fields[usageTypeIndex].opts = this.sortArray(usageTypes[0].data) } } + + if (backupOfferingIndex > -1) { + const backupOfferings = response.filter(item => item.type === 'backupofferingid') + if (backupOfferings?.length > 0) { + this.fields[backupOfferingIndex].opts = this.sortArray(backupOfferings[0].data) + } + } }).finally(() => { if (typeIndex > -1) { this.fields[typeIndex].loading = false @@ -715,6 +732,9 @@ export default { if (usageTypeIndex > -1) { this.fields[usageTypeIndex].loading = false } + if (backupOfferingIndex > -1) { + this.fields[backupOfferingIndex].loading = false + } if (Array.isArray(arrayField)) { this.fillFormFieldValues() } @@ -1309,6 +1329,19 @@ export default { }) }) }, + fetchBackupOfferings (searchKeyword) { + return new Promise((resolve, reject) => { + api('listBackupOfferings').then(json => { + const backupOfferings = json.listbackupofferingsresponse.backupoffering + resolve({ + type: 'backupofferingid', + data: backupOfferings + }) + }).catch(error => { + reject(error.response.headers['x-description']) + }) + }) + }, onSearch (value) { this.paramsFilter = {} this.searchQuery = value diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 2fbd5fbb753d..0fa04795c83e 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -418,8 +418,12 @@ export default { title: 'label.backup', icon: 'cloud-upload-outlined', permission: ['listBackups'], - columns: ['name', 'status', 'size', 'virtualsize', 'virtualmachinename', 'intervaltype', 'type', 'created', 'account', 'domain', 'zone'], + columns: ['name', 'status', 'size', 'virtualsize', 'virtualmachinename', 'backupofferingname', 'intervaltype', 'type', 'created', 'account', 'domain', 'zone'], details: ['name', 'description', 'virtualmachinename', 'id', 'intervaltype', 'type', 'externalid', 'size', 'virtualsize', 'volumes', 'backupofferingname', 'zone', 'account', 'domain', 'created'], + searchFilters: () => { + var filters = ['keyword', 'zoneid', 'domainid', 'account', 'backupofferingid'] + return filters + }, actions: [ { api: 'restoreBackup', From c31da8be79ac37433bddcfbd7403e22f477cccc8 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 6 May 2025 11:49:51 +0530 Subject: [PATCH 087/147] use listBackupOfferings instead of listConfiguration in startbackup.vue to get the provider name --- .../api/response/BackupOfferingResponse.java | 8 ++++++++ .../backup/dao/BackupOfferingDaoImpl.java | 1 + ui/src/views/compute/StartBackup.vue | 13 +++++++------ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupOfferingResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupOfferingResponse.java index 480ebcfb13d3..0e895fa4e967 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupOfferingResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupOfferingResponse.java @@ -41,6 +41,10 @@ public class BackupOfferingResponse extends BaseResponse { @Param(description = "description for the backup offering") private String description; + @SerializedName(ApiConstants.PROVIDER) + @Param(description = "provider name", since = "4.21.0") + private String provider; + @SerializedName(ApiConstants.EXTERNAL_ID) @Param(description = "external ID on the provider side") private String externalId; @@ -69,6 +73,10 @@ public void setExternalId(String externalId) { this.externalId = externalId; } + public void setProvider(String provider) { + this.provider = provider; + } + public void setName(String name) { this.name = name; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupOfferingDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupOfferingDaoImpl.java index 0568a0185bba..9d67d07fe5e3 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupOfferingDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupOfferingDaoImpl.java @@ -58,6 +58,7 @@ public BackupOfferingResponse newBackupOfferingResponse(BackupOffering offering) response.setName(offering.getName()); response.setDescription(offering.getDescription()); response.setExternalId(offering.getExternalId()); + response.setProvider(offering.getProvider()); response.setUserDrivenBackups(offering.isUserDrivenBackupAllowed()); if (zone != null) { response.setZoneId(zone.getUuid()); diff --git a/ui/src/views/compute/StartBackup.vue b/ui/src/views/compute/StartBackup.vue index 45f784c85c54..167b012d2abf 100644 --- a/ui/src/views/compute/StartBackup.vue +++ b/ui/src/views/compute/StartBackup.vue @@ -64,17 +64,17 @@ export default { }, data () { return { - plugin: null, + provider: null, loading: false } }, created () { this.initForm() - this.getBackupProviderPlugin() + this.getBackupProvider() }, computed: { canSetNameAndDescription () { - return ['nas', 'dummy'].includes(this.plugin) + return ['nas', 'dummy'].includes(this.provider) } }, methods: { @@ -82,10 +82,11 @@ export default { this.formRef = ref() this.form = ({}) }, - getBackupProviderPlugin () { + getBackupProvider () { this.loading = true - api('listConfigurations', { name: 'backup.framework.provider.plugin' }).then(json => { - this.plugin = json.listconfigurationsresponse.configuration[0].value + api('listBackupOfferings', { id: this.resource.backupofferingid }).then(json => { + this.provider = json.listbackupofferingsresponse.backupoffering[0].provider + console.log('this.provider', this.provider) }).finally(() => { this.loading = false }) From 1e164efdc2907d9fc840f3ff4a10941f3b97dfbc Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Tue, 6 May 2025 14:22:01 +0530 Subject: [PATCH 088/147] Don't show select deployment infra section to end users in create instance from backup form --- ui/src/components/view/DeployVMFromBackup.vue | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ui/src/components/view/DeployVMFromBackup.vue b/ui/src/components/view/DeployVMFromBackup.vue index 3f50ec9aa032..9a6e7defab40 100644 --- a/ui/src/components/view/DeployVMFromBackup.vue +++ b/ui/src/components/view/DeployVMFromBackup.vue @@ -45,7 +45,10 @@
- + diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 60616b61a4c6..8bc8c2ee54cb 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -418,12 +418,23 @@ export default { title: 'label.backup', icon: 'cloud-upload-outlined', permission: ['listBackups'], + params: { listvmdetails: 'true' }, columns: ['name', 'status', 'size', 'virtualsize', 'virtualmachinename', 'backupofferingname', 'intervaltype', 'type', 'created', 'account', 'domain', 'zone'], details: ['name', 'description', 'virtualmachinename', 'id', 'intervaltype', 'type', 'externalid', 'size', 'virtualsize', 'volumes', 'backupofferingname', 'zone', 'account', 'domain', 'created'], searchFilters: () => { var filters = ['name', 'zoneid', 'domainid', 'account', 'backupofferingid'] return filters }, + tabs: [ + { + name: 'details', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/DetailsTab.vue'))) + }, + { + name: 'instance.metadata', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/BackupMetadata.vue'))) + } + ], actions: [ { api: 'restoreBackup', diff --git a/ui/src/views/compute/InstanceTab.vue b/ui/src/views/compute/InstanceTab.vue index bb05dbc681c9..877d0ebff3f6 100644 --- a/ui/src/views/compute/InstanceTab.vue +++ b/ui/src/views/compute/InstanceTab.vue @@ -60,7 +60,7 @@ apiName="listBackups" :resource="resource" :params="{virtualmachineid: dataResource.id}" - :columns="['name', 'status', 'size', 'virtualsize', 'type', 'created']" + :columns="['name', 'status', 'size', 'virtualsize', 'type', 'intervaltype', 'created']" :routerlinks="(record) => { return { name: '/backup/' + record.id } }" :showSearch="false"/> diff --git a/ui/src/views/storage/CreateVMFromBackup.vue b/ui/src/views/storage/CreateVMFromBackup.vue index 037def72f6a8..f9dbf535d060 100644 --- a/ui/src/views/storage/CreateVMFromBackup.vue +++ b/ui/src/views/storage/CreateVMFromBackup.vue @@ -127,9 +127,12 @@ export default { this.dataPreFill.templateid = this.vmdetails.templateid this.dataPreFill.isoid = this.vmdetails.templateid this.dataPreFill.allowIpAddressesFetch = !this.resource.virtualmachineid - this.dataPreFill.networkids = (this.vmdetails.networkids || '').split(',') - this.dataPreFill.ipAddresses = (this.vmdetails.ipaddresses || '').split(',') - this.dataPreFill.macAddresses = (this.vmdetails.macaddresses || '').split(',') + if (this.vmdetails.nics) { + const nics = JSON.parse(this.vmdetails.nics) + this.dataPreFill.networkids = nics.map(nic => nic.networkid) + this.dataPreFill.ipAddresses = nics.map(nic => nic.ipaddress) + this.dataPreFill.macAddresses = nics.map(nic => nic.macaddress) + } const volumes = JSON.parse(this.resource.volumes) const disksdetails = volumes.map((volume, index) => ({ name: volume.path, From 4bacb50d61147b1b251ae07a00d1cf2e90f5494b Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 21 Jul 2025 11:00:05 +0530 Subject: [PATCH 127/147] Save and restore Instance settings to/from backup --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../cloudstack/backup/BackupManager.java | 2 +- .../backup/DummyBackupProvider.java | 2 +- .../cloudstack/backup/NASBackupProvider.java | 2 +- .../backup/NetworkerBackupProvider.java | 2 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 34 +++++++++++++++---- .../cloudstack/backup/BackupManagerImpl.java | 28 ++++++++++++--- .../cloudstack/backup/BackupManagerTest.java | 4 +-- ui/src/components/view/BackupMetadata.vue | 31 +++++++++++++---- 9 files changed, 82 insertions(+), 24 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 8f2cbaf4571e..4d8e6eefd803 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -607,6 +607,7 @@ public class ApiConstants { public static final String VM_DETAILS = "vmdetails"; public static final String VM_LIMIT = "vmlimit"; public static final String VM_TOTAL = "vmtotal"; + public static final String VM_SETTINGS = "vmsettings"; public static final String VM_TYPE = "vmtype"; public static final String VNET = "vnet"; public static final String IS_VOLATILE = "isvolatile"; diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index b2e7b275ea7f..2f7b1b67bcfd 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -262,7 +262,7 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer void checkDiskOfferingSizeAgainstBackup(List dataDiskOfferingsInfo, Backup backup); - Map getVmDetailsForBackup(VirtualMachine vm); + Map getBackupDetailsFromVM(VirtualMachine vm); String createVolumeInfoFromVolumes(List vmVolumes); diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index 4e3134a1978a..05b4b52ccb83 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -172,7 +172,7 @@ public Pair takeBackup(VirtualMachine vm, Boolean quiesceVM) { backup.setName(backupManager.getBackupNameFromVM(vm)); List volumes = new ArrayList<>(volumeDao.findByInstance(vm.getId())); backup.setBackedUpVolumes(backupManager.createVolumeInfoFromVolumes(volumes)); - Map details = backupManager.getVmDetailsForBackup(vm); + Map details = backupManager.getBackupDetailsFromVM(vm); backup.setDetails(details); backup = backupDao.persist(backup); diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 413fee885424..2fdde06da9ac 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -240,7 +240,7 @@ private BackupVO createBackupObject(VirtualMachine vm, String backupPath) { backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); backup.setName(backupManager.getBackupNameFromVM(vm)); - Map details = backupManager.getVmDetailsForBackup(vm); + Map details = backupManager.getBackupDetailsFromVM(vm); backup.setDetails(details); return backupDao.persist(backup); diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index 1be1403b9545..783c964bc4fa 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -521,7 +521,7 @@ public Pair takeBackup(VirtualMachine vm, Boolean quiesceVM) { if (backup != null) { List volumes = new ArrayList<>(volumeDao.findByInstance(vm.getId())); backup.setBackedUpVolumes(backupManager.createVolumeInfoFromVolumes(volumes)); - Map details = backupManager.getVmDetailsForBackup(vm); + Map details = backupManager.getBackupDetailsFromVM(vm); backup.setDetails(details); backupDao.persist(backup); return new Pair<>(true, backup); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index ac1a510c1bb6..f58e189d9ce0 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.StringReader; import java.io.UnsupportedEncodingException; +import java.lang.reflect.Type; import java.net.URLDecoder; import java.text.SimpleDateFormat; import java.time.LocalDateTime; @@ -400,6 +401,8 @@ import com.cloud.vm.snapshot.VMSnapshotManager; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; @@ -9411,7 +9414,6 @@ private void checkRootDiskSizeAgainstBackup(Long instanceVolumeSize,DiskOffering @Override public UserVm allocateVMFromBackup(CreateVMFromBackupCmd cmd) throws InsufficientCapacityException, ResourceAllocationException, ResourceUnavailableException { - //Verify that all objects exist before passing them to the service Account owner = _accountService.getActiveAccountById(cmd.getEntityOwnerId()); Long zoneId = cmd.getZoneId(); DataCenter zone = _dcDao.findById(zoneId); @@ -9453,19 +9455,21 @@ public UserVm allocateVMFromBackup(CreateVMFromBackupCmd cmd) throws Insufficien } verifyServiceOffering(cmd, serviceOffering); - Long templateId; VirtualMachineTemplate template; if (cmd.getTemplateId() != null) { - templateId = cmd.getTemplateId(); + Long templateId = cmd.getTemplateId(); template = _templateDao.findById(templateId); if (template == null) { throw new InvalidParameterValueException("Unable to use template " + templateId); } } else { - templateId = backupVm.getTemplateId(); - template = _templateDao.findById(templateId); + String templateUuid = backup.getDetail(ApiConstants.TEMPLATE_ID); + if (templateUuid == null) { + throw new CloudRuntimeException("Backup doesn't contain Template uuid. Please specify a valid Template/ISO while creating the instance"); + } + template = _templateDao.findByUuid(templateUuid); if (template == null) { - throw new CloudRuntimeException("Unable to find template associated with the backup. Please specify a valid template while creating instance"); + throw new CloudRuntimeException("Unable to find template associated with the backup. Please specify a valid Template/ISO while creating instance"); } } verifyTemplate(cmd, template, serviceOffering.getId()); @@ -9534,7 +9538,23 @@ public UserVm allocateVMFromBackup(CreateVMFromBackupCmd cmd) throws Insufficien ipToNetworkMap = backupManager.getIpToNetworkMapFromBackup(backup, cmd.getPreserveIp(), networkIds); } - return createVirtualMachine(cmd, zone, owner, serviceOffering, template, hypervisorType, diskOfferingId, size, overrideDiskOfferingId, dataDiskOfferingsInfo, networkIds, ipToNetworkMap, null, null); + UserVm vm = createVirtualMachine(cmd, zone, owner, serviceOffering, template, hypervisorType, diskOfferingId, size, overrideDiskOfferingId, dataDiskOfferingsInfo, networkIds, ipToNetworkMap, null, null); + + String vmSettingsFromBackup = backup.getDetail(ApiConstants.VM_SETTINGS); + if (vm != null && vmSettingsFromBackup != null) { + UserVmVO vmVO = _vmDao.findById(vm.getId()); + Map details = userVmDetailsDao.listDetailsKeyPairs(vm.getId()); + vmVO.setDetails(details); + + Type type = new TypeToken>(){}.getType(); + Map vmDetailsFromBackup = new Gson().fromJson(vmSettingsFromBackup, type); + for (Map.Entry entry : vmDetailsFromBackup.entrySet()) { + vmVO.setDetail(entry.getKey(), entry.getValue()); + } + _vmDao.saveDetails(vmVO); + } + + return vm; } @Override diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 6814d143201e..d6bdf5ee33cc 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.backup; +import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -53,9 +54,11 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.VolumeApiService; import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.DomainManager; import com.cloud.user.ResourceLimitService; import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.vm.UserVmDetailVO; import com.cloud.vm.VirtualMachineManager; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -149,8 +152,10 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; public class BackupManagerImpl extends ManagerBase implements BackupManager { @@ -191,6 +196,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { @Inject private UserVmJoinDao userVmJoinDao; @Inject + private UserVmDetailsDao userVmDetailsDao; + @Inject private NetworkDao networkDao; @Inject private NetworkService networkService; @@ -324,12 +331,24 @@ public boolean deleteBackupOffering(final Long offeringId) { } @Override - public Map getVmDetailsForBackup(VirtualMachine vm) { + public Map getBackupDetailsFromVM(VirtualMachine vm) { HashMap details = new HashMap<>(); - ServiceOffering serviceOffering = serviceOfferingDao.findById(vm.getServiceOfferingId()); + + ServiceOffering serviceOffering = serviceOfferingDao.findById(vm.getServiceOfferingId()); details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); - List userVmJoinVOs = userVmJoinDao.searchByIds(vm.getId()); + VirtualMachineTemplate template = vmTemplateDao.findById(vm.getTemplateId()); + details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + List vmDetails = userVmDetailsDao.listDetails(vm.getId()); + HashMap settings = new HashMap<>(); + for (UserVmDetailVO detail : vmDetails) { + settings.put(detail.getName(), detail.getValue()); + } + if (!settings.isEmpty()) { + details.put(ApiConstants.VM_SETTINGS, new Gson().toJson(settings)); + } + + List userVmJoinVOs = userVmJoinDao.searchByIds(vm.getId()); if (userVmJoinVOs != null && !userVmJoinVOs.isEmpty()) { List> nics = new ArrayList<>(); Set seen = new HashSet<>(); @@ -1102,7 +1121,8 @@ public Map getIpToNetworkMapFromBackup(Backup backup, "Please specify at least one valid network while creating instance"); } - List> nics = new Gson().fromJson(nicsJson, List.class); + Type type = new TypeToken>>(){}.getType(); + List> nics = new Gson().fromJson(nicsJson, type); for (Map nic : nics) { String networkUuid = nic.get(ApiConstants.NETWORK_ID); diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index 6d5fe84551fa..08cbf3129e3d 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -804,7 +804,7 @@ public void testBackupSyncTask() { } @Test - public void testGetVmDetailsForBackup() { + public void testGetBackupDetailsFromVM() { Long vmId = 1L; VirtualMachine vm = mock(VirtualMachine.class); when(vm.getServiceOfferingId()).thenReturn(1L); @@ -819,7 +819,7 @@ public void testGetVmDetailsForBackup() { List userVmJoinVOs = Collections.singletonList(userVmJoinVO); when(userVmJoinDao.searchByIds(vmId)).thenReturn(userVmJoinVOs); - Map details = backupManager.getVmDetailsForBackup(vm); + Map details = backupManager.getBackupDetailsFromVM(vm); assertEquals("service-offering-uuid", details.get(ApiConstants.SERVICE_OFFERING_ID)); assertEquals("mocked-network-uuid", details.get(ApiConstants.NETWORK_IDS)); diff --git a/ui/src/components/view/BackupMetadata.vue b/ui/src/components/view/BackupMetadata.vue index 3e4aa6a0a425..1cbd2444a7f0 100644 --- a/ui/src/components/view/BackupMetadata.vue +++ b/ui/src/components/view/BackupMetadata.vue @@ -7,11 +7,11 @@
{{ getFieldLabel(item) }}
-
-
- - {{ nic.networkname || nic.networkid }} - +
+
+ + {{ nic.networkname || nic.networkid }} +
IP Address: {{ nic.ipaddress }} | IPv6: {{ nic.ip6address }} @@ -30,6 +30,14 @@ {{ getServiceOfferingDisplayName() }}
+
+
+ {{ key }} +
+ {{ value }} +
+
+
{{ backupMetadata[item] }}
@@ -66,13 +74,13 @@ export default { } fieldOrder.push('serviceofferingid') fieldOrder.push('nics') + fieldOrder.push('vmsettings') return fieldOrder.filter(field => this.backupMetadata[field] !== undefined) }, getNicEntities () { if (this.backupMetadata.nics) { - const nics = JSON.parse(this.backupMetadata.nics) - return nics + return JSON.parse(this.backupMetadata.nics) } return [] }, @@ -80,6 +88,9 @@ export default { if (field === 'templateid') { return this.backupMetadata.isiso === 'true' ? this.$t('label.iso') : this.$t('label.template') } + if (field === 'vmsettings') { + return this.$t('label.settings') + } return this.$t('label.' + String(field).toLowerCase()) }, getTemplateDisplayName () { @@ -93,6 +104,12 @@ export default { return this.backupMetadata.serviceofferingname } return this.backupMetadata.serviceofferingid + }, + getVmSettings () { + if (this.backupMetadata.vmsettings) { + return JSON.parse(this.backupMetadata.vmsettings) + } + return {} } } } From b5830c62c9a0deab9aa4d8c99a8cb3f586a588d2 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 21 Jul 2025 11:54:52 +0530 Subject: [PATCH 128/147] Fix UT --- .../com/cloud/vm/UserVmManagerImplTest.java | 8 ++-- .../cloudstack/backup/BackupManagerTest.java | 40 ++++++++++++++----- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index 9947a500d43c..130eefb88f43 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -3367,11 +3367,11 @@ public void testAllocateVMFromBackupUsingBackupValues() throws InsufficientCapac when(backupDao.findById(backupId)).thenReturn(backup); UserVmVO userVmVO = new UserVmVO(); - userVmVO.setTemplateId(templateId); when(userVmDao.findByIdIncludingRemoved(vmId)).thenReturn(userVmVO); VMTemplateVO template = mock(VMTemplateVO.class); when(template.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); - when(templateDao.findById(templateId)).thenReturn(template); + when(backup.getDetail(ApiConstants.TEMPLATE_ID)).thenReturn("template-uuid"); + when(templateDao.findByUuid("template-uuid")).thenReturn(template); DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); when(backup.getDetail(ApiConstants.SERVICE_OFFERING_ID)).thenReturn("service-offering-uuid"); @@ -3542,11 +3542,11 @@ public void testAllocateVMFromBackupUsingBackupValuesWithISO() throws Insufficie when(backupDao.findById(backupId)).thenReturn(backup); UserVmVO userVmVO = new UserVmVO(); - userVmVO.setTemplateId(isoId); when(userVmDao.findByIdIncludingRemoved(vmId)).thenReturn(userVmVO); VMTemplateVO iso = mock(VMTemplateVO.class); when(iso.getFormat()).thenReturn(Storage.ImageFormat.ISO); - when(templateDao.findById(isoId)).thenReturn(iso); + when(backup.getDetail(ApiConstants.TEMPLATE_ID)).thenReturn("iso-uuid"); + when(templateDao.findByUuid("iso-uuid")).thenReturn(iso); ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index 08cbf3129e3d..f37cc0ee3de1 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -41,12 +41,14 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; import com.cloud.exception.ResourceAllocationException; +import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; @@ -58,9 +60,11 @@ import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.vm.UserVmDetailVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.api.ApiConstants; @@ -112,6 +116,8 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.atLeastOnce; +import javax.inject.Inject; + @RunWith(MockitoJUnitRunner.class) public class BackupManagerTest { @Spy @@ -181,6 +187,9 @@ public class BackupManagerTest { @Mock private NetworkService networkService; + @Mock + private UserVmDetailsDao userVmDetailsDao; + private AccountVO account; private UserVO user; @@ -808,11 +817,21 @@ public void testGetBackupDetailsFromVM() { Long vmId = 1L; VirtualMachine vm = mock(VirtualMachine.class); when(vm.getServiceOfferingId()).thenReturn(1L); + when(vm.getTemplateId()).thenReturn(2L); when(vm.getId()).thenReturn(vmId); ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); when(serviceOffering.getUuid()).thenReturn("service-offering-uuid"); when(serviceOfferingDao.findById(1L)).thenReturn(serviceOffering); + VMTemplateVO template = mock(VMTemplateVO.class); + when(template.getUuid()).thenReturn("template-uuid"); + when(vmTemplateDao.findById(2L)).thenReturn(template); + + UserVmDetailVO userVmDetail = mock(UserVmDetailVO.class); + when(userVmDetail.getName()).thenReturn("mocked-detail-name"); + when(userVmDetail.getValue()).thenReturn("mocked-detail-value"); + List vmDetails = Collections.singletonList(userVmDetail); + when(userVmDetailsDao.listDetails(vmId)).thenReturn(vmDetails); UserVmJoinVO userVmJoinVO = mock(UserVmJoinVO.class); when(userVmJoinVO.getNetworkUuid()).thenReturn("mocked-network-uuid"); @@ -822,7 +841,8 @@ public void testGetBackupDetailsFromVM() { Map details = backupManager.getBackupDetailsFromVM(vm); assertEquals("service-offering-uuid", details.get(ApiConstants.SERVICE_OFFERING_ID)); - assertEquals("mocked-network-uuid", details.get(ApiConstants.NETWORK_IDS)); + assertEquals("[{\"networkid\":\"mocked-network-uuid\"}]", details.get(ApiConstants.NICS)); + assertEquals("{\"mocked-detail-name\":\"mocked-detail-value\"}", details.get(ApiConstants.VM_SETTINGS)); } @Test @@ -1284,7 +1304,6 @@ public void testGetIpToNetworkMapFromBackup() { // Test case 1: Missing network information Backup backup1 = mock(Backup.class); - when(backup1.getDetail(ApiConstants.NETWORK_IDS)).thenReturn(null); List networkIds1 = new ArrayList<>(); try { backupManager.getIpToNetworkMapFromBackup(backup1, true, networkIds1); @@ -1295,10 +1314,10 @@ public void testGetIpToNetworkMapFromBackup() { // Test case 2: IP preservation enabled with IP information Backup backup2 = mock(Backup.class); - when(backup2.getDetail(ApiConstants.NETWORK_IDS)).thenReturn(networkUuid1 + "," + networkUuid2); - when(backup2.getDetail(ApiConstants.IP_ADDRESSES)).thenReturn(ip1 + "," + ip2); - when(backup2.getDetail(ApiConstants.IP6_ADDRESSES)).thenReturn(ipv61 + "," + ipv62); - when(backup2.getDetail(ApiConstants.MAC_ADDRESSES)).thenReturn(mac1 + "," + mac2); + String nicsJson = String.format("[{\"networkid\":\"%s\",\"ipaddress\":\"%s\",\"ip6address\":\"%s\",\"macaddress\":\"%s\"}," + + "{\"networkid\":\"%s\",\"ipaddress\":\"%s\",\"ip6address\":\"%s\",\"macaddress\":\"%s\"}]", + networkUuid1, ip1, ipv61, mac1, networkUuid2, ip2, ipv62, mac2); + when(backup2.getDetail(ApiConstants.NICS)).thenReturn(nicsJson); NetworkVO network1 = mock(NetworkVO.class); NetworkVO network2 = mock(NetworkVO.class); @@ -1324,10 +1343,8 @@ public void testGetIpToNetworkMapFromBackup() { // Test case 3: IP preservation enabled but missing IP information Backup backup3 = mock(Backup.class); - when(backup3.getDetail(ApiConstants.NETWORK_IDS)).thenReturn(networkUuid1); - when(backup3.getDetail(ApiConstants.IP_ADDRESSES)).thenReturn(null); - when(backup3.getDetail(ApiConstants.IP6_ADDRESSES)).thenReturn(null); - when(backup3.getDetail(ApiConstants.MAC_ADDRESSES)).thenReturn(null); + nicsJson = String.format("[{\"networkid\":\"%s\"}]", networkUuid1); + when(backup3.getDetail(ApiConstants.NICS)).thenReturn(nicsJson); List networkIds3 = new ArrayList<>(); Map result3 = backupManager.getIpToNetworkMapFromBackup(backup3, true, networkIds3); @@ -1339,7 +1356,8 @@ public void testGetIpToNetworkMapFromBackup() { // Test case 4: IP preservation disabled Backup backup4 = mock(Backup.class); - when(backup4.getDetail(ApiConstants.NETWORK_IDS)).thenReturn(networkUuid1); + nicsJson = String.format("[{\"networkid\":\"%s\"}]", networkUuid1); + when(backup4.getDetail(ApiConstants.NICS)).thenReturn(nicsJson); List networkIds4 = new ArrayList<>(); Map result4 = backupManager.getIpToNetworkMapFromBackup(backup4, false, networkIds4); From a3f7bfa68d12ff290332fd110c25a1bf7506b550 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 21 Jul 2025 12:00:14 +0530 Subject: [PATCH 129/147] review comments --- .../api/command/admin/vm/CreateVMFromBackupCmdByAdmin.java | 4 ++-- .../api/command/user/vm/CreateVMFromBackupCmd.java | 3 --- .../java/org/apache/cloudstack/backup/BackupProvider.java | 6 +++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/CreateVMFromBackupCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/CreateVMFromBackupCmdByAdmin.java index 9716a67d0eeb..d95f17ef304c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/CreateVMFromBackupCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/CreateVMFromBackupCmdByAdmin.java @@ -39,10 +39,10 @@ authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) public class CreateVMFromBackupCmdByAdmin extends CreateVMFromBackupCmd implements AdminCmd { - @Parameter(name = ApiConstants.POD_ID, type = CommandType.UUID, entityType = PodResponse.class, description = "destination Pod ID to deploy the VM to - parameter available for root admin only", since = "4.13") + @Parameter(name = ApiConstants.POD_ID, type = CommandType.UUID, entityType = PodResponse.class, description = "destination Pod ID to deploy the VM to - parameter available for root admin only", since = "4.21") private Long podId; - @Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.UUID, entityType = ClusterResponse.class, description = "destination Cluster ID to deploy the VM to - parameter available for root admin only", since = "4.13") + @Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.UUID, entityType = ClusterResponse.class, description = "destination Cluster ID to deploy the VM to - parameter available for root admin only", since = "4.21") private Long clusterId; public Long getPodId() { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java index cec0cfafa63b..309b46478ba4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java @@ -50,9 +50,6 @@ authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) public class CreateVMFromBackupCmd extends BaseDeployVMCmd { - @Inject - BackupManager backupManager; - ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index 3a0ce16ef3a2..1eb36f895565 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -104,13 +104,13 @@ public interface BackupProvider { void syncBackupMetrics(Long zoneId); /** - * This method should TODO - * @param vm the machine to get restore point for + * Returns a list of Backup.RestorePoint + * @param vm the machine to get the restore points for */ List listRestorePoints(VirtualMachine vm); /** - * This method should TODO + * Creates and returns an entry in the backups table by getting the information from restorePoint and vm. * * @param restorePoint the restore point to create a backup for * @param vm The machine for which to create a backup From 65ac762dfed732cf2a0bff5e552e48377d28dc08 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 21 Jul 2025 12:02:24 +0530 Subject: [PATCH 130/147] add license to BackupMetadata.vue --- ui/src/components/view/BackupMetadata.vue | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ui/src/components/view/BackupMetadata.vue b/ui/src/components/view/BackupMetadata.vue index 1cbd2444a7f0..fea93a4e9c72 100644 --- a/ui/src/components/view/BackupMetadata.vue +++ b/ui/src/components/view/BackupMetadata.vue @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +