From 7d638fc36f69b5ce9d254dfb74a1f2ec71e5054c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jandre?= <48719461+JoaoJandre@users.noreply.github.com> Date: Mon, 17 Jun 2024 16:54:01 -0300 Subject: [PATCH 01/34] KVM incremental snapshot feature --- .../agent/properties/AgentProperties.java | 10 + .../agent/api/ConvertSnapshotAnswer.java | 36 + .../agent/api/ConvertSnapshotCommand.java | 42 ++ .../agent/api/RecreateCheckpointsCommand.java | 49 ++ .../StorageSubsystemCommandHandlerBase.java | 2 +- .../storage/template/TemplateConstants.java | 2 +- .../storage/to/SnapshotObjectTO.java | 57 +- .../cloudstack/storage/to/VolumeObjectTO.java | 21 + .../service/VolumeOrchestrationService.java | 6 + .../subsystem/api/storage/SnapshotInfo.java | 12 + .../api/storage/SnapshotService.java | 4 + .../subsystem/api/storage/StorageAction.java | 1 + .../cloud/vm/VirtualMachineManagerImpl.java | 77 ++ .../orchestration/VolumeOrchestrator.java | 33 + .../vm/VirtualMachineManagerImplTest.java | 77 ++ .../orchestration/VolumeOrchestratorTest.java | 58 ++ .../main/java/com/cloud/host/dao/HostDao.java | 3 + .../java/com/cloud/host/dao/HostDaoImpl.java | 23 + .../datastore/db/SnapshotDataStoreDao.java | 6 + .../db/SnapshotDataStoreDaoImpl.java | 59 +- .../datastore/db/SnapshotDataStoreVO.java | 33 +- .../META-INF/db/schema-41910to42000.sql | 5 + .../snapshot/DefaultSnapshotStrategy.java | 80 +- .../storage/snapshot/SnapshotObject.java | 67 +- .../storage/snapshot/SnapshotServiceImpl.java | 117 ++- .../snapshot/DefaultSnapshotStrategyTest.java | 99 ++- .../snapshot/SnapshotServiceImplTest.java | 157 +++- .../ObjectInDataStoreManagerImpl.java | 30 +- .../endpoint/DefaultEndPointSelector.java | 61 +- .../db/SnapshotDataStoreDaoImplTest.java | 4 +- .../storage/volume/VolumeServiceImpl.java | 19 +- .../resource/LibvirtComputingResource.java | 126 +++- .../kvm/resource/LibvirtXMLParser.java | 21 + .../LibvirtConvertSnapshotCommandWrapper.java | 93 +++ .../wrapper/LibvirtMigrateCommandWrapper.java | 32 +- ...virtRecreateCheckpointsCommandWrapper.java | 51 ++ .../LibvirtRevertSnapshotCommandWrapper.java | 22 +- .../wrapper/LibvirtStartCommandWrapper.java | 10 + .../kvm/storage/IscsiAdmStoragePool.java | 6 + .../kvm/storage/KVMStoragePool.java | 2 + .../kvm/storage/KVMStorageProcessor.java | 709 +++++++++++++++--- .../kvm/storage/LibvirtStorageAdaptor.java | 7 +- .../kvm/storage/LibvirtStoragePool.java | 5 + .../kvm/storage/MultipathSCSIPool.java | 6 + .../kvm/storage/ScaleIOStoragePool.java | 6 + .../apache/cloudstack/utils/qemu/QemuImg.java | 36 + .../LibvirtComputingResourceTest.java | 76 +- ...bvirtRevertSnapshotCommandWrapperTest.java | 38 +- .../kvm/storage/KVMStorageProcessorTest.java | 74 +- .../CloudStackPrimaryDataStoreDriverImpl.java | 42 +- .../kvm/storage/LinstorStoragePool.java | 6 + .../kvm/storage/StorPoolStoragePool.java | 6 + scripts/storage/qcow2/managesnapshot.sh | 2 +- .../java/com/cloud/configuration/Config.java | 1 - .../cloud/storage/CreateSnapshotPayload.java | 9 + .../cloud/storage/VolumeApiServiceImpl.java | 5 + .../storage/snapshot/SnapshotManager.java | 18 +- .../storage/snapshot/SnapshotManagerImpl.java | 160 +++- .../cloud/template/TemplateManagerImpl.java | 8 +- .../java/com/cloud/test/DatabaseConfig.java | 3 - .../cloudstack/snapshot/SnapshotHelper.java | 8 + .../resource/NfsSecondaryStorageResource.java | 52 +- .../java/com/cloud/utils/script/Script.java | 14 + 63 files changed, 2480 insertions(+), 424 deletions(-) create mode 100644 core/src/main/java/com/cloud/agent/api/ConvertSnapshotAnswer.java create mode 100644 core/src/main/java/com/cloud/agent/api/ConvertSnapshotCommand.java create mode 100644 core/src/main/java/com/cloud/agent/api/RecreateCheckpointsCommand.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertSnapshotCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRecreateCheckpointsCommandWrapper.java diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index b27ba651e4f9..87144276f304 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -810,6 +810,16 @@ public Property getWorkers() { */ public static final Property HOST_TAGS = new Property<>("host.tags", null, String.class); + /** + * Timeout (in seconds) to wait for the incremental snapshot to complete. + * */ + public static final Property INCREMENTAL_SNAPSHOT_TIMEOUT = new Property<>("incremental.snapshot.timeout", 10800); + + /** + * Timeout (in seconds) to wait for the snapshot reversion to complete. + * */ + public static final Property REVERT_SNAPSHOT_TIMEOUT = new Property<>("revert.snapshot.timeout", 10800); + public static class Property { private String name; private T defaultValue; diff --git a/core/src/main/java/com/cloud/agent/api/ConvertSnapshotAnswer.java b/core/src/main/java/com/cloud/agent/api/ConvertSnapshotAnswer.java new file mode 100644 index 000000000000..c19a061f736c --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ConvertSnapshotAnswer.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.cloud.agent.api; + +import org.apache.cloudstack.storage.to.SnapshotObjectTO; + +public class ConvertSnapshotAnswer extends Answer { + + private SnapshotObjectTO snapshotObjectTO; + + public ConvertSnapshotAnswer(SnapshotObjectTO snapshotObjectTO) { + super(null); + this.snapshotObjectTO = snapshotObjectTO; + } + + public SnapshotObjectTO getSnapshotObjectTO() { + return snapshotObjectTO; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/ConvertSnapshotCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertSnapshotCommand.java new file mode 100644 index 000000000000..e456d4b6b122 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ConvertSnapshotCommand.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.cloud.agent.api; + +import org.apache.cloudstack.storage.to.SnapshotObjectTO; + +public class ConvertSnapshotCommand extends Command { + + public static final String TEMP_SNAPSHOT_NAME = "_temp"; + + SnapshotObjectTO snapshotObjectTO; + + public SnapshotObjectTO getSnapshotObjectTO() { + return snapshotObjectTO; + } + + public ConvertSnapshotCommand(SnapshotObjectTO snapshotObjectTO) { + this.snapshotObjectTO = snapshotObjectTO; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/RecreateCheckpointsCommand.java b/core/src/main/java/com/cloud/agent/api/RecreateCheckpointsCommand.java new file mode 100644 index 000000000000..154b611be98f --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/RecreateCheckpointsCommand.java @@ -0,0 +1,49 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import org.apache.cloudstack.storage.to.VolumeObjectTO; + +import java.util.List; + +public class RecreateCheckpointsCommand extends Command { + + private List volumes; + + private String vmName; + + public RecreateCheckpointsCommand(List volumes, String vmName) { + this.volumes = volumes; + this.vmName = vmName; + } + + public List getDisks() { + return volumes; + } + + public String getVmName() { + return vmName; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java index 7d8225462cab..163b8e208c36 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java @@ -142,7 +142,7 @@ protected Answer execute(CreateObjectCommand cmd) { } return new CreateObjectAnswer("not supported type"); } catch (Exception e) { - logger.debug("Failed to create object: " + data.getObjectType() + ": " + e.toString()); + logger.error(String.format("Failed to create object [%s] due to [%s].", data.getObjectType(), e), e); return new CreateObjectAnswer(e.toString()); } } diff --git a/core/src/main/java/com/cloud/storage/template/TemplateConstants.java b/core/src/main/java/com/cloud/storage/template/TemplateConstants.java index d6622bed73ef..349997da1b7a 100644 --- a/core/src/main/java/com/cloud/storage/template/TemplateConstants.java +++ b/core/src/main/java/com/cloud/storage/template/TemplateConstants.java @@ -24,7 +24,7 @@ public final class TemplateConstants { public static final String DEFAULT_SNAPSHOT_ROOT_DIR = "snapshots"; public static final String DEFAULT_VOLUME_ROOT_DIR = "volumes"; public static final String DEFAULT_TMPLT_FIRST_LEVEL_DIR = "tmpl/"; - + public static final String DEFAULT_CHECKPOINT_ROOT_DIR = "checkpoints"; public static final String DEFAULT_SYSTEM_VM_TEMPLATE_PATH = "template/tmpl/1/"; public static final int DEFAULT_TMPLT_COPY_PORT = 80; diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java index 76b93909b8c1..47fa6d49e4ab 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java @@ -35,12 +35,15 @@ public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO { private VolumeObjectTO volume; private String parentSnapshotPath; private DataStoreTO dataStore; + private DataStoreTO imageStore; + private boolean kvmIncrementalSnapshot = false; private String vmName; private String name; private HypervisorType hypervisorType; private long id; private boolean quiescevm; private String[] parents; + private String checkpointPath; private Long physicalSize = (long) 0; private long accountId; @@ -59,22 +62,28 @@ public SnapshotObjectTO(SnapshotInfo snapshot) { this.setVmName(vol.getAttachedVmName()); } - SnapshotInfo parentSnapshot = snapshot.getParent(); - ArrayList parentsArry = new ArrayList(); - if (parentSnapshot != null) { - this.parentSnapshotPath = parentSnapshot.getPath(); - while(parentSnapshot != null) { - parentsArry.add(parentSnapshot.getPath()); - parentSnapshot = parentSnapshot.getParent(); - } - parents = parentsArry.toArray(new String[parentsArry.size()]); - ArrayUtils.reverse(parents); - } - this.dataStore = snapshot.getDataStore().getTO(); this.setName(snapshot.getName()); this.hypervisorType = snapshot.getHypervisorType(); this.quiescevm = false; + + this.checkpointPath = snapshot.getCheckpointPath(); + this.kvmIncrementalSnapshot = snapshot.isKvmIncrementalSnapshot(); + + SnapshotInfo parentSnapshot = snapshot.getParent(); + + if (parentSnapshot == null || (HypervisorType.KVM.equals(snapshot.getHypervisorType()) && !parentSnapshot.isKvmIncrementalSnapshot())) { + return; + } + + ArrayList parentsArray = new ArrayList<>(); + this.parentSnapshotPath = parentSnapshot.getPath(); + while (parentSnapshot != null) { + parentsArray.add(parentSnapshot.getPath()); + parentSnapshot = parentSnapshot.getParent(); + } + parents = parentsArray.toArray(new String[parentsArray.size()]); + ArrayUtils.reverse(parents); } @Override @@ -91,6 +100,30 @@ public void setDataStore(DataStoreTO store) { this.dataStore = store; } + public DataStoreTO getImageStore() { + return imageStore; + } + + public void setImageStore(DataStoreTO imageStore) { + this.imageStore = imageStore; + } + + public boolean isKvmIncrementalSnapshot() { + return kvmIncrementalSnapshot; + } + + public void setKvmIncrementalSnapshot(boolean kvmIncrementalSnapshot) { + this.kvmIncrementalSnapshot = kvmIncrementalSnapshot; + } + + public String getCheckpointPath() { + return checkpointPath; + } + + public void setCheckpointPath(String checkpointPath) { + this.checkpointPath = checkpointPath; + } + @Override public String getPath() { return this.path; diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java index 6514038ac623..a160d4f9a141 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java @@ -32,6 +32,8 @@ import com.cloud.storage.Volume; import java.util.Arrays; +import java.util.List; +import java.util.Set; public class VolumeObjectTO extends DownloadableObjectTO implements DataTO { private String uuid; @@ -75,6 +77,8 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO { @LogLevel(LogLevel.Log4jLevel.Off) private byte[] passphrase; private String encryptFormat; + private List checkpointPaths; + private Set checkpointImageStoreUrls; public VolumeObjectTO() { @@ -394,4 +398,21 @@ public void clearPassphrase() { public boolean requiresEncryption() { return passphrase != null && passphrase.length > 0; } + + + public List getCheckpointPaths() { + return checkpointPaths; + } + + public void setCheckpointPaths(List checkpointPaths) { + this.checkpointPaths = checkpointPaths; + } + + public Set getCheckpointImageStoreUrls() { + return checkpointImageStoreUrls; + } + + public void setCheckpointImageStoreUrls(Set checkpointImageStoreUrls) { + this.checkpointImageStoreUrls = checkpointImageStoreUrls; + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index 7950dda4d68e..8f0e16b3e7c2 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -23,6 +23,7 @@ import java.util.Set; import com.cloud.exception.ResourceAllocationException; +import com.cloud.utils.Pair; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; @@ -179,4 +180,9 @@ DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachin * Unmanage VM volumes */ void unmanageVolumes(long vmId); + + /** + * Retrieves the volume's checkpoints paths to be used in the KVM processor. If there are no checkpoints, it will return an empty list. + */ + Pair, Set> getVolumeCheckpointPathsAndImageStoreUrls(long volumeId, HypervisorType hypervisorType); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java index 3213484694eb..d47aa5865a0f 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java @@ -32,6 +32,10 @@ public interface SnapshotInfo extends DataObject, Snapshot { String getPath(); + DataStore getImageStore(); + + void setImageStore(DataStore imageStore); + SnapshotInfo getChild(); List getChildren(); @@ -56,6 +60,14 @@ public interface SnapshotInfo extends DataObject, Snapshot { void markBackedUp() throws CloudRuntimeException; + String getCheckpointPath(); + + void setCheckpointPath(String checkpointPath); + + void setKvmIncrementalSnapshot(boolean isKvmIncrementalSnapshot); + + boolean isKvmIncrementalSnapshot(); + Snapshot getSnapshotVO(); long getAccountId(); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java index d2e085fe90cf..d7b6b2ec75b7 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java @@ -25,8 +25,12 @@ public interface SnapshotService { SnapshotResult takeSnapshot(SnapshotInfo snapshot); + DataStore findSnapshotImageStore(SnapshotInfo snapshot); + SnapshotInfo backupSnapshot(SnapshotInfo snapshot); + SnapshotInfo convertSnapshot(SnapshotInfo snapshotInfo); + boolean deleteSnapshot(SnapshotInfo snapshot); boolean revertSnapshot(SnapshotInfo snapshot); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StorageAction.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StorageAction.java index dc93f4aa41e1..32e528fca199 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StorageAction.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StorageAction.java @@ -22,6 +22,7 @@ public enum StorageAction { TAKESNAPSHOT, BACKUPSNAPSHOT, DELETESNAPSHOT, + CONVERTSNAPSHOT, MIGRATEVOLUME, DELETEVOLUME } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 4772a5ad92de..9fed614b8545 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -49,12 +49,14 @@ import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; +import com.cloud.agent.api.RecreateCheckpointsCommand; import com.cloud.configuration.Resource; import com.cloud.domain.Domain; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.ResourceAllocationException; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcDao; +import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.user.dao.AccountDao; import com.cloud.event.ActionEventUtils; import com.google.gson.Gson; @@ -72,6 +74,7 @@ import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; @@ -91,6 +94,7 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.reservation.dao.ReservationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.utils.identity.ManagementServerNode; @@ -403,6 +407,16 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac @Inject private DomainDao domainDao; + @Inject + private SnapshotDataStoreDao snapshotDataStoreDao; + + @Inject + private SnapshotManager snapshotManager; + + @Inject + private VolumeDataFactory volumeDataFactory; + + VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this); Map _vmGurus = new HashMap<>(); @@ -2852,6 +2866,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy _networkMgr.commitNicForMigration(vmSrc, profile); volumeMgr.release(vm.getId(), srcHostId); _networkMgr.setHypervisorHostname(profile, dest, true); + recreateCheckpointsKvmOnVmAfterMigration(vm, dstHostId); updateVmPod(vm, dstHostId); } @@ -3301,6 +3316,7 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo _networkMgr.commitNicForMigration(vmSrc, profile); volumeMgr.release(vm.getId(), srcHostId); _networkMgr.setHypervisorHostname(profile, destination, true); + endSnapshotChainForVolumes(volumeToPoolMap, vm.getHypervisorType()); } work.setStep(Step.Done); @@ -3308,6 +3324,67 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo } } + protected void endSnapshotChainForVolumes(Map volumeToPoolMap, HypervisorType hypervisorType) { + Set volumes = volumeToPoolMap.keySet(); + volumes.forEach(volume -> { + Volume volumeOnDestination = _volsDao.findByPoolIdName(volumeToPoolMap.get(volume).getId(), volume.getName()); + snapshotManager.endSnapshotChainForVolume(volumeOnDestination.getId(), hypervisorType); + }); + } + + protected void recreateCheckpointsKvmOnVmAfterMigration(VMInstanceVO vm, long hostId) { + if (!HypervisorType.KVM.equals(vm.getHypervisorType())) { + logger.debug("Will not recreate checkpoint on VM as it is not running on KVM, thus it is not needed."); + return; + } + + List volumes = getVmVolumesWithCheckpointsToRecreate(vm); + + if (volumes.isEmpty()) { + logger.debug("Will not recreate checkpoints on VM as its volumes do not have any checkpoints associated with them."); + return; + } + + RecreateCheckpointsCommand recreateCheckpointsCommand = new RecreateCheckpointsCommand(volumes, vm.getInstanceName()); + Answer answer = null; + try { + logger.debug(String.format("Recreating the volume checkpoints with URLs [%s] of volumes [%s] on %s as part of the migration process.", volumes.stream().map(VolumeObjectTO::getCheckpointPaths).collect(Collectors.toList()), volumes, vm)); + answer = _agentMgr.send(hostId, recreateCheckpointsCommand); + } catch (AgentUnavailableException | OperationTimedoutException e) { + logger.error(String.format("Exception while sending command to host [%s] to recreate checkpoints with URLs [%s] of volumes [%s] on %s due to: [%s].", hostId, volumes.stream().map(VolumeObjectTO::getCheckpointPaths).collect(Collectors.toList()), volumes, vm, e.getMessage()), e); + throw new CloudRuntimeException(e); + } finally { + if (answer != null && answer.getResult()) { + logger.debug(String.format("Successfully recreated checkpoints on VM [%s].", vm)); + return; + } + + logger.debug(String.format("Migration on VM [%s] was successful; however, we weren't able to recreate the checkpoints on it. Marking the snapshot chain as ended." + + " Next snapshot will create a new snapshot chain.", vm)); + + volumes.forEach(volumeObjectTO -> snapshotManager.endSnapshotChainForVolume(volumeObjectTO.getId(), HypervisorType.KVM)); + } + } + + + protected List getVmVolumesWithCheckpointsToRecreate(VMInstanceVO vm) { + List vmVolumes = _volsDao.findByInstance(vm.getId()); + List volumes = new ArrayList<>(); + + for (VolumeVO volume : vmVolumes) { + Pair, Set> volumeCheckpointPathsAndImageStoreUrls = volumeMgr.getVolumeCheckpointPathsAndImageStoreUrls(volume.getId(), HypervisorType.KVM); + if (volumeCheckpointPathsAndImageStoreUrls.first().isEmpty()) { + continue; + } + VolumeObjectTO volumeTo = new VolumeObjectTO(); + volumeTo.setCheckpointPaths(volumeCheckpointPathsAndImageStoreUrls.first()); + volumeTo.setCheckpointImageStoreUrls(volumeCheckpointPathsAndImageStoreUrls.second()); + volumes.add(volumeTo); + } + return volumes; + } + + @Override public VirtualMachineTO toVmTO(final VirtualMachineProfile profile) { final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(profile.getVirtualMachine().getHypervisorType()); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index c6d156d470b4..bb85c5ed7fba 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -42,6 +42,7 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.user.AccountManager; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; @@ -81,11 +82,14 @@ import org.apache.cloudstack.secret.dao.PassphraseDao; import org.apache.cloudstack.snapshot.SnapshotHelper; import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -198,6 +202,8 @@ public enum UserVmCloneType { @Inject protected PrimaryDataStoreDao _storagePoolDao = null; @Inject + protected ImageStoreDao imageStoreDao; + @Inject protected TemplateDataStoreDao _vmTemplateStoreDao = null; @Inject protected VolumeDao _volumeDao; @@ -574,6 +580,11 @@ public VolumeInfo createVolumeFromSnapshot(Volume volume, Snapshot snapshot, Use throw e; } + boolean kvmIncrementalSnapshot = SnapshotManager.kvmIncrementalSnapshot.valueIn(_hostDao.findClusterIdByVolumeInfo(snapInfo.getBaseVolume())); + if (kvmIncrementalSnapshot && DataStoreRole.Image.equals(dataStoreRole)) { + snapInfo = snapshotHelper.convertSnapshotIfNeeded(snapInfo); + } + // don't try to perform a sync if the DataStoreRole of the snapshot is equal to DataStoreRole.Primary if (!DataStoreRole.Primary.equals(dataStoreRole) || kvmSnapshotOnlyInPrimaryStorage) { try { @@ -1975,8 +1986,30 @@ public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws Sto _vmCloneSettingDao.persist(vmCloneSettingVO); } } + Pair, Set> volumeCheckPointPathsAndImageStoreUrls = getVolumeCheckpointPathsAndImageStoreUrls(volumeInfo.getId(), vm.getHypervisorType()); + ((VolumeObjectTO) volTO).setCheckpointPaths(volumeCheckPointPathsAndImageStoreUrls.first()); + ((VolumeObjectTO) volTO).setCheckpointImageStoreUrls(volumeCheckPointPathsAndImageStoreUrls.second()); + } + } + @Override + public Pair, Set> getVolumeCheckpointPathsAndImageStoreUrls(long volumeId, HypervisorType hypervisorType) { + List checkpointPaths = new ArrayList<>(); + Set imageStoreIds = new HashSet<>(); + Set imageStoreUrls = new HashSet<>(); + if (HypervisorType.KVM.equals(hypervisorType)) { + List snapshotDataStoreVos = _snapshotDataStoreDao.listReadyByVolumeIdAndCheckpointPathNotNull(volumeId); + snapshotDataStoreVos.forEach(snapshotDataStoreVO -> { + checkpointPaths.add(snapshotDataStoreVO.getKvmCheckpointPath()); + if (DataStoreRole.Image.equals(snapshotDataStoreVO.getRole())) { + imageStoreIds.add(snapshotDataStoreVO.getDataStoreId()); + } + }); + imageStoreUrls = imageStoreIds.stream().map(id -> imageStoreDao.findById(id).getUrl()).collect(Collectors.toSet()); + logger.debug(String.format("Found [%s] snapshots [%s] that have checkpoints for volume with id [%s].", snapshotDataStoreVos.size(), snapshotDataStoreVos, volumeId)); } + + return new Pair<>(checkpointPaths, imageStoreUrls); } private void handleCheckAndRepairVolume(Volume vol, Long hostId) { diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index 9b32980087c0..2b8e9bbe69d2 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -45,10 +45,13 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcDao; +import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import org.apache.cloudstack.context.CallContext; @@ -56,6 +59,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.commons.collections.MapUtils; import org.junit.Assert; import org.junit.Before; @@ -150,6 +154,9 @@ public class VirtualMachineManagerImplTest { @Mock private ServiceOfferingVO serviceOfferingMock; + @Mock + private SnapshotManager snapshotManagerMock; + @Mock private DiskOfferingVO diskOfferingMock; @@ -1236,4 +1243,74 @@ public void testGetDiskOfferingSuitabilityForVm() { assertFalse(result.get(1L)); assertTrue(result.get(2L)); } + + @Test + public void recreateCheckpointsKvmOnVmAfterMigrationTestReturnIfNotKvm() { + Mockito.doReturn(HypervisorType.VMware).when(vmInstanceMock).getHypervisorType(); + + virtualMachineManagerImpl.recreateCheckpointsKvmOnVmAfterMigration(vmInstanceMock, 0); + + Mockito.verify(volumeDaoMock, Mockito.never()).findByInstance(Mockito.anyLong()); + } + + @Test + public void recreateCheckpointsKvmOnVmAfterMigrationTestReturnIfVolumesDoNotHaveCheckpoints() throws OperationTimedoutException, AgentUnavailableException { + Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); + Mockito.doReturn(new ArrayList()).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); + + virtualMachineManagerImpl.recreateCheckpointsKvmOnVmAfterMigration(vmInstanceMock, 0); + + Mockito.verify(agentManagerMock, Mockito.never()).send(Mockito.anyLong(), (Command) any()); + } + + @Test (expected = CloudRuntimeException.class) + public void recreateCheckpointsKvmOnVmAfterMigrationTestAgentUnavailableThrowsCloudRuntimeExceptionAndEndsSnapshotChains() throws OperationTimedoutException, AgentUnavailableException { + Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); + Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); + + Mockito.doThrow(new AgentUnavailableException(0)).when(agentManagerMock).send(Mockito.anyLong(), (Command) any()); + Mockito.doNothing().when(snapshotManagerMock).endSnapshotChainForVolume(Mockito.anyLong(), Mockito.any()); + + virtualMachineManagerImpl.recreateCheckpointsKvmOnVmAfterMigration(vmInstanceMock, 0); + + Mockito.verify(snapshotManagerMock, Mockito.times(1)).endSnapshotChainForVolume(Mockito.anyLong(),any()); + } + + @Test (expected = CloudRuntimeException.class) + public void recreateCheckpointsKvmOnVmAfterMigrationTestOperationTimedoutExceptionThrowsCloudRuntimeExceptionAndEndsSnapshotChains() throws OperationTimedoutException, AgentUnavailableException { + Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); + Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); + + Mockito.doThrow(new OperationTimedoutException(null, 0, 0, 0, false)).when(agentManagerMock).send(Mockito.anyLong(), (Command) any()); + Mockito.doNothing().when(snapshotManagerMock).endSnapshotChainForVolume(Mockito.anyLong(), Mockito.any()); + + virtualMachineManagerImpl.recreateCheckpointsKvmOnVmAfterMigration(vmInstanceMock, 0); + + Mockito.verify(snapshotManagerMock, Mockito.times(1)).endSnapshotChainForVolume(Mockito.anyLong(),any()); + } + + @Test + public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationFails() throws OperationTimedoutException, AgentUnavailableException { + Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); + Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); + + Mockito.doReturn(new com.cloud.agent.api.Answer(null, false, null)).when(agentManagerMock).send(Mockito.anyLong(), (Command) any()); + Mockito.doNothing().when(snapshotManagerMock).endSnapshotChainForVolume(Mockito.anyLong(), Mockito.any()); + + virtualMachineManagerImpl.recreateCheckpointsKvmOnVmAfterMigration(vmInstanceMock, 0); + + Mockito.verify(snapshotManagerMock, Mockito.times(1)).endSnapshotChainForVolume(Mockito.anyLong(),any()); + } + + @Test + public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationSucceeds() throws OperationTimedoutException, AgentUnavailableException { + Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); + Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); + + Mockito.doReturn(new com.cloud.agent.api.Answer(null, true, null)).when(agentManagerMock).send(Mockito.anyLong(), (Command) any()); + + virtualMachineManagerImpl.recreateCheckpointsKvmOnVmAfterMigration(vmInstanceMock, 0); + + Mockito.verify(snapshotManagerMock, Mockito.never()).endSnapshotChainForVolume(Mockito.anyLong(),any()); + } } diff --git a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java index a37845c86c2f..8cfc2ff200ab 100644 --- a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java +++ b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java @@ -18,13 +18,17 @@ import java.util.ArrayList; import java.util.Date; +import java.util.List; +import java.util.Set; import com.cloud.hypervisor.Hypervisor; import com.cloud.offering.DiskOffering; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage; import com.cloud.storage.Volume; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; +import com.cloud.utils.Pair; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; @@ -32,6 +36,10 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.commons.lang3.ObjectUtils; import org.junit.Assert; import org.junit.Before; @@ -66,6 +74,13 @@ public class VolumeOrchestratorTest { @Mock protected VolumeDao volumeDao; + @Mock + private SnapshotDataStoreDao snapshotDataStoreDaoMock; + + @Mock + private ImageStoreDao imageStoreDaoMock; + + @Spy @InjectMocks private VolumeOrchestrator volumeOrchestrator = new VolumeOrchestrator(); @@ -208,4 +223,47 @@ public void testImportVolume() { Mockito.verify(volume, Mockito.times(1)).setChainInfo(chainInfo); Mockito.verify(volume, Mockito.times(1)).setState(Volume.State.Ready); } + + @Test + public void getVolumeCheckpointPathsAndImageStoreUrlsTestReturnEmptyListsIfNotKVM() { + Pair, Set> result = volumeOrchestrator.getVolumeCheckpointPathsAndImageStoreUrls(0, Hypervisor.HypervisorType.VMware); + + Assert.assertTrue(result.first().isEmpty()); + Assert.assertTrue(result.second().isEmpty()); + } + + @Test + public void getVolumeCheckpointPathsAndImageStoreUrlsTestReturnCheckpointIfKVM() { + SnapshotDataStoreVO snapshotDataStoreVO = new SnapshotDataStoreVO(); + snapshotDataStoreVO.setKvmCheckpointPath("Test"); + snapshotDataStoreVO.setRole(DataStoreRole.Primary); + + Mockito.doReturn(List.of(snapshotDataStoreVO)).when(snapshotDataStoreDaoMock).listReadyByVolumeIdAndCheckpointPathNotNull(Mockito.anyLong()); + + Pair, Set> result = volumeOrchestrator.getVolumeCheckpointPathsAndImageStoreUrls(0, Hypervisor.HypervisorType.KVM); + + Assert.assertEquals("Test", result.first().get(0)); + Assert.assertTrue(result.second().isEmpty()); + } + + @Test + public void getVolumeCheckpointPathsAndImageStoreUrlsTestReturnCheckpointIfKVMAndImageStore() { + SnapshotDataStoreVO snapshotDataStoreVO = new SnapshotDataStoreVO(); + snapshotDataStoreVO.setKvmCheckpointPath("Test"); + snapshotDataStoreVO.setRole(DataStoreRole.Image); + snapshotDataStoreVO.setDataStoreId(13); + + Mockito.doReturn(List.of(snapshotDataStoreVO)).when(snapshotDataStoreDaoMock).listReadyByVolumeIdAndCheckpointPathNotNull(Mockito.anyLong()); + + ImageStoreVO imageStoreVO = new ImageStoreVO(); + imageStoreVO.setUrl("URL"); + Mockito.doReturn(imageStoreVO).when(imageStoreDaoMock).findById(Mockito.anyLong()); + + Pair, Set> result = volumeOrchestrator.getVolumeCheckpointPathsAndImageStoreUrls(0, Hypervisor.HypervisorType.KVM); + + Assert.assertEquals("Test", result.first().get(0)); + Assert.assertTrue(result.second().contains("URL")); + Assert.assertEquals(1, result.second().size()); + } + } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index ca180e2323fd..d77a54998dc2 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -29,6 +29,7 @@ import com.cloud.resource.ResourceState; import com.cloud.utils.db.GenericDao; import com.cloud.utils.fsm.StateDao; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; /** * Data Access Object for server @@ -167,4 +168,6 @@ public interface HostDao extends GenericDao, StateDao findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags); List findClustersThatMatchHostTagRule(String computeOfferingTags); + + Long findClusterIdByVolumeInfo(VolumeInfo volumeInfo); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 5faa877b458f..768eb0a08ead 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -34,6 +34,8 @@ import javax.inject.Inject; import javax.persistence.TableGenerator; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; import org.apache.commons.collections.CollectionUtils; @@ -73,6 +75,7 @@ import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.UpdateBuilder; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.commons.lang3.ObjectUtils; import java.util.Arrays; @@ -1546,4 +1549,24 @@ private String createSqlFindHostToExecuteCommand(boolean useDisabledHosts) { } return String.format(sqlFindHostInZoneToExecuteCommand, hostResourceStatus); } + + + @Override + public Long findClusterIdByVolumeInfo(VolumeInfo volumeInfo) { + VirtualMachine virtualMachine = volumeInfo.getAttachedVM(); + if (virtualMachine == null) { + return null; + } + + Long hostId = ObjectUtils.defaultIfNull(virtualMachine.getHostId(), virtualMachine.getLastHostId()); + Host host = findById(hostId); + + if (host == null) { + logger.warn(String.format("VM [%s] has null host on DB, either this VM was never started, or there is some inconsistency on the DB.", virtualMachine.getUuid())); + return null; + } + + return host.getClusterId(); + } + } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index 344ff8b2a699..5c3ce2be26a0 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -46,6 +46,12 @@ public interface SnapshotDataStoreDao extends GenericDao listReadyByVolumeIdAndCheckpointPathNotNull(long volumeId); + List listBySnapshot(long snapshotId, DataStoreRole role); List listReadyBySnapshot(long snapshotId, DataStoreRole role); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index c095f4222e76..f75587d0eb43 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -58,6 +58,10 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase searchFilteringStoreIdEqStoreRoleEqStateNeqRefCntNeq; protected SearchBuilder searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq; private SearchBuilder stateSearch; @@ -67,6 +71,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase dataStoreAndInstallPathSearch; private SearchBuilder storeAndSnapshotIdsSearch; private SearchBuilder storeSnapshotDownloadStatusSearch; + private SearchBuilder searchFilteringVolumeIdEqAndStateEqAndKVMCheckpointPathNotNull; protected static final List HYPERVISORS_SUPPORTING_SNAPSHOTS_CHAINING = List.of(Hypervisor.HypervisorType.XenServer); @@ -151,6 +156,12 @@ public boolean configure(String name, Map params) throws Configu storeSnapshotDownloadStatusSearch.and("downloadState", storeSnapshotDownloadStatusSearch.entity().getDownloadState(), SearchCriteria.Op.IN); storeSnapshotDownloadStatusSearch.done(); + searchFilteringVolumeIdEqAndStateEqAndKVMCheckpointPathNotNull = createSearchBuilder(); + searchFilteringVolumeIdEqAndStateEqAndKVMCheckpointPathNotNull.and(VOLUME_ID, searchFilteringVolumeIdEqAndStateEqAndKVMCheckpointPathNotNull.entity().getVolumeId(), SearchCriteria.Op.EQ); + searchFilteringVolumeIdEqAndStateEqAndKVMCheckpointPathNotNull.and(STATE, searchFilteringVolumeIdEqAndStateEqAndKVMCheckpointPathNotNull.entity().getState(), SearchCriteria.Op.EQ); + searchFilteringVolumeIdEqAndStateEqAndKVMCheckpointPathNotNull.and(KVM_CHECKPOINT_PATH, searchFilteringVolumeIdEqAndStateEqAndKVMCheckpointPathNotNull.entity().getKvmCheckpointPath(), SearchCriteria.Op.NNULL); + searchFilteringVolumeIdEqAndStateEqAndKVMCheckpointPathNotNull.done(); + return true; } @@ -283,7 +294,13 @@ protected SnapshotDataStoreVO findOldestOrLatestSnapshotForVolume(long volumeId, @Override @DB public SnapshotDataStoreVO findParent(DataStoreRole role, Long storeId, Long volumeId) { - if (!isSnapshotChainingRequired(volumeId)) { + return findParent(role, storeId, volumeId, false); + } + + @Override + @DB + public SnapshotDataStoreVO findParent(DataStoreRole role, Long storeId, Long volumeId, boolean kvmIncrementalSnapshot) { + if (!isSnapshotChainingRequired(volumeId, kvmIncrementalSnapshot)) { logger.trace(String.format("Snapshot chaining is not required for snapshots of volume [%s]. Returning null as parent.", volumeId)); return null; } @@ -292,13 +309,28 @@ public SnapshotDataStoreVO findParent(DataStoreRole role, Long storeId, Long vol sc.setParameters(VOLUME_ID, volumeId); sc.setParameters(STORE_ROLE, role.toString()); sc.setParameters(STATE, ObjectInDataStoreStateMachine.State.Ready.name()); - sc.setParameters(STORE_ID, storeId); + sc.setParametersIfNotNull(STORE_ID, storeId); List snapshotList = listBy(sc, new Filter(SnapshotDataStoreVO.class, CREATED, false, null, null)); - if (CollectionUtils.isNotEmpty(snapshotList)) { - return snapshotList.get(0); + if (CollectionUtils.isEmpty(snapshotList)) { + return null; } - return null; + + SnapshotDataStoreVO parent = snapshotList.get(0); + + if (kvmIncrementalSnapshot && parent.getKvmCheckpointPath() == null) { + return null; + } + + return parent; + + } + + @Override + public SnapshotDataStoreVO findBySnapshotIdAndDataStoreRoleAndState(long snapshotId, DataStoreRole role, State state) { + SearchCriteria sc = createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, role); + sc.setParameters(STATE, state); + return findOneBy(sc); } @Override @@ -485,13 +517,26 @@ protected SearchCriteria createSearchCriteriaBySnapshotIdAn return sc; } - protected boolean isSnapshotChainingRequired(long volumeId) { + protected boolean isSnapshotChainingRequired(long volumeId, boolean kvmIncrementalSnapshot) { SearchCriteria sc = snapshotVOSearch.create(); sc.setParameters(VOLUME_ID, volumeId); SnapshotVO snapshot = snapshotDao.findOneBy(sc); - return snapshot != null && HYPERVISORS_SUPPORTING_SNAPSHOTS_CHAINING.contains(snapshot.getHypervisorType()); + if (snapshot == null) { + return false; + } + + Hypervisor.HypervisorType hypervisorType = snapshot.getHypervisorType(); + return HYPERVISORS_SUPPORTING_SNAPSHOTS_CHAINING.contains(hypervisorType) || (Hypervisor.HypervisorType.KVM.equals(hypervisorType) && kvmIncrementalSnapshot); + } + + @Override + public List listReadyByVolumeIdAndCheckpointPathNotNull(long volumeId) { + SearchCriteria sc = searchFilteringVolumeIdEqAndStateEqAndKVMCheckpointPathNotNull.create(); + sc.setParameters(VOLUME_ID, volumeId); + sc.setParameters(STATE, State.Ready); + return listBy(sc); } @Override diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java index a1dc05fce58b..f2a8d3712f61 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java @@ -29,6 +29,8 @@ import javax.persistence.Temporal; import javax.persistence.TemporalType; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -80,12 +82,18 @@ public class SnapshotDataStoreVO implements StateObject= deltaSnap) { - fullBackup = true; - } else { - fullBackup = false; - } + fullBackup = i >= deltaSnap; } else if (oldestSnapshotOnPrimary.getId() != parentSnapshotOnPrimaryStore.getId()){ // if there is an snapshot entry for previousPool(primary storage) of migrated volume, delete it because CS created one more snapshot entry for current pool snapshotStoreDao.remove(oldestSnapshotOnPrimary.getId()); @@ -323,11 +316,30 @@ protected boolean destroySnapshotEntriesAndFiles(SnapshotVO snapshotVo, Long zon } /** - * Updates the snapshot to {@link Snapshot.State#Destroyed}. + * Updates the snapshot to {@link Snapshot.State#Destroyed}. If using the KVM hypervisor and the snapshot was the end of a chain, + * will mark their parents as end of chain as well. */ protected void updateSnapshotToDestroyed(SnapshotVO snapshotVo) { snapshotVo.setState(Snapshot.State.Destroyed); snapshotDao.update(snapshotVo.getId(), snapshotVo); + + if (!HypervisorType.KVM.equals(snapshotVo.getHypervisorType())) { + return; + } + + List snapshotDataStoreVoList = snapshotStoreDao.findBySnapshotId(snapshotVo.getSnapshotId()); + SnapshotDataStoreVO snapshotDataStoreVo = snapshotDataStoreVoList.get(0); + + if (!snapshotDataStoreVo.isEndOfChain()) { + return; + } + + List parentSnapshotDataStoreVoList = snapshotStoreDao.findBySnapshotId(snapshotDataStoreVo.getParentSnapshotId()); + for (SnapshotDataStoreVO parentSnapshotDatastoreVo : parentSnapshotDataStoreVoList) { + parentSnapshotDatastoreVo.setEndOfChain(true); + snapshotStoreDao.update(parentSnapshotDatastoreVo.getId(), parentSnapshotDatastoreVo); + } + } protected boolean deleteSnapshotInfos(SnapshotVO snapshotVo, Long zoneId) { @@ -357,52 +369,24 @@ protected Boolean deleteSnapshotInfo(SnapshotInfo snapshotInfo, SnapshotVO snaps if (isLastSnapshotRef) { snapshotObject.processEvent(Snapshot.Event.DestroyRequested); } - if (!DataStoreRole.Primary.equals(dataStore.getRole())) { - verifyIfTheSnapshotIsBeingUsedByAnyVolume(snapshotObject); - if (deleteSnapshotChain(snapshotInfo, storageToString)) { - logger.debug(String.format("%s was deleted on %s. We will mark the snapshot as destroyed.", snapshotVo, storageToString)); - } else { - logger.debug(String.format("%s was not deleted on %s; however, we will mark the snapshot as destroyed for future garbage collecting.", snapshotVo, - storageToString)); - } - snapshotStoreDao.updateDisplayForSnapshotStoreRole(snapshotVo.getId(), dataStore.getId(), dataStore.getRole(), false); - if (isLastSnapshotRef) { - snapshotObject.processEvent(Snapshot.Event.OperationSucceeded); - } - return true; - } else if (deleteSnapshotInPrimaryStorage(snapshotInfo, snapshotVo, storageToString, snapshotObject, isLastSnapshotRef)) { - snapshotStoreDao.updateDisplayForSnapshotStoreRole(snapshotVo.getId(), dataStore.getId(), dataStore.getRole(), false); - return true; + verifyIfTheSnapshotIsBeingUsedByAnyVolume(snapshotObject); + if (deleteSnapshotChain(snapshotInfo, storageToString)) { + logger.debug(String.format("%s was deleted on %s. We will mark the snapshot as destroyed.", snapshotVo, storageToString)); + } else { + logger.debug(String.format("%s was not deleted on %s; however, we will mark the snapshot as destroyed for future garbage collecting.", snapshotVo, + storageToString)); } - logger.debug(String.format("Failed to delete %s on %s.", snapshotVo, storageToString)); + snapshotStoreDao.updateDisplayForSnapshotStoreRole(snapshotVo.getId(), dataStore.getId(), dataStore.getRole(), false); if (isLastSnapshotRef) { - snapshotObject.processEvent(Snapshot.Event.OperationFailed); + snapshotObject.processEvent(Snapshot.Event.OperationSucceeded); } + return true; } catch (NoTransitionException ex) { logger.warn(String.format("Failed to delete %s on %s due to %s.", snapshotVo, storageToString, ex.getMessage()), ex); } return false; } - protected boolean deleteSnapshotInPrimaryStorage(SnapshotInfo snapshotInfo, SnapshotVO snapshotVo, - String storageToString, SnapshotObject snapshotObject, boolean isLastSnapshotRef) throws NoTransitionException { - try { - if (snapshotSvr.deleteSnapshot(snapshotInfo)) { - String msg = String.format("%s was deleted on %s.", snapshotVo, storageToString); - if (isLastSnapshotRef) { - msg = String.format("%s We will mark the snapshot as destroyed.", msg); - snapshotObject.processEvent(Snapshot.Event.OperationSucceeded); - } - logger.debug(msg); - return true; - } - } catch (CloudRuntimeException ex) { - logger.warn(String.format("Unable do delete snapshot %s on %s due to [%s]. The reference will be marked as 'Destroying' for future garbage collecting.", - snapshotVo, storageToString, ex.getMessage()), ex); - } - return false; - } - protected void verifyIfTheSnapshotIsBeingUsedByAnyVolume(SnapshotObject snapshotObject) throws NoTransitionException { List volumesFromSnapshot = _volumeDetailsDaoImpl.findDetails("SNAPSHOT_ID", String.valueOf(snapshotObject.getSnapshotId()), null); if (CollectionUtils.isEmpty(volumesFromSnapshot)) { diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index 961a647d7a8c..e4bcf43d2742 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -64,8 +64,11 @@ public class SnapshotObject implements SnapshotInfo { protected Logger logger = LogManager.getLogger(getClass()); private SnapshotVO snapshot; private DataStore store; + private DataStore imageStore; private Object payload; private Boolean fullBackup; + private String checkpointPath; + private boolean kvmIncrementalSnapshot = false; private String url; @Inject protected SnapshotDao snapshotDao; @@ -112,10 +115,12 @@ public DataStore getStore() { public SnapshotInfo getParent() { SnapshotDataStoreVO snapStoreVO = snapshotStoreDao.findByStoreSnapshot(store.getRole(), store.getId(), snapshot.getId()); - Long parentId = null; if (snapStoreVO != null) { - parentId = snapStoreVO.getParentSnapshotId(); - if (parentId != null && parentId != 0) { + long parentId = snapStoreVO.getParentSnapshotId(); + if (parentId != 0) { + if (HypervisorType.KVM.equals(snapshot.getHypervisorType())) { + return getParentInfoFromImageIfPossible(parentId); + } return snapshotFactory.getSnapshot(parentId, store); } } @@ -123,6 +128,32 @@ public SnapshotInfo getParent() { return null; } + /** + * Returns the snapshotInfo of the passed snapshot parentId. If the parent is on image store, will return the info from image store; + * else, will return the info from primary store. + * */ + protected SnapshotInfo getParentInfoFromImageIfPossible(long parentId) { + List parentSnapshotDatastoreVos = snapshotStoreDao.findBySnapshotId(parentId); + + if (parentSnapshotDatastoreVos.isEmpty()) { + return null; + } + + SnapshotDataStoreVO parent = null; + + for (SnapshotDataStoreVO parentSnapshotDatastoreVo : parentSnapshotDatastoreVos) { + parent = parentSnapshotDatastoreVo; + if (parentSnapshotDatastoreVo.getRole().equals(DataStoreRole.Image)) { + break; + } + } + + SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(parentId, parent.getDataStoreId(), parent.getRole()); + snapshotInfo.setKvmIncrementalSnapshot(parent.getKvmCheckpointPath() != null); + + return snapshotInfo; + } + @Override public SnapshotInfo getChild() { QueryBuilder sc = QueryBuilder.create(SnapshotDataStoreVO.class); @@ -216,6 +247,16 @@ public DataStore getDataStore() { return store; } + @Override + public DataStore getImageStore() { + return imageStore; + } + + @Override + public void setImageStore(DataStore imageStore) { + this.imageStore = imageStore; + } + @Override public Long getSize() { return snapshot.getSize(); @@ -454,6 +495,26 @@ public Boolean getFullBackup() { return fullBackup; } + @Override + public String getCheckpointPath() { + return checkpointPath; + } + + @Override + public void setCheckpointPath(String checkpointPath) { + this.checkpointPath = checkpointPath; + } + + @Override + public void setKvmIncrementalSnapshot(boolean isKvmIncrementalSnapshot) { + this.kvmIncrementalSnapshot = isKvmIncrementalSnapshot; + } + + @Override + public boolean isKvmIncrementalSnapshot() { + return kvmIncrementalSnapshot; + } + @Override public boolean delete() { if (store != null) { diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index dafc40e0674d..af86e20ea2e4 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -22,6 +22,10 @@ import javax.inject.Inject; +import com.cloud.agent.api.ConvertSnapshotAnswer; +import com.cloud.agent.api.ConvertSnapshotCommand; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionService; @@ -38,6 +42,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageAction; import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; @@ -50,6 +55,7 @@ import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; @@ -57,6 +63,8 @@ import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -101,6 +109,8 @@ public class SnapshotServiceImpl implements SnapshotService { EndPointSelector epSelector; @Inject ConfigurationDao _configDao; + @Inject + HostDao hostDao; @Inject private HeuristicRuleHelper heuristicRuleHelper; @@ -232,27 +242,27 @@ protected Void createSnapshotAsyncCallback(AsyncCallbackDispatcher future = new AsyncCallFuture(); try { - CreateSnapshotContext context = new CreateSnapshotContext(null, snap.getBaseVolume(), snapshotOnPrimary, future); + CreateSnapshotContext context = new CreateSnapshotContext(null, snap.getBaseVolume(), snapshotOnStorage, future); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createSnapshotAsyncCallback(null, null)).setContext(context); - PrimaryDataStoreDriver primaryStore = (PrimaryDataStoreDriver)snapshotOnPrimary.getDataStore().getDriver(); + PrimaryDataStoreDriver primaryStore = (PrimaryDataStoreDriver)snapshotOnStorage.getDataStore().getDriver(); primaryStore.takeSnapshot(snapshot, caller); } catch (Exception e) { logger.debug("Failed to take snapshot: " + snapshot.getId(), e); @@ -281,22 +291,56 @@ public SnapshotResult takeSnapshot(SnapshotInfo snap) { try { result = future.get(); + + updateSnapSizeAndCheckpointPathIfPossible(result, snap); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_ON_PRIMARY, snap.getAccountId(), snap.getDataCenterId(), snap.getId(), - snap.getName(), null, null, snapshotOnPrimary.getSize(), snapshotOnPrimary.getSize(), snap.getClass().getName(), snap.getUuid()); + snap.getName(), null, null, snapshotOnStorage.getSize(), snapshotOnStorage.getSize(), snap.getClass().getName(), snap.getUuid()); return result; - } catch (InterruptedException e) { - logger.debug("Failed to create snapshot", e); - throw new CloudRuntimeException("Failed to create snapshot", e); - } catch (ExecutionException e) { - logger.debug("Failed to create snapshot", e); - throw new CloudRuntimeException("Failed to create snapshot", e); + } catch (InterruptedException | ExecutionException e) { + String message = String.format("Failed to create snapshot [%s] due to [%s].", snapshot, e.getMessage()); + logger.error(message, e); + throw new CloudRuntimeException(message, e); } } + /** + * Updates the snapshot physical size if the answer is an instance of CreateObjectAnswer and the returned physical size if bigger than 0. + * Also updates the checkpoint path if possible. + * */ + protected void updateSnapSizeAndCheckpointPathIfPossible(SnapshotResult result, SnapshotInfo snapshotInfo) { + SnapshotDataStoreVO snapshotStore; + Answer answer = result.getAnswer(); + + if (!answer.getResult() || !(answer instanceof CreateObjectAnswer)) { + return; + } + + SnapshotInfo resultSnapshot = result.getSnapshot(); + if (snapshotInfo.getImageStore() != null) { + snapshotInfo.getImageStore().create(resultSnapshot); + snapshotStore = _snapshotStoreDao.findBySnapshotIdAndDataStoreRoleAndState(resultSnapshot.getSnapshotId(), DataStoreRole.Image, ObjectInDataStoreStateMachine.State.Allocated); + } else { + snapshotStore = _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Primary, resultSnapshot.getDataStore().getId(), resultSnapshot.getSnapshotId()); + } + + SnapshotObjectTO snapshotObjectTo = (SnapshotObjectTO) ((CreateObjectAnswer) answer).getData(); + + Long physicalSize = snapshotObjectTo.getPhysicalSize(); + if (NumberUtils.compare(physicalSize, 0L) > 0) { + snapshotStore.setPhysicalSize(physicalSize); + } + + snapshotStore.setKvmCheckpointPath(snapshotObjectTo.getCheckpointPath()); + _snapshotStoreDao.update(snapshotStore.getId(), snapshotStore); + } + + // if a snapshot has parent snapshot, the new snapshot should be stored in // the same store as its parent since // we are taking delta snapshot - private DataStore findSnapshotImageStore(SnapshotInfo snapshot) { + @Override + public DataStore findSnapshotImageStore(SnapshotInfo snapshot) { Boolean fullSnapshot = true; Boolean snapshotFullBackup = snapshot.getFullBackup(); if (snapshotFullBackup != null) { @@ -339,6 +383,40 @@ protected DataStore getImageStoreForSnapshot(Long dataCenterId, SnapshotInfo sna return imageStore; } + /** + * Converts a given snapshot that is on the secondary storage. The original and its backing chains will be maintained, the converted snapshot must be later deleted if not used. + * The original purpose of this method is to work with KVM incremental snapshots, copying the snapshot to a temporary location and consolidating the snapshot chain. + * @param snapshotInfo The snapshot to be converted + * @return the snapshotInfo given with the updated path. This should not be persisted on the DB, otherwise the original snapshot will be lost. + * */ + @Override + public SnapshotInfo convertSnapshot(SnapshotInfo snapshotInfo) { + SnapshotObject snapObj = (SnapshotObject)snapshotInfo; + + Answer answer = null; + try { + snapObj.processEvent(Snapshot.Event.BackupToSecondary); + + SnapshotObjectTO snapshotObjectTO = (SnapshotObjectTO) snapshotInfo.getTO(); + ConvertSnapshotCommand cmd = new ConvertSnapshotCommand(snapshotObjectTO); + + EndPoint ep = epSelector.select(snapshotInfo, StorageAction.CONVERTSNAPSHOT); + + answer = ep.sendMessage(cmd); + + if (answer != null && answer.getResult()) { + snapObj.processEvent(Snapshot.Event.OperationSucceeded); + snapObj.setPath(((ConvertSnapshotAnswer) answer).getSnapshotObjectTO().getPath()); + return snapObj; + } + snapObj.processEvent(Snapshot.Event.OperationNotPerformed); + } catch (NoTransitionException e) { + logger.debug("Failed to change snapshot {} state.", snapObj.getUuid(), e); + } + + throw new CloudRuntimeException(String.format("Failed to convert snapshot [%s]%s.", snapObj.getUuid(), answer != null ? String.format(" due to [%s]", answer.getDetails()) : "")); + } + @Override public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { SnapshotObject snapObj = (SnapshotObject)snapshot; @@ -404,7 +482,7 @@ protected Void copySnapshotAsyncCallback(AsyncCallbackDispatcher future = new AsyncCallFuture(); DeleteSnapshotContext context = new DeleteSnapshotContext(null, snapInfo, future); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java index 09e5c85b770f..7669ddc29482 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java @@ -20,12 +20,12 @@ import java.util.ArrayList; import java.util.List; +import com.cloud.hypervisor.Hypervisor; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.junit.Assert; @@ -77,7 +77,7 @@ public class DefaultSnapshotStrategyTest { VolumeDetailsDao volumeDetailsDaoMock; @Mock - SnapshotService snapshotServiceMock; + private SnapshotDataStoreVO snapshotDataStoreVOMock; @Mock SnapshotZoneDao snapshotZoneDaoMock; @@ -106,6 +106,51 @@ public void validateRetrieveSnapshotEntries() { Assert.assertTrue(result.contains(snapshotInfo2Mock)); } + @Test + public void updateSnapshotToDestroyedTestNotKvm() { + Mockito.doReturn(Hypervisor.HypervisorType.VMware).when(snapshotVoMock).getHypervisorType(); + + defaultSnapshotStrategySpy.updateSnapshotToDestroyed(snapshotVoMock); + + Mockito.verify(snapshotVoMock).setState(Snapshot.State.Destroyed); + Mockito.verify(snapshotVoMock, Mockito.never()).getSnapshotId(); + } + + @Test + public void updateSnapshotToDestroyedTestKvmAndIsNotEndOfChain() { + Mockito.doReturn(Hypervisor.HypervisorType.KVM).when(snapshotVoMock).getHypervisorType(); + Mockito.doReturn(2L).when(snapshotVoMock).getSnapshotId(); + + SnapshotDataStoreVO snapshotDataStoreVO = new SnapshotDataStoreVO(); + snapshotDataStoreVO.setEndOfChain(false); + Mockito.doReturn(List.of(snapshotDataStoreVO)).when(snapshotDataStoreDao).findBySnapshotId(2); + + defaultSnapshotStrategySpy.updateSnapshotToDestroyed(snapshotVoMock); + + Mockito.verify(snapshotVoMock).setState(Snapshot.State.Destroyed); + Mockito.verify(snapshotDataStoreDao, Mockito.times(1)).findBySnapshotId(Mockito.anyLong()); + } + + @Test + public void updateSnapshotToDestroyedTestKvmAndIsEndOfChain() { + Mockito.doReturn(Hypervisor.HypervisorType.KVM).when(snapshotVoMock).getHypervisorType(); + Mockito.doReturn(2L).when(snapshotVoMock).getSnapshotId(); + + SnapshotDataStoreVO snapshotDataStoreVO = new SnapshotDataStoreVO(); + snapshotDataStoreVO.setEndOfChain(true); + snapshotDataStoreVO.setParentSnapshotId(8); + Mockito.doReturn(List.of(snapshotDataStoreVO)).when(snapshotDataStoreDao).findBySnapshotId(2); + + Mockito.doReturn(List.of(snapshotDataStoreVOMock)).when(snapshotDataStoreDao).findBySnapshotId(8); + + defaultSnapshotStrategySpy.updateSnapshotToDestroyed(snapshotVoMock); + + Mockito.verify(snapshotVoMock).setState(Snapshot.State.Destroyed); + Mockito.verify(snapshotDataStoreDao, Mockito.times(2)).findBySnapshotId(Mockito.anyLong()); + Mockito.verify(snapshotDataStoreVOMock).setEndOfChain(true); + Mockito.verify(snapshotDataStoreDao).update(Mockito.anyLong(), Mockito.any()); + } + @Test public void validateUpdateSnapshotToDestroyed() { Mockito.doReturn(true).when(snapshotDaoMock).update(Mockito.anyLong(), Mockito.any()); @@ -146,37 +191,12 @@ public void deleteSnapshotInfoTestReturnTrueIfCanDeleteTheSnapshotOnPrimaryStora Mockito.doReturn(dataStoreMock).when(snapshotInfo1Mock).getDataStore(); Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); - Mockito.doReturn(true).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Primary); boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); Assert.assertTrue(result); } - @Test - public void deleteSnapshotInfoTestReturnFalseIfCannotDeleteTheSnapshotOnPrimaryStorage() throws NoTransitionException { - Mockito.doReturn(dataStoreMock).when(snapshotInfo1Mock).getDataStore(); - Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); - Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); - Mockito.doReturn(false).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Primary); - - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); - Assert.assertFalse(result); - } - - @Test - public void deleteSnapshotInfoTestReturnFalseIfDeleteSnapshotOnPrimaryStorageThrowsACloudRuntimeException() throws NoTransitionException { - Mockito.doReturn(dataStoreMock).when(snapshotInfo1Mock).getDataStore(); - Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); - Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); - Mockito.doThrow(CloudRuntimeException.class).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Primary); - - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); - Assert.assertFalse(result); - } - @Test public void deleteSnapshotInfoTestReturnTrueIfCanDeleteTheSnapshotChainForSecondaryStorage() throws NoTransitionException { Mockito.doReturn(dataStoreMock).when(snapshotInfo1Mock).getDataStore(); @@ -233,31 +253,6 @@ public void verifyIfTheSnapshotIsBeingUsedByAnyVolumeTestDetailsIsNotEmptyThrowC defaultSnapshotStrategySpy.verifyIfTheSnapshotIsBeingUsedByAnyVolume(snapshotObjectMock); } - @Test - public void deleteSnapshotInPrimaryStorageTestReturnTrueIfDeleteReturnsTrue() throws NoTransitionException { - Mockito.doReturn(true).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); - Assert.assertTrue(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, snapshotObjectMock, true)); - } - - @Test - public void deleteSnapshotInPrimaryStorageTestReturnTrueIfDeleteNotLastRefReturnsTrue() throws NoTransitionException { - Mockito.doReturn(true).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Assert.assertTrue(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, snapshotObjectMock, false)); - } - - @Test - public void deleteSnapshotInPrimaryStorageTestReturnFalseIfDeleteReturnsFalse() throws NoTransitionException { - Mockito.doReturn(false).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, null, true)); - } - - @Test - public void deleteSnapshotInPrimaryStorageTestReturnFalseIfDeleteThrowsException() throws NoTransitionException { - Mockito.doThrow(CloudRuntimeException.class).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, null, true)); - } - @Test public void testGetSnapshotImageStoreRefNull() { SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java index 6d59b6f36e0a..e32725ebb23d 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java @@ -18,7 +18,9 @@ */ package org.apache.cloudstack.storage.snapshot; +import com.cloud.agent.api.Answer; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ImageStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; @@ -30,7 +32,11 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.storage.command.CreateObjectAnswer; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -61,7 +67,7 @@ public class SnapshotServiceImplTest { HeuristicRuleHelper heuristicRuleHelperMock; @Mock - SnapshotInfo snapshotMock; + SnapshotInfo snapshotInfoMock; @Mock VolumeInfo volumeInfoMock; @@ -69,12 +75,34 @@ public class SnapshotServiceImplTest { @Mock DataStoreManager dataStoreManagerMock; + @Mock + private SnapshotResult snapshotResultMock; + + @Mock + private CreateObjectAnswer createObjectAnswerMock; + + @Mock + private ImageStore imageStoreMock; + + @Mock + private SnapshotDataStoreVO snapshotDataStoreVoMock; + + @Mock + private SnapshotDataStoreDao snapshotDataStoreDaoMock; + + @Mock + private DataStore dataStoreMock; + + @Mock + private SnapshotObjectTO snapshotObjectTOMock; + + private static final long DUMMY_ID = 1L; @Test public void testRevertSnapshotWithNoPrimaryStorageEntry() throws Exception { - Mockito.when(snapshotMock.getId()).thenReturn(DUMMY_ID); - Mockito.when(snapshotMock.getVolumeId()).thenReturn(DUMMY_ID); + Mockito.when(snapshotInfoMock.getId()).thenReturn(DUMMY_ID); + Mockito.when(snapshotInfoMock.getVolumeId()).thenReturn(DUMMY_ID); Mockito.when(_snapshotFactory.getSnapshotOnPrimaryStore(1L)).thenReturn(null); Mockito.when(volFactory.getVolume(DUMMY_ID, DataStoreRole.Primary)).thenReturn(volumeInfoMock); @@ -89,7 +117,7 @@ public void testRevertSnapshotWithNoPrimaryStorageEntry() throws Exception { Mockito.when(mock.get()).thenReturn(result); Mockito.when(result.isFailed()).thenReturn(false); })) { - Assert.assertTrue(snapshotService.revertSnapshot(snapshotMock)); + Assert.assertTrue(snapshotService.revertSnapshot(snapshotInfoMock)); } } @@ -97,9 +125,9 @@ public void testRevertSnapshotWithNoPrimaryStorageEntry() throws Exception { public void getImageStoreForSnapshotTestShouldListFreeImageStoresWithNoHeuristicRule() { Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))). thenReturn(null); - Mockito.when(snapshotMock.getDataCenterId()).thenReturn(DUMMY_ID); + Mockito.when(snapshotInfoMock.getDataCenterId()).thenReturn(DUMMY_ID); - snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock); + snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotInfoMock); Mockito.verify(dataStoreManagerMock, Mockito.times(1)).getImageStoreWithFreeCapacity(Mockito.anyLong()); } @@ -110,8 +138,123 @@ public void getImageStoreForSnapshotTestShouldReturnImageStoreReturnedByTheHeuri Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))). thenReturn(dataStore); - snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock); + snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotInfoMock); Mockito.verify(dataStoreManagerMock, Mockito.times(0)).getImageStoreWithFreeCapacity(Mockito.anyLong()); } + + @Test + public void updateSnapSizeAndCheckpointPathIfPossibleTestResultIsFalse() { + Mockito.doReturn(createObjectAnswerMock).when(snapshotResultMock).getAnswer(); + Mockito.doReturn(false).when(createObjectAnswerMock).getResult(); + + snapshotService.updateSnapSizeAndCheckpointPathIfPossible(snapshotResultMock, snapshotInfoMock); + + Mockito.verify(snapshotInfoMock, Mockito.never()).getImageStore(); + } + + @Test + public void updateSnapSizeAndCheckpointPathIfPossibleTestResultIsTrueAnswerIsNotCreateObjectAnswer() { + Answer answer = new Answer(null, true, null); + + Mockito.doReturn(answer).when(snapshotResultMock).getAnswer(); + + snapshotService.updateSnapSizeAndCheckpointPathIfPossible(snapshotResultMock, snapshotInfoMock); + + Mockito.verify(snapshotInfoMock, Mockito.never()).getImageStore(); + } + + @Test + public void updateSnapSizeAndCheckpointPathIfPossibleTestResultIsTrueAnswerIsCreateObjectAnswerAndImageStoreIsNotNullAndPhysicalSizeIsZero() { + Mockito.doReturn(createObjectAnswerMock).when(snapshotResultMock).getAnswer(); + Mockito.doReturn(true).when(createObjectAnswerMock).getResult(); + + Mockito.doReturn(snapshotInfoMock).when(snapshotResultMock).getSnapshot(); + + Mockito.doReturn(dataStoreMock).when(snapshotInfoMock).getImageStore(); + + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findBySnapshotIdAndDataStoreRoleAndState(Mockito.anyLong(), Mockito.any(), Mockito.any()); + + Mockito.doReturn(snapshotObjectTOMock).when(createObjectAnswerMock).getData(); + Mockito.doReturn("checkpath").when(snapshotObjectTOMock).getCheckpointPath(); + Mockito.doReturn(0L).when(snapshotObjectTOMock).getPhysicalSize(); + + snapshotService.updateSnapSizeAndCheckpointPathIfPossible(snapshotResultMock, snapshotInfoMock); + + Mockito.verify(snapshotDataStoreVoMock).setKvmCheckpointPath("checkpath"); + Mockito.verify(snapshotDataStoreVoMock, Mockito.never()).setPhysicalSize(Mockito.anyLong()); + + Mockito.verify(snapshotDataStoreDaoMock).update(Mockito.anyLong(), Mockito.any()); + } + + @Test + public void updateSnapSizeAndCheckpointPathIfPossibleTestResultIsTrueAnswerIsCreateObjectAnswerAndImageStoreIsNotNullAndPhysicalSizeGreaterThanZero() { + Mockito.doReturn(createObjectAnswerMock).when(snapshotResultMock).getAnswer(); + Mockito.doReturn(true).when(createObjectAnswerMock).getResult(); + + Mockito.doReturn(snapshotInfoMock).when(snapshotResultMock).getSnapshot(); + + Mockito.doReturn(dataStoreMock).when(snapshotInfoMock).getImageStore(); + + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findBySnapshotIdAndDataStoreRoleAndState(Mockito.anyLong(), Mockito.any(), Mockito.any()); + + Mockito.doReturn(snapshotObjectTOMock).when(createObjectAnswerMock).getData(); + Mockito.doReturn("checkpath").when(snapshotObjectTOMock).getCheckpointPath(); + Mockito.doReturn(1000L).when(snapshotObjectTOMock).getPhysicalSize(); + + snapshotService.updateSnapSizeAndCheckpointPathIfPossible(snapshotResultMock, snapshotInfoMock); + + Mockito.verify(snapshotDataStoreVoMock).setKvmCheckpointPath("checkpath"); + Mockito.verify(snapshotDataStoreVoMock).setPhysicalSize(1000L); + + Mockito.verify(snapshotDataStoreDaoMock).update(Mockito.anyLong(), Mockito.any()); + } + + @Test + public void updateSnapSizeAndCheckpointPathIfPossibleTestResultIsTrueAnswerIsCreateObjectAnswerAndImageStoreIsNullAndPhysicalSizeIsZero() { + Mockito.doReturn(createObjectAnswerMock).when(snapshotResultMock).getAnswer(); + Mockito.doReturn(true).when(createObjectAnswerMock).getResult(); + + Mockito.doReturn(snapshotInfoMock).when(snapshotResultMock).getSnapshot(); + + Mockito.doReturn(null).when(snapshotInfoMock).getImageStore(); + Mockito.doReturn(dataStoreMock).when(snapshotInfoMock).getDataStore(); + + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findByStoreSnapshot(Mockito.any(), Mockito.anyLong(), Mockito.anyLong()); + + Mockito.doReturn(snapshotObjectTOMock).when(createObjectAnswerMock).getData(); + Mockito.doReturn("checkpath").when(snapshotObjectTOMock).getCheckpointPath(); + Mockito.doReturn(0L).when(snapshotObjectTOMock).getPhysicalSize(); + + snapshotService.updateSnapSizeAndCheckpointPathIfPossible(snapshotResultMock, snapshotInfoMock); + + Mockito.verify(snapshotDataStoreVoMock).setKvmCheckpointPath("checkpath"); + Mockito.verify(snapshotDataStoreVoMock, Mockito.never()).setPhysicalSize(Mockito.anyLong()); + + Mockito.verify(snapshotDataStoreDaoMock).update(Mockito.anyLong(), Mockito.any()); + } + + @Test + public void updateSnapSizeAndCheckpointPathIfPossibleTestResultIsTrueAnswerIsCreateObjectAnswerAndImageStoreIsNullAndPhysicalSizeGreaterThanZero() { + Mockito.doReturn(createObjectAnswerMock).when(snapshotResultMock).getAnswer(); + Mockito.doReturn(true).when(createObjectAnswerMock).getResult(); + + Mockito.doReturn(snapshotInfoMock).when(snapshotResultMock).getSnapshot(); + + Mockito.doReturn(null).when(snapshotInfoMock).getImageStore(); + Mockito.doReturn(dataStoreMock).when(snapshotInfoMock).getDataStore(); + + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findByStoreSnapshot(Mockito.any(), Mockito.anyLong(), Mockito.anyLong()); + + Mockito.doReturn(snapshotObjectTOMock).when(createObjectAnswerMock).getData(); + Mockito.doReturn("checkpath").when(snapshotObjectTOMock).getCheckpointPath(); + Mockito.doReturn(1000L).when(snapshotObjectTOMock).getPhysicalSize(); + + snapshotService.updateSnapSizeAndCheckpointPathIfPossible(snapshotResultMock, snapshotInfoMock); + + Mockito.verify(snapshotDataStoreVoMock).setKvmCheckpointPath("checkpath"); + Mockito.verify(snapshotDataStoreVoMock).setPhysicalSize(1000L); + + Mockito.verify(snapshotDataStoreDaoMock).update(Mockito.anyLong(), Mockito.any()); + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java index 570a47a752c6..9f861c0d3e2e 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java @@ -18,6 +18,9 @@ import javax.inject.Inject; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.snapshot.SnapshotManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.springframework.stereotype.Component; @@ -84,6 +87,10 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { SnapshotDao snapshotDao; @Inject VolumeDao volumeDao; + + @Inject + HostDao hostDao; + protected StateMachine2 stateMachines; public ObjectInDataStoreManagerImpl() { @@ -130,12 +137,17 @@ public DataObject create(DataObject obj, DataStore dataStore) { ss.setVolumeId(snapshotInfo.getVolumeId()); ss.setSize(snapshotInfo.getSize()); // this is the virtual size of snapshot in primary storage. ss.setPhysicalSize(snapshotInfo.getSize()); // this physical size will get updated with actual size once the snapshot backup is done. - SnapshotDataStoreVO snapshotDataStoreVO = snapshotDataStoreDao.findParent(dataStore.getRole(), dataStore.getId(), snapshotInfo.getVolumeId()); + Long clusterId = hostDao.findClusterIdByVolumeInfo(snapshotInfo.getBaseVolume()); + boolean kvmIncrementalSnapshot = SnapshotManager.kvmIncrementalSnapshot.valueIn(clusterId); + SnapshotDataStoreVO snapshotDataStoreVO = snapshotDataStoreDao.findParent(dataStore.getRole(), dataStore.getId(), snapshotInfo.getVolumeId(), kvmIncrementalSnapshot); + snapshotDataStoreVO = tryToGetSnapshotOnSecondaryIfNotOnPrimaryAndIsKVM(dataStore, snapshotInfo, snapshotDataStoreVO, kvmIncrementalSnapshot); if (snapshotDataStoreVO != null) { //Double check the snapshot is removed or not SnapshotVO parentSnap = snapshotDao.findById(snapshotDataStoreVO.getSnapshotId()); - if (parentSnap != null) { + if (parentSnap != null && !snapshotDataStoreVO.isEndOfChain()) { ss.setParentSnapshotId(snapshotDataStoreVO.getSnapshotId()); + } else if (snapshotDataStoreVO.isEndOfChain()) { + logger.debug("Snapshot [{}] will begin a new chain, as the last one has finished.", ss.getSnapshotId()); } else { logger.debug("find inconsistent db for snapshot " + snapshotDataStoreVO.getSnapshotId()); } @@ -179,9 +191,12 @@ public DataObject create(DataObject obj, DataStore dataStore) { ss.setRole(dataStore.getRole()); ss.setSize(snapshot.getSize()); ss.setVolumeId(snapshot.getVolumeId()); - SnapshotDataStoreVO snapshotDataStoreVO = snapshotDataStoreDao.findParent(dataStore.getRole(), dataStore.getId(), snapshot.getVolumeId()); - if (snapshotDataStoreVO != null) { + Long clusterId = hostDao.findClusterIdByVolumeInfo(snapshot.getBaseVolume()); + SnapshotDataStoreVO snapshotDataStoreVO = snapshotDataStoreDao.findParent(dataStore.getRole(), dataStore.getId(), snapshot.getVolumeId(), SnapshotManager.kvmIncrementalSnapshot.valueIn(clusterId)); + if (snapshotDataStoreVO != null && !snapshotDataStoreVO.isEndOfChain()) { ss.setParentSnapshotId(snapshotDataStoreVO.getSnapshotId()); + } else { + logger.debug("Snapshot [{}] will begin a new chain, as the last one has finished.", ss.getSnapshotId()); } ss.setInstallPath(TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR + "/" + snapshotDao.findById(obj.getId()).getAccountId() + "/" + snapshot.getVolumeId()); ss.setState(ObjectInDataStoreStateMachine.State.Allocated); @@ -201,6 +216,13 @@ public DataObject create(DataObject obj, DataStore dataStore) { return this.get(obj, dataStore, null); } + private SnapshotDataStoreVO tryToGetSnapshotOnSecondaryIfNotOnPrimaryAndIsKVM(DataStore dataStore, SnapshotInfo snapshotInfo, SnapshotDataStoreVO snapshotDataStoreVO, boolean kvmIncrementalSnapshot) { + if (snapshotDataStoreVO == null && Hypervisor.HypervisorType.KVM.equals(snapshotInfo.getHypervisorType()) && DataStoreRole.Primary.equals(dataStore.getRole())) { + snapshotDataStoreVO = snapshotDataStoreDao.findParent(DataStoreRole.Image, null, snapshotInfo.getVolumeId(), kvmIncrementalSnapshot); + } + return snapshotDataStoreVO; + } + @Override public boolean delete(DataObject dataObj) { long objId = dataObj.getId(); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index a621e8a076d1..4bb8d0052d1c 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -461,35 +461,48 @@ public EndPoint select(DataObject object, StorageAction action) { @Override public EndPoint select(DataObject object, StorageAction action, boolean encryptionRequired) { - if (action == StorageAction.TAKESNAPSHOT) { - SnapshotInfo snapshotInfo = (SnapshotInfo)object; - if (snapshotInfo.getHypervisorType() == Hypervisor.HypervisorType.KVM) { - VolumeInfo volumeInfo = snapshotInfo.getBaseVolume(); - VirtualMachine vm = volumeInfo.getAttachedVM(); - if ((vm != null) && (vm.getState() == VirtualMachine.State.Running)) { - Long hostId = vm.getHostId(); - return getEndPointFromHostId(hostId); + switch (action) { + case DELETESNAPSHOT: + case TAKESNAPSHOT: + case CONVERTSNAPSHOT: + SnapshotInfo snapshotInfo = (SnapshotInfo) object; + if (Hypervisor.HypervisorType.KVM.equals(snapshotInfo.getHypervisorType())) { + VolumeInfo volumeInfo = snapshotInfo.getBaseVolume(); + VirtualMachine vm = volumeInfo.getAttachedVM(); + if (vm == null) { + break; + } + if (vm.getState() == VirtualMachine.State.Running) { + Long hostId = vm.getHostId(); + return getEndPointFromHostId(hostId); + } + Long hostId = vm.getLastHostId(); + return hostId != null ? getEndPointFromHostId(hostId) : select(object, encryptionRequired); } - } - } else if (action == StorageAction.MIGRATEVOLUME) { - VolumeInfo volume = (VolumeInfo)object; - if (volume.getHypervisorType() == Hypervisor.HypervisorType.Hyperv || volume.getHypervisorType() == Hypervisor.HypervisorType.VMware) { - VirtualMachine vm = volume.getAttachedVM(); - if ((vm != null) && (vm.getState() == VirtualMachine.State.Running)) { - Long hostId = vm.getHostId(); - return getEndPointFromHostId(hostId); + break; + case MIGRATEVOLUME: { + VolumeInfo volume = (VolumeInfo) object; + if (volume.getHypervisorType() == Hypervisor.HypervisorType.Hyperv || volume.getHypervisorType() == Hypervisor.HypervisorType.VMware) { + VirtualMachine vm = volume.getAttachedVM(); + if ((vm != null) && (vm.getState() == VirtualMachine.State.Running)) { + Long hostId = vm.getHostId(); + return getEndPointFromHostId(hostId); + } } + break; } - } else if (action == StorageAction.DELETEVOLUME) { - VolumeInfo volume = (VolumeInfo)object; - if (volume.getHypervisorType() == Hypervisor.HypervisorType.VMware) { - VirtualMachine vm = volume.getAttachedVM(); - if (vm != null) { - Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); - if (hostId != null) { - return getEndPointFromHostId(hostId); + case DELETEVOLUME: { + VolumeInfo volume = (VolumeInfo) object; + if (volume.getHypervisorType() == Hypervisor.HypervisorType.VMware) { + VirtualMachine vm = volume.getAttachedVM(); + if (vm != null) { + Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); + if (hostId != null) { + return getEndPointFromHostId(hostId); + } } } + break; } } return select(object, encryptionRequired); diff --git a/engine/storage/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java b/engine/storage/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java index cea6ac29f6ef..ac7240275d9c 100644 --- a/engine/storage/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java +++ b/engine/storage/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java @@ -88,7 +88,7 @@ public void isSnapshotChainingRequiredTestSnapshotIsNullReturnFalse() { snapshotDataStoreDaoImplSpy.snapshotVOSearch = searchBuilderMock; Mockito.doReturn(searchCriteriaMock).when(searchBuilderMock).create(); Mockito.doReturn(null).when(snapshotDaoMock).findOneBy(Mockito.any()); - Assert.assertFalse(snapshotDataStoreDaoImplSpy.isSnapshotChainingRequired(2)); + Assert.assertFalse(snapshotDataStoreDaoImplSpy.isSnapshotChainingRequired(2, false)); } @Test @@ -99,7 +99,7 @@ public void isSnapshotChainingRequiredTestSnapshotIsNotNullReturnAccordingHyperv for (Hypervisor.HypervisorType hypervisorType : Hypervisor.HypervisorType.values()) { Mockito.doReturn(hypervisorType).when(snapshotVoMock).getHypervisorType(); - boolean result = snapshotDataStoreDaoImplSpy.isSnapshotChainingRequired(2); + boolean result = snapshotDataStoreDaoImplSpy.isSnapshotChainingRequired(2, false); if (SnapshotDataStoreDaoImpl.HYPERVISORS_SUPPORTING_SNAPSHOTS_CHAINING.contains(hypervisorType)) { Assert.assertTrue(result); diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 24a0db0b74a7..2378fc872b8f 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -502,8 +502,10 @@ public Void deleteVolumeCallback(AsyncCallbackDispatcher snapshotDataStoreVOList = _snapshotStoreDao.findBySnapshotId(snapshotDataStoreVO.getSnapshotId()); + for (SnapshotDataStoreVO snapshotStore : snapshotDataStoreVOList) { + if (DataStoreRole.Image.equals(snapshotStore.getRole())) { + _snapshotStoreDao.remove(snapshotDataStoreVO.getId()); + return; + } + } + + snapshotApiService.deleteSnapshot(snapshotDataStoreVO.getSnapshotId(), null); + } + @Override public boolean cloneVolume(long volumeId, long baseVolId) { // TODO Auto-generated method stub diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index b5ec716e8053..ae7151436275 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -46,7 +46,18 @@ import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; @@ -69,6 +80,7 @@ import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.commons.collections.MapUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.BooleanUtils; @@ -306,6 +318,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String TUNGSTEN_PATH = "scripts/vm/network/tungsten"; + public static final String CHECKPOINT_CREATE_COMMAND = "virsh checkpoint-create --domain %s --xmlfile %s --redefine"; + + public static final String CHECKPOINT_DELETE_COMMAND = "virsh checkpoint-delete --domain %s --checkpointname %s"; + private String modifyVlanPath; private String versionStringPath; private String patchScriptPath; @@ -366,6 +382,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private final static long HYPERVISOR_LIBVIRT_VERSION_SUPPORTS_IO_URING = 6003000; private final static long HYPERVISOR_QEMU_VERSION_SUPPORTS_IO_URING = 5000000; + private static final int MINIMUM_LIBVIRT_VERSION_FOR_INCREMENTAL_SNAPSHOT = 7006000; + + private static final int MINIMUM_QEMU_VERSION_FOR_INCREMENTAL_SNAPSHOT = 6001000; + protected HypervisorType hypervisorType; protected String hypervisorURI; protected long hypervisorLibvirtVersion; @@ -491,11 +511,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String CGROUP_V2 = "cgroup2fs"; - protected long getHypervisorLibvirtVersion() { + public long getHypervisorLibvirtVersion() { return hypervisorLibvirtVersion; } - protected long getHypervisorQemuVersion() { + public long getHypervisorQemuVersion() { return hypervisorQemuVersion; } @@ -1853,6 +1873,10 @@ boolean isDirectAttachedNetwork(final String type) { } public String startVM(final Connect conn, final String vmName, final String domainXML) throws LibvirtException, InternalErrorException { + return startVM(conn, vmName, domainXML, 0); + } + + public String startVM(final Connect conn, final String vmName, final String domainXML, int flags) throws LibvirtException, InternalErrorException { try { /* We create a transient domain here. When this method gets @@ -1880,7 +1904,7 @@ public String startVM(final Connect conn, final String vmName, final String doma } } - conn.domainCreateXML(domainXML, 0); + conn.domainCreateXML(domainXML, flags); } catch (final LibvirtException e) { throw e; } @@ -4544,6 +4568,102 @@ protected long getMemoryFreeInKBs(Domain dm) throws LibvirtException { return freeMemory; } + public void removeCheckpointsOnVm(String vmName, String volumeUuid, List checkpointPaths) { + logger.debug("Removing checkpoints with paths [{}] of volume [{}] on VM [{}].", checkpointPaths, volumeUuid, vmName); + String checkpointName; + for (String checkpointPath : checkpointPaths) { + checkpointName = checkpointPath.substring(checkpointPath.lastIndexOf("/") + 1); + Script.runSimpleBashScript(String.format(CHECKPOINT_DELETE_COMMAND, vmName, checkpointName)); + } + logger.debug("Removed all checkpoints of volume [{}] on VM [{}].", volumeUuid, vmName); + } + + public boolean recreateCheckpointsOnVm(List volumes, String vmName, Connect conn) { + logger.debug("Trying to recreate checkpoints on VM [{}] with volumes [{}].", vmName, volumes); + try { + validateLibvirtAndQemuVersionForIncrementalSnapshots(); + } catch (CloudRuntimeException e) { + logger.warn("Will not recreate the checkpoints on VM as {}", e.getMessage(), e); + return false; + } + List diskDefs = getDisks(conn, vmName); + Map mapDiskToDiskDef = mapVolumeToDiskDef(volumes, diskDefs); + + for (VolumeObjectTO volume : volumes) { + if (CollectionUtils.isEmpty(volume.getCheckpointPaths())) { + continue; + } + volume.getCheckpointImageStoreUrls().forEach(uri -> getStoragePoolMgr().getStoragePoolByURI(uri)); + recreateCheckpointsOfDisk(vmName, volume, mapDiskToDiskDef); + } + logger.debug("Successfully recreated all checkpoints on VM [{}].", vmName); + return true; + } + + protected void recreateCheckpointsOfDisk(String vmName, VolumeObjectTO volume, Map mapDiskToDiskDef) { + for (String path : volume.getCheckpointPaths()) { + DiskDef diskDef = mapDiskToDiskDef.get(volume); + if (diskDef != null) { + try { + updateDiskLabelOnXml(path, diskDef.getDiskLabel()); + } catch (ParserConfigurationException | IOException | SAXException | TransformerException | XPathExpressionException e) { + logger.error("Exception while parsing checkpoint XML with path [{}].", path, e); + throw new CloudRuntimeException(e); + } + } else { + logger.debug("Could not map [{}] to any disk definition. Will try to recreate snapshot without updating disk label.", volume); + } + + logger.trace("Recreating checkpoint with path [{}] on VM [{}].", path, vmName); + Script.runSimpleBashScript(String.format(CHECKPOINT_CREATE_COMMAND, vmName, path)); + } + } + + /** + * Changes the value of the disk label of the checkpoint XML found in {@code path} to {@code label}. This method assumes that the checkpoint only contains one disk. + * @param path the path to the checkpoint XML to be updated + * @param label the new label to be used for the disk + * */ + private void updateDiskLabelOnXml(String path, String label) throws ParserConfigurationException, IOException, SAXException, XPathExpressionException, TransformerException { + logger.trace("Updating checkpoint with path [{}] to use disk label [{}].", path, label); + + DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory(); + DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + Document doc = docBuilder.parse(new File(path)); + + XPath xPath = XPathFactory.newInstance().newXPath(); + Node diskNode = (Node) xPath.compile("/domaincheckpoint/disks/disk").evaluate(doc, XPathConstants.NODE); + diskNode.getAttributes().getNamedItem("name").setNodeValue(label); + + Transformer tf = TransformerFactory.newInstance().newTransformer(); + tf.setOutputProperty(OutputKeys.INDENT, "yes"); + tf.setOutputProperty(OutputKeys.METHOD, "xml"); + tf.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "4"); + + DOMSource domSource = new DOMSource(doc); + StreamResult sr = new StreamResult(new File(path)); + tf.transform(domSource, sr); + } + + protected Map mapVolumeToDiskDef(List volumeTos, List diskDefs) { + HashMap diskToDiskDefHashMap = new HashMap<>(); + for (VolumeObjectTO volumeTo : volumeTos) { + for (DiskDef diskDef : diskDefs) { + if (StringUtils.contains(diskDef.getDiskPath(), volumeTo.getPath())) { + diskToDiskDefHashMap.put(volumeTo, diskDef); + } + } + } + return diskToDiskDefHashMap; + } + + public void validateLibvirtAndQemuVersionForIncrementalSnapshots() { + if (getHypervisorLibvirtVersion() < MINIMUM_LIBVIRT_VERSION_FOR_INCREMENTAL_SNAPSHOT || getHypervisorQemuVersion() < MINIMUM_QEMU_VERSION_FOR_INCREMENTAL_SNAPSHOT) { + throw new CloudRuntimeException(String.format("Hypervisor version is insufficient, should have at least libvirt [%s] and qemu [%s] but we have [%s] and [%s].", + MINIMUM_LIBVIRT_VERSION_FOR_INCREMENTAL_SNAPSHOT, MINIMUM_QEMU_VERSION_FOR_INCREMENTAL_SNAPSHOT, getHypervisorLibvirtVersion(), getHypervisorQemuVersion())); + } + } + private boolean canBridgeFirewall(final String prvNic) { final Script cmd = new Script(securityGroupPath, timeout, LOGGER); cmd.add("can_bridge_firewall"); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtXMLParser.java index f5de9b754120..5acfb60ee47e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtXMLParser.java @@ -16,16 +16,23 @@ // under the License. package com.cloud.hypervisor.kvm.resource; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.StringReader; import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.w3c.dom.Document; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import org.xml.sax.helpers.DefaultHandler; @@ -65,6 +72,20 @@ public boolean parseDomainXML(String domXML) { return false; } + public static String getXml(Document doc) throws TransformerException { + TransformerFactory transformerFactory = ParserUtils.getSaferTransformerFactory(); + Transformer transformer = transformerFactory.newTransformer(); + + DOMSource source = new DOMSource(doc); + + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + StreamResult result = new StreamResult(byteArrayOutputStream); + + transformer.transform(source, result); + + return byteArrayOutputStream.toString(); + } + @Override public void characters(char[] ch, int start, int length) throws SAXException { } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertSnapshotCommandWrapper.java new file mode 100644 index 000000000000..43aa6dd02bc6 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertSnapshotCommandWrapper.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.ConvertSnapshotAnswer; +import com.cloud.agent.api.ConvertSnapshotCommand; +import com.cloud.agent.api.to.DataStoreTO; + +import com.cloud.agent.api.to.NfsTO; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.resource.CommandWrapper; + +import com.cloud.resource.ResourceWrapper; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.libvirt.LibvirtException; + +import java.io.File; + +@ResourceWrapper(handles = ConvertSnapshotCommand.class) +public class LibvirtConvertSnapshotCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(ConvertSnapshotCommand command, LibvirtComputingResource serverResource) { + SnapshotObjectTO snapshotObjectTO = command.getSnapshotObjectTO(); + DataStoreTO imageStore = snapshotObjectTO.getDataStore(); + + logger.debug(String.format("Converting snapshot [%s] in image store [%s].", snapshotObjectTO.getId(), imageStore.getUuid())); + + if (!(imageStore instanceof NfsTO)) { + return new Answer(command, false, "Image Store must be NFS."); + } + NfsTO nfsImageStore = (NfsTO)imageStore; + + String secondaryStoragePoolUrl = nfsImageStore.getUrl(); + + try { + KVMStoragePool secondaryStorage = serverResource.getStoragePoolMgr().getStoragePoolByURI(secondaryStoragePoolUrl); + + String snapshotRelativePath = snapshotObjectTO.getPath(); + String snapshotPath = secondaryStorage.getLocalPathFor(snapshotRelativePath); + + String tempSnapshotPath = snapshotPath + ConvertSnapshotCommand.TEMP_SNAPSHOT_NAME; + + logger.debug(String.format("Converting snapshot [%s] to [%s]. The original snapshot is at [%s].", snapshotObjectTO.getId(), tempSnapshotPath, snapshotPath)); + + QemuImg qemuImg = new QemuImg(command.getWait()); + + QemuImgFile snapshot = new QemuImgFile(snapshotPath, QemuImg.PhysicalDiskFormat.QCOW2); + QemuImgFile tempSnapshot = new QemuImgFile(tempSnapshotPath, QemuImg.PhysicalDiskFormat.QCOW2); + + qemuImg.convert(snapshot, tempSnapshot); + + SnapshotObjectTO convertedSnapshot = new SnapshotObjectTO(); + convertedSnapshot.setPath(snapshotRelativePath + ConvertSnapshotCommand.TEMP_SNAPSHOT_NAME); + + final File snapFile = new File(tempSnapshotPath); + + if (!snapFile.exists()) { + return new Answer(command, false, "Failed to convert snapshot."); + } + + convertedSnapshot.setPhysicalSize(snapFile.length()); + logger.debug(String.format("Successfully converted snapshot [%s] to [%s].", snapshotObjectTO.getId(), tempSnapshotPath)); + + return new ConvertSnapshotAnswer(convertedSnapshot); + } catch (LibvirtException | QemuImgException ex) { + logger.error(String.format("Failed to convert snapshot [%s] due to %s.", snapshotObjectTO, ex.getMessage()), ex); + return new Answer(command, ex); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index e15a32876927..1487e5bc85c0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -19,7 +19,6 @@ package com.cloud.hypervisor.kvm.resource.wrapper; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URISyntaxException; @@ -39,12 +38,9 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; +import com.cloud.hypervisor.kvm.resource.LibvirtXMLParser; import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.FilenameUtils; @@ -437,7 +433,7 @@ protected String updateVmSharesIfNeeded(MigrateCommand migrateCommand, String xm logger.info(String.format("VM [%s] will have CPU shares altered from [%s] to [%s] as part of migration because the cgroups version differs between hosts.", migrateCommand.getVmName(), currentShares, newVmCpuShares)); sharesNode.setTextContent(String.valueOf(newVmCpuShares)); - return getXml(document); + return LibvirtXMLParser.getXml(document); } /** @@ -507,7 +503,7 @@ protected String replaceDpdkInterfaces(String xmlDesc, Map dpdkP } } - return getXml(doc); + return LibvirtXMLParser.getXml(doc); } /** @@ -682,7 +678,7 @@ protected String replaceStorage(String xmlDesc, Map disks, String vmName) { @@ -775,7 +771,7 @@ protected String replaceCdromIsoPath(String xmlDesc, String vmName, String oldIs newChildSourceNode.setAttribute("file", newIsoVolumePath); diskNode.appendChild(newChildSourceNode); logger.debug(String.format("Replaced ISO path [%s] with [%s] in VM [%s] XML configuration.", oldIsoVolumePath, newIsoVolumePath, vmName)); - return getXml(doc); + return LibvirtXMLParser.getXml(doc); } } } @@ -784,7 +780,7 @@ protected String replaceCdromIsoPath(String xmlDesc, String vmName, String oldIs } } - return getXml(doc); + return LibvirtXMLParser.getXml(doc); } private String getPathFromSourceText(Set paths, String sourceText) { @@ -839,20 +835,6 @@ private String getSourceText(Node diskNode) { return null; } - private String getXml(Document doc) throws TransformerException { - TransformerFactory transformerFactory = ParserUtils.getSaferTransformerFactory(); - Transformer transformer = transformerFactory.newTransformer(); - - DOMSource source = new DOMSource(doc); - - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - StreamResult result = new StreamResult(byteArrayOutputStream); - - transformer.transform(source, result); - - return byteArrayOutputStream.toString(); - } - private String replaceDiskSourceFile(String xmlDesc, String isoPath, String vmName) throws IOException, SAXException, ParserConfigurationException, TransformerException { InputStream in = IOUtils.toInputStream(xmlDesc); @@ -875,7 +857,7 @@ private String replaceDiskSourceFile(String xmlDesc, String isoPath, String vmNa } } } - return getXml(doc); + return LibvirtXMLParser.getXml(doc); } private boolean findDiskNode(Document doc, NodeList devicesChildNodes, String vmName, String isoPath) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRecreateCheckpointsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRecreateCheckpointsCommandWrapper.java new file mode 100644 index 000000000000..87b58eec21e3 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRecreateCheckpointsCommandWrapper.java @@ -0,0 +1,51 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.RecreateCheckpointsCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.log4j.Logger; +import org.libvirt.LibvirtException; + +import java.util.List; + +@ResourceWrapper(handles = RecreateCheckpointsCommand.class) +public class LibvirtRecreateCheckpointsCommandWrapper extends CommandWrapper { + + private static Logger logger = Logger.getLogger(LibvirtRecreateCheckpointsCommandWrapper.class); + @Override + public Answer execute(RecreateCheckpointsCommand command, LibvirtComputingResource serverResource) { + String vmName = command.getVmName(); + List volumes = command.getDisks(); + + boolean result; + try { + result = serverResource.recreateCheckpointsOnVm(volumes, vmName, serverResource.getLibvirtUtilitiesHelper().getConnectionByVmName(vmName)); + } catch (LibvirtException e) { + logger.error(String.format("Failed to recreate checkpoints on VM [%s] due to %s", vmName, e.getMessage()), e); + return new Answer(command, e); + } + return new Answer(command, result, null); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertSnapshotCommandWrapper.java index d2f1ef827a15..410d14053d8c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertSnapshotCommandWrapper.java @@ -20,14 +20,14 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import java.io.File; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.util.Set; import java.util.HashSet; import java.util.Arrays; +import com.cloud.agent.properties.AgentProperties; +import com.cloud.agent.properties.AgentPropertiesFileHandler; import org.apache.cloudstack.storage.command.RevertSnapshotCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.SnapshotObjectTO; @@ -53,6 +53,10 @@ import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.libvirt.LibvirtException; @ResourceWrapper(handles = RevertSnapshotCommand.class) public class LibvirtRevertSnapshotCommandWrapper extends CommandWrapper { @@ -166,7 +170,7 @@ protected void revertVolumeToSnapshot(SnapshotObjectTO snapshotOnPrimaryStorage, try { replaceVolumeWithSnapshot(volumePath, snapshotPath); logger.debug(String.format("Successfully reverted volume [%s] to snapshot [%s].", volumeObjectTo, snapshotToPrint)); - } catch (IOException ex) { + } catch (LibvirtException | QemuImgException ex) { throw new CloudRuntimeException(String.format("Unable to revert volume [%s] to snapshot [%s] due to [%s].", volumeObjectTo, snapshotToPrint, ex.getMessage()), ex); } } @@ -208,9 +212,15 @@ protected Pair getSnapshot(SnapshotObjectTO snapshotOn /** * Replaces the current volume with the snapshot. - * @throws IOException If can't replace the current volume with the snapshot. + * @throws LibvirtException If can't replace the current volume with the snapshot. + * @throws QemuImgException If can't replace the current volume with the snapshot. */ - protected void replaceVolumeWithSnapshot(String volumePath, String snapshotPath) throws IOException { - Files.copy(Paths.get(snapshotPath), Paths.get(volumePath), StandardCopyOption.REPLACE_EXISTING); + protected void replaceVolumeWithSnapshot(String volumePath, String snapshotPath) throws LibvirtException, QemuImgException { + logger.debug(String.format("Replacing volume at [%s] with snapshot that is at [%s].", volumePath, snapshotPath)); + QemuImg qemuImg = new QemuImg(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REVERT_SNAPSHOT_TIMEOUT) * 1000); + QemuImgFile volumeImg = new QemuImgFile(volumePath, QemuImg.PhysicalDiskFormat.QCOW2); + QemuImgFile snapshotImg = new QemuImgFile(snapshotPath, QemuImg.PhysicalDiskFormat.QCOW2); + + qemuImg.convert(snapshotImg, volumeImg); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java index 32d687ff98c4..a90b33496187 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java @@ -21,9 +21,13 @@ import java.io.File; import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import com.cloud.agent.resource.virtualnetwork.VRScripts; import com.cloud.utils.FileUtil; +import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.libvirt.Connect; import org.libvirt.DomainInfo.DomainState; import org.libvirt.LibvirtException; @@ -89,6 +93,12 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource libvirtComputingResource.applyDefaultNetworkRules(conn, vmSpec, false); + if (vmSpec.getType() == VirtualMachine.Type.User) { + List volumes = Arrays.stream(vmSpec.getDisks()).filter(diskTO -> diskTO.getData() instanceof VolumeObjectTO). + map(diskTO -> (VolumeObjectTO) diskTO.getData()).collect(Collectors.toList()); + libvirtComputingResource.recreateCheckpointsOnVm(volumes, vmName, conn); + } + // pass cmdline info to system vms if (vmSpec.getType() != VirtualMachine.Type.User || (vmSpec.getBootArgs() != null && vmSpec.getBootArgs().contains(UserVmManager.CKS_NODE))) { // try to patch and SSH into the systemvm for up to 5 minutes diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java index a89650f6eb65..e02437c987f4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.hypervisor.kvm.storage; +import java.io.File; import java.util.List; import java.util.Map; @@ -181,6 +182,11 @@ public Map getDetails() { return null; } + @Override + public String getLocalPathFor(String relativePath) { + return String.format("%s%s%s", getLocalPath(), File.separator, relativePath); + } + @Override public String toString() { return new ToStringBuilder(this, ToStringStyle.JSON_STYLE).append("uuid", getUuid()).append("path", getLocalPath()).toString(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java index 674799c0bbe3..46701d2eaa37 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java @@ -92,6 +92,8 @@ public default KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDis public Map getDetails(); + String getLocalPathFor(String relativePath); + public boolean isPoolSupportHA(); public String getHearthBeatPath(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index b9671c872d13..6b802c497299 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -25,8 +25,11 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermissions; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Arrays; @@ -40,7 +43,16 @@ import java.util.stream.Collectors; import javax.naming.ConfigurationException; - +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; + +import com.cloud.hypervisor.kvm.resource.LibvirtXMLParser; import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.direct.download.DirectDownloadHelper; @@ -72,9 +84,12 @@ import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuObject; +import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; @@ -134,6 +149,10 @@ import com.cloud.utils.script.Script; import com.cloud.utils.storage.S3.S3Utils; import com.cloud.vm.VmDetailConstants; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; public class KVMStorageProcessor implements StorageProcessor { protected Logger logger = LogManager.getLogger(getClass()); @@ -155,6 +174,49 @@ public class KVMStorageProcessor implements StorageProcessor { */ private long waitDelayForVirshCommands = 1000l; + private int incrementalSnapshotTimeout; + + private static final String CHECKPOINT_XML_TEMP_DIR = "/tmp/cloudstack/checkpointXMLs"; + + private static final String BACKUP_XML_TEMP_DIR = "/tmp/cloudstack/backupXMLs"; + + private static final String BACKUP_BEGIN_COMMAND = "virsh backup-begin --domain %s --backupxml %s --checkpointxml %s"; + + private static final String BACKUP_XML = ""; + + private static final String INCREMENTAL_BACKUP_XML = "%s"; + + private static final String CHECKPOINT_XML = "%s"; + + private static final String CHECKPOINT_DUMP_XML_COMMAND = "virsh checkpoint-dumpxml --domain %s --checkpointname %s --no-domain"; + + private static final String DOMJOBINFO_COMPLETED_COMMAND = "virsh domjobinfo --domain %s --completed"; + + private static final String DOMJOBABORT_COMMAND = "virsh domjobabort --domain %s"; + + private static String CHECKPOINT_DELETE_COMMAND = "virsh checkpoint-delete --domain %s --checkpointname %s"; + + private static final String DUMMY_VM_XML = "\n" + + " %s\n" + + " 256\n" + + " 256\n" + + " 1\n" + + " \n" + + " hvm\n" + + " \n" + + " \n" + + " \n" + + " /usr/bin/qemu-system-x86_64\n" + + " \n" + + " \n"+ + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + + public KVMStorageProcessor(final KVMStoragePoolManager storagePoolMgr, final LibvirtComputingResource resource) { this.storagePoolMgr = storagePoolMgr; this.resource = resource; @@ -181,6 +243,8 @@ public boolean configure(final String name, final Map params) th } _cmdsTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.CMDS_TIMEOUT) * 1000; + + incrementalSnapshotTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.INCREMENTAL_SNAPSHOT_TIMEOUT) * 1000; return true; } @@ -1524,6 +1588,8 @@ public Answer attachVolume(final AttachCommand cmd) { vol.getIopsReadRate(), vol.getIopsReadRateMax(), vol.getIopsReadRateMaxLength(), vol.getIopsWriteRate(), vol.getIopsWriteRateMax(), vol.getIopsWriteRateMaxLength(), volCacheMode, encryptDetails, disk.getDetails()); + resource.recreateCheckpointsOnVm(List.of((VolumeObjectTO) disk.getData()), vmName, conn); + return new AttachAnswer(disk); } catch (final LibvirtException e) { logger.debug("Failed to attach volume: " + vol.getPath() + ", due to ", e); @@ -1562,6 +1628,8 @@ public Answer dettachVolume(final DettachCommand cmd) { storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath()); + resource.removeCheckpointsOnVm(vmName, vol.getUuid(), vol.getCheckpointPaths()); + return new DettachAnswer(disk); } catch (final LibvirtException e) { logger.debug("Failed to detach volume: " + vol.getPath() + ", due to ", e); @@ -1713,6 +1781,7 @@ public Answer createVolume(final CreateObjectCommand cmd) { public Answer createSnapshot(final CreateObjectCommand cmd) { final SnapshotObjectTO snapshotTO = (SnapshotObjectTO)cmd.getData(); final PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)snapshotTO.getDataStore(); + DataStoreTO imageStoreTo = snapshotTO.getImageStore(); final VolumeObjectTO volume = snapshotTO.getVolume(); final String snapshotName = UUID.randomUUID().toString(); final String vmName = volume.getVmName(); @@ -1730,101 +1799,41 @@ public Answer createSnapshot(final CreateObjectCommand cmd) { } } - if (state == DomainInfo.DomainState.VIR_DOMAIN_RUNNING && volume.requiresEncryption()) { + if (DomainInfo.DomainState.VIR_DOMAIN_RUNNING.equals(state) && volume.requiresEncryption()) { throw new CloudRuntimeException("VM is running, encrypted volume snapshots aren't supported"); } - final KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); + KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); + KVMStoragePool secondaryPool = imageStoreTo != null ? storagePoolMgr.getStoragePoolByURI(imageStoreTo.getUrl()) : null; - final KVMPhysicalDisk disk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), volume.getPath()); + KVMPhysicalDisk disk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), volume.getPath()); String diskPath = disk.getPath(); String snapshotPath = diskPath + File.separator + snapshotName; - if (state == DomainInfo.DomainState.VIR_DOMAIN_RUNNING && !primaryPool.isExternalSnapshot()) { - - validateAvailableSizeOnPoolToTakeVolumeSnapshot(primaryPool, disk); - - try { - snapshotPath = getSnapshotPathInPrimaryStorage(primaryPool.getLocalPath(), snapshotName); - - String diskLabel = takeVolumeSnapshot(resource.getDisks(conn, vmName), snapshotName, diskPath, vm); - String convertResult = convertBaseFileToSnapshotFileInPrimaryStorageDir(primaryPool, diskPath, snapshotPath, volume, cmd.getWait()); - - mergeSnapshotIntoBaseFile(vm, diskLabel, diskPath, snapshotName, volume, conn); - - validateConvertResult(convertResult, snapshotPath); - } catch (LibvirtException e) { - if (!e.getMessage().contains(LIBVIRT_OPERATION_NOT_SUPPORTED_MESSAGE)) { - throw e; - } - - logger.info(String.format("It was not possible to take live disk snapshot for volume [%s], in VM [%s], due to [%s]. We will take full snapshot of the VM" - + " and extract the disk instead. Consider upgrading your QEMU binary.", volume, vmName, e.getMessage())); - - takeFullVmSnapshotForBinariesThatDoesNotSupportLiveDiskSnapshot(vm, snapshotName, vmName); - primaryPool.createFolder(TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR); - extractDiskFromFullVmSnapshot(disk, volume, snapshotPath, snapshotName, vmName, vm); - } - - /* - * libvirt on RHEL6 doesn't handle resume event emitted from - * qemu - */ - vm = resource.getDomain(conn, vmName); - state = vm.getInfo().state; - if (state == DomainInfo.DomainState.VIR_DOMAIN_PAUSED) { - vm.resume(); + SnapshotObjectTO newSnapshot = new SnapshotObjectTO(); + if (DomainInfo.DomainState.VIR_DOMAIN_RUNNING.equals(state) && !primaryPool.isExternalSnapshot()) { + if (snapshotTO.isKvmIncrementalSnapshot()) { + newSnapshot = takeIncrementalVolumeSnapshotOfRunningVm(snapshotTO, primaryPool, secondaryPool, snapshotName, volume, vm, conn, cmd.getWait()); + } else { + newSnapshot = takeFullVolumeSnapshotOfRunningVm(cmd, primaryPool, secondaryPool, disk, snapshotName, conn, vmName, diskPath, vm, volume, snapshotPath); } } else { - /** - * For RBD we can't use libvirt to do our snapshotting or any Bash scripts. - * libvirt also wants to store the memory contents of the Virtual Machine, - * but that's not possible with RBD since there is no way to store the memory - * contents in RBD. - * - * So we rely on the Java bindings for RBD to create our snapshot - * - * This snapshot might not be 100% consistent due to writes still being in the - * memory of the Virtual Machine, but if the VM runs a kernel which supports - * barriers properly (>2.6.32) this won't be any different then pulling the power - * cord out of a running machine. - */ if (primaryPool.getType() == StoragePoolType.RBD) { - try { - Rados r = radosConnect(primaryPool); - - final IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); - final Rbd rbd = new Rbd(io); - final RbdImage image = rbd.open(disk.getName()); - - logger.debug("Attempting to create RBD snapshot " + disk.getName() + "@" + snapshotName); - image.snapCreate(snapshotName); - - rbd.close(image); - r.ioCtxDestroy(io); - } catch (final Exception e) { - logger.error("A RBD snapshot operation on " + disk.getName() + " failed. The error was: " + e.getMessage()); - } + takeRbdVolumeSnapshotOfStoppedVm(primaryPool, disk, snapshotName); + newSnapshot.setPath(snapshotPath); } else if (primaryPool.getType() == StoragePoolType.CLVM) { - /* VM is not running, create a snapshot by ourself */ - final Script command = new Script(_manageSnapshotPath, _cmdsTimeout, logger); - command.add(MANAGE_SNAPSTHOT_CREATE_OPTION, disk.getPath()); - command.add(NAME_OPTION, snapshotName); - final String result = command.execute(); - if (result != null) { - logger.debug("Failed to manage snapshot: " + result); - return new CreateObjectAnswer("Failed to manage snapshot: " + result); - } + CreateObjectAnswer result = takeClvmVolumeSnapshotOfStoppedVm(disk, snapshotName); + if (result != null) return result; + newSnapshot.setPath(snapshotPath); } else { - snapshotPath = getSnapshotPathInPrimaryStorage(primaryPool.getLocalPath(), snapshotName); - String convertResult = convertBaseFileToSnapshotFileInPrimaryStorageDir(primaryPool, diskPath, snapshotPath, volume, cmd.getWait()); - validateConvertResult(convertResult, snapshotPath); + if (snapshotTO.isKvmIncrementalSnapshot()) { + newSnapshot = takeIncrementalVolumeSnapshotOfStoppedVm(snapshotTO, primaryPool, secondaryPool, snapshotName, volume, conn, cmd.getWait()); + } else { + newSnapshot = takeFullVolumeSnapshotOfStoppedVm(cmd, primaryPool, secondaryPool, snapshotName, diskPath, volume); + } } } - final SnapshotObjectTO newSnapshot = new SnapshotObjectTO(); - - newSnapshot.setPath(snapshotPath); return new CreateObjectAnswer(newSnapshot); } catch (CloudRuntimeException | LibvirtException | IOException ex) { String errorMsg = String.format("Failed take snapshot for volume [%s], in VM [%s], due to [%s].", volume, vmName, ex.getMessage()); @@ -1835,6 +1844,488 @@ public Answer createSnapshot(final CreateObjectCommand cmd) { } } + private SnapshotObjectTO createSnapshotToAndUpdatePathAndSize(String path, String fullPath) { + final File snapFile = new File(fullPath); + long size = 0; + + if (snapFile.exists()) { + size = snapFile.length(); + } + + SnapshotObjectTO snapshotObjectTo = new SnapshotObjectTO(); + + snapshotObjectTo.setPath(path); + snapshotObjectTo.setPhysicalSize(size); + + return snapshotObjectTo; + } + + private SnapshotObjectTO takeIncrementalVolumeSnapshotOfStoppedVm(SnapshotObjectTO snapshotObjectTO, KVMStoragePool primaryPool, KVMStoragePool secondaryPool, + String snapshotName, VolumeObjectTO volumeObjectTo, Connect conn, int wait) throws LibvirtException { + resource.validateLibvirtAndQemuVersionForIncrementalSnapshots(); + Domain vm = null; + logger.debug("Taking incremental volume snapshot of volume [{}]. Snapshot will be copied to [{}].", volumeObjectTo, + ObjectUtils.defaultIfNull(secondaryPool, primaryPool)); + try { + String vmName = String.format("DUMMY-VM-%s", snapshotName); + String vmXml = String.format(DUMMY_VM_XML, vmName, primaryPool.getLocalPathFor(volumeObjectTo.getPath())); + + logger.debug("Creating dummy VM with volume [{}] to take an incremental snapshot of it.", volumeObjectTo); + resource.startVM(conn, vmName, vmXml, Domain.CreateFlags.PAUSED); + + vm = resource.getDomain(conn, vmName); + + resource.recreateCheckpointsOnVm(List.of(volumeObjectTo), vmName, conn); + + return takeIncrementalVolumeSnapshotOfRunningVm(snapshotObjectTO, primaryPool, secondaryPool, snapshotName, volumeObjectTo, vm, conn, wait); + } catch (InternalErrorException | LibvirtException | CloudRuntimeException e) { + logger.error("Failed to take incremental volume snapshot of volume [{}] due to {}.", volumeObjectTo, e.getMessage(), e); + throw new CloudRuntimeException(e); + } finally { + if (vm != null) { + vm.destroy(); + } + } + } + + private SnapshotObjectTO takeIncrementalVolumeSnapshotOfRunningVm(SnapshotObjectTO snapshotObjectTO, KVMStoragePool primaryPool, KVMStoragePool secondaryPool, + String snapshotName, VolumeObjectTO volumeObjectTo, Domain vm, Connect conn, int wait) { + logger.debug("Taking incremental volume snapshot of volume [{}] attached to running VM [{}]. Snapshot will be copied to [{}].", volumeObjectTo, volumeObjectTo.getVmName(), + ObjectUtils.defaultIfNull(secondaryPool, primaryPool)); + resource.validateLibvirtAndQemuVersionForIncrementalSnapshots(); + + Pair fullSnapshotPathAndDirPath = getFullSnapshotOrCheckpointPathAndDirPathOnCorrectStorage(primaryPool, secondaryPool, snapshotName, volumeObjectTo, false); + + String diskLabel; + String vmName; + try { + List disks = resource.getDisks(conn, vm.getName()); + diskLabel = getDiskLabelToSnapshot(disks, volumeObjectTo.getPath(), vm); + vmName = vm.getName(); + } catch (LibvirtException e) { + logger.error("Failed to get VM's disks or VM name due to: [{}].", e.getMessage(), e); + throw new CloudRuntimeException(e); + } + + String[] parents = snapshotObjectTO.getParents(); + String fullSnapshotPath = fullSnapshotPathAndDirPath.first(); + + String backupXml = generateBackupXml(volumeObjectTo, parents, diskLabel, fullSnapshotPath); + String checkpointXml = String.format(CHECKPOINT_XML, snapshotName, diskLabel); + + Path backupXmlPath = createFileAndWrite(backupXml, BACKUP_XML_TEMP_DIR, snapshotName); + Path checkpointXmlPath = createFileAndWrite(checkpointXml, CHECKPOINT_XML_TEMP_DIR, snapshotName); + + String backupCommand = String.format(BACKUP_BEGIN_COMMAND, vmName, backupXmlPath.toString(), checkpointXmlPath.toString()); + + createFolderOnCorrectStorage(primaryPool, secondaryPool, fullSnapshotPathAndDirPath); + + if (Script.runSimpleBashScript(backupCommand) == null) { + throw new CloudRuntimeException(String.format("Error backing up using backupXML [%s], checkpointXML [%s] for volume [%s].", backupXml, checkpointXml, + volumeObjectTo)); + } + + try { + waitForBackup(vmName); + } catch (CloudRuntimeException ex) { + cancelBackupJob(snapshotObjectTO); + throw ex; + } + + rebaseSnapshot(secondaryPool, fullSnapshotPath, snapshotName, parents, wait); + + try { + Files.setPosixFilePermissions(Path.of(fullSnapshotPath), PosixFilePermissions.fromString("rw-r--r--")); + } catch (IOException ex) { + logger.warn("Failed to change permissions of snapshot [{}], snapshot download will not be possible.", snapshotName); + } + + String checkpointPath = dumpCheckpoint(primaryPool, secondaryPool, snapshotName, volumeObjectTo, vmName, parents); + + SnapshotObjectTO result = createSnapshotToAndUpdatePathAndSize(secondaryPool == null ? fullSnapshotPath : fullSnapshotPathAndDirPath.second() + File.separator + snapshotName, + fullSnapshotPath); + + result.setCheckpointPath(checkpointPath); + + return result; + } + + protected void createFolderOnCorrectStorage(KVMStoragePool primaryPool, KVMStoragePool secondaryPool, Pair fullSnapshotPathAndDirPath) { + if (secondaryPool == null) { + primaryPool.createFolder(fullSnapshotPathAndDirPath.second()); + } else { + secondaryPool.createFolder(fullSnapshotPathAndDirPath.second()); + } + } + + protected String generateBackupXml(VolumeObjectTO volumeObjectTo, String[] parents, String diskLabel, String fullSnapshotPath) { + if (parents == null) { + logger.debug("Snapshot of volume [{}] does not have a parent, taking a full snapshot.", volumeObjectTo); + return String.format(BACKUP_XML, diskLabel, fullSnapshotPath); + } else { + logger.debug("Snapshot of volume [{}] has parents [{}], taking an incremental snapshot.", volumeObjectTo, Arrays.toString(parents)); + String parentCheckpointName = getParentCheckpointName(parents); + return String.format(INCREMENTAL_BACKUP_XML, parentCheckpointName, diskLabel, fullSnapshotPath); + } + } + + private void waitForBackup(String vmName) throws CloudRuntimeException { + int timeout = incrementalSnapshotTimeout; + logger.debug("Waiting for backup of VM [{}] to finish, timeout is [{}].", vmName, timeout); + + String result; + + while (timeout > 0) { + result = checkBackupJob(vmName); + + if (result.contains("Completed") && result.contains("Backup")) { + return; + } + + timeout -= 10000; + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + throw new CloudRuntimeException(e); + } + } + + throw new CloudRuntimeException(String.format("Timeout while waiting for incremental snapshot for VM [%s] to finish.", vmName)); + } + + private void cancelBackupJob(SnapshotObjectTO snapshotObjectTO) { + Script.runSimpleBashScript(String.format(DOMJOBABORT_COMMAND, snapshotObjectTO.getVmName())); + + String result = checkBackupJob(snapshotObjectTO.getVmName()); + + if (result.contains("Backup") && result.contains("Cancelled")) { + logger.debug("Successfully canceled incremental snapshot job."); + } else { + logger.warn("Couldn't cancel the incremental snapshot job correctly. Job status is [{}].", result); + } + } + + private String checkBackupJob(String vmName) { + return Script.runSimpleBashScriptWithFullResult(String.format(DOMJOBINFO_COMPLETED_COMMAND, vmName), 10); + } + + protected void rebaseSnapshot(KVMStoragePool secondaryPool, String snapshotPath, String snapshotName, String[] parents, int wait) { + if (parents == null) { + logger.debug("No need to rebase snapshot [{}], this snapshot has no parents, therefore it is the first on its backing chain.", snapshotName); + return; + } + String parentSnapshotPath; + + if (secondaryPool == null) { + parentSnapshotPath = parents[parents.length - 1]; + } else { + parentSnapshotPath = secondaryPool.getLocalPath() + File.separator + parents[parents.length - 1]; + } + + QemuImgFile snapshotFile = new QemuImgFile(snapshotPath); + QemuImgFile parentSnapshotFile = new QemuImgFile(parentSnapshotPath); + + logger.debug("Rebasing snapshot [{}] with parent [{}].", snapshotName, parentSnapshotPath); + + try { + QemuImg qemuImg = new QemuImg(wait); + qemuImg.rebase(snapshotFile, parentSnapshotFile, PhysicalDiskFormat.QCOW2.toString(), false); + } catch (LibvirtException | QemuImgException e) { + logger.error("Exception while rebasing incremental snapshot [{}] due to: [{}].", snapshotName, e.getMessage(), e); + throw new CloudRuntimeException(e); + } + } + + protected String getParentCheckpointName(String[] parents) { + String immediateParentPath = parents[parents.length -1]; + return immediateParentPath.substring(immediateParentPath.lastIndexOf(File.separator) + 1); + } + + private Path createFileAndWrite(String content, String dir, String fileName) { + File dirFile = new File(dir); + if (!dirFile.exists()) { + dirFile.mkdirs(); + } + + Path filePath = Path.of(dirFile.getPath(), fileName); + try { + return Files.write(filePath, content.getBytes()); + } catch (IOException ex) { + String message = String.format("Error while writing file [%s].", filePath); + logger.error(message, ex); + throw new CloudRuntimeException(message, ex); + } + } + + private String dumpCheckpoint(KVMStoragePool primaryPool, KVMStoragePool secondaryPool, String snapshotName, VolumeObjectTO volumeObjectTo, String vmName, String[] snapshotParents) { + String result = Script.runSimpleBashScriptWithFullResult(String.format(CHECKPOINT_DUMP_XML_COMMAND, vmName, snapshotName), 10); + + String snapshotParent = snapshotParents == null ? null : snapshotParents[snapshotParents.length -1]; + + return cleanupCheckpointXmlDumpCheckpointAndRedefine(result, primaryPool, secondaryPool, snapshotName, volumeObjectTo, snapshotParent, vmName); + } + + private String cleanupCheckpointXmlDumpCheckpointAndRedefine(String checkpointXml, KVMStoragePool primaryPool, KVMStoragePool secondaryPool, String snapshotName, VolumeObjectTO volumeObjectTo, String snapshotParent, String vmName) { + String updatedCheckpointXml; + try { + updatedCheckpointXml = updateCheckpointXml(checkpointXml, snapshotParent); + } catch (TransformerException | ParserConfigurationException | IOException | SAXException | + XPathExpressionException e) { + logger.error("Exception while parsing checkpoint XML [{}].", checkpointXml, e); + throw new CloudRuntimeException(e); + } + + Pair checkpointFullPathAndDirPath = getFullSnapshotOrCheckpointPathAndDirPathOnCorrectStorage(primaryPool, secondaryPool, snapshotName, volumeObjectTo, true); + + String fullPath = checkpointFullPathAndDirPath.first(); + String dirPath = checkpointFullPathAndDirPath.second(); + + KVMStoragePool workPool = ObjectUtils.defaultIfNull(secondaryPool, primaryPool); + workPool.createFolder(dirPath); + + logger.debug("Saving checkpoint of volume [{}], attached to VM [{}], referring to snapshot [{}] to path [{}].", volumeObjectTo, vmName, snapshotName, fullPath); + createFileAndWrite(updatedCheckpointXml, workPool.getLocalPath() + File.separator + dirPath, snapshotName); + + logger.debug("Redefining checkpoint on VM [{}].", vmName); + Script.runSimpleBashScript(String.format(LibvirtComputingResource.CHECKPOINT_CREATE_COMMAND, vmName, fullPath)); + + return fullPath; + } + + /** + * Updates the checkpoint XML, setting the parent to {@code snapshotParent} and removing any disks that were not backed up. + * @param checkpointXml checkpoint XML to be parsed + * @param snapshotParent snapshot parent + * */ + private String updateCheckpointXml(String checkpointXml, String snapshotParent) throws ParserConfigurationException, XPathExpressionException, IOException, SAXException, TransformerException { + logger.debug("Parsing checkpoint XML [{}].", checkpointXml); + + InputStream in = IOUtils.toInputStream(checkpointXml); + DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory(); + DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + Document doc = docBuilder.parse(in); + XPath xPath = XPathFactory.newInstance().newXPath(); + + updateParent(snapshotParent, doc, xPath); + + removeUnnecessaryDisks(doc, xPath); + + String finalXml = LibvirtXMLParser.getXml(doc); + + logger.debug("Checkpoint XML after parsing is [{}].", finalXml); + + return finalXml; + } + + /** + * Removes all the disk definitions on the checkpoint XML from disks that were not affected. + * @param checkpointXml the checkpoint XML to be updated. + * */ + private void removeUnnecessaryDisks(Document checkpointXml, XPath xPath) throws XPathExpressionException { + Node disksNode = (Node) xPath.compile("/domaincheckpoint/disks").evaluate(checkpointXml, XPathConstants.NODE); + NodeList disksNodeChildren = disksNode.getChildNodes(); + for (int j = 0; j < disksNodeChildren.getLength(); j++) { + Node diskNode = disksNodeChildren.item(j); + if (diskNode == null) { + continue; + } + if ("disk".equals(diskNode.getNodeName()) && "no".equals(diskNode.getAttributes().getNamedItem("checkpoint").getNodeValue())) { + disksNode.removeChild(diskNode); + logger.trace("Removing node [{}].", diskNode); + } + } + } + + /** + * Updates the parent on the {@code checkpointXml} to {@code snapshotParent}. If {@code snapshotParent} is null, removes the parent. + * @param checkpointXml the checkpoint XML to be updated + * @param snapshotParent the snapshot parent. Inform null if no parent. + * */ + private void updateParent(String snapshotParent, Document checkpointXml, XPath xPath) throws XPathExpressionException { + if (snapshotParent == null) { + Object parentNodeObject = xPath.compile("/domaincheckpoint/parent").evaluate(checkpointXml, XPathConstants.NODE); + if (parentNodeObject == null) { + return; + } + Node parentNode = (Node) parentNodeObject; + parentNode.getParentNode().removeChild(parentNode); + return; + } + + Node parentNameNode = (Node) xPath.compile("/domaincheckpoint/parent/name").evaluate(checkpointXml, XPathConstants.NODE); + parentNameNode.setNodeValue(snapshotParent); + } + + /** + * If imageStore is not null, copy the snapshot directly to secondary storage, else, copy it to the primary storage. + * + * @return SnapshotObjectTO of the new snapshot. + * */ + private SnapshotObjectTO takeFullVolumeSnapshotOfRunningVm(CreateObjectCommand cmd, KVMStoragePool primaryPool, KVMStoragePool secondaryPool, KVMPhysicalDisk disk, String snapshotName, + Connect conn, String vmName, String diskPath, Domain vm, VolumeObjectTO volume, String snapshotPath) throws IOException, LibvirtException { + logger.debug("Taking full volume snapshot of volume [{}] attached to running VM [{}]. Snapshot will be copied to [{}].", volume, vmName, + ObjectUtils.defaultIfNull(secondaryPool, primaryPool)); + + validateAvailableSizeOnPoolToTakeVolumeSnapshot(primaryPool, disk); + String relativePath = null; + try { + String diskLabel = takeVolumeSnapshot(resource.getDisks(conn, vmName), snapshotName, diskPath, vm); + + Pair fullSnapPathAndDirPath = getFullSnapshotOrCheckpointPathAndDirPathOnCorrectStorage(primaryPool, secondaryPool, snapshotName, volume, false); + + snapshotPath = fullSnapPathAndDirPath.first(); + String directoryPath = fullSnapPathAndDirPath.second(); + relativePath = directoryPath + File.separator + snapshotName; + + String convertResult = convertBaseFileToSnapshotFileInStorageDir(ObjectUtils.defaultIfNull(secondaryPool, primaryPool), diskPath, snapshotPath, directoryPath, volume, cmd.getWait()); + + mergeSnapshotIntoBaseFile(vm, diskLabel, diskPath, snapshotName, volume, conn); + + validateConvertResult(convertResult, snapshotPath); + } catch (LibvirtException e) { + if (!e.getMessage().contains(LIBVIRT_OPERATION_NOT_SUPPORTED_MESSAGE)) { + throw e; + } + + logger.info("It was not possible to take live disk snapshot for volume [{}], in VM [{}], due to [{}]. We will take full snapshot of the VM" + + " and extract the disk instead. Consider upgrading your QEMU binary.", volume, vmName, e.getMessage()); + + takeFullVmSnapshotForBinariesThatDoesNotSupportLiveDiskSnapshot(vm, snapshotName, vmName); + primaryPool.createFolder(TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR); + extractDiskFromFullVmSnapshot(disk, volume, snapshotPath, snapshotName, vmName, vm); + } + + /* + * libvirt on RHEL6 doesn't handle resume event emitted from + * qemu + */ + vm = resource.getDomain(conn, vmName); + DomainInfo.DomainState state = vm.getInfo().state; + if (state == DomainInfo.DomainState.VIR_DOMAIN_PAUSED) { + vm.resume(); + } + + return createSnapshotToAndUpdatePathAndSize(secondaryPool == null ? snapshotPath : relativePath, snapshotPath); + } + + + private SnapshotObjectTO takeFullVolumeSnapshotOfStoppedVm(CreateObjectCommand cmd, KVMStoragePool primaryPool, KVMStoragePool secondaryPool, String snapshotName, String diskPath, VolumeObjectTO volume) throws IOException { + logger.debug("Taking full volume snapshot of volume [{}]. Snapshot will be copied to [{}].", volume, + ObjectUtils.defaultIfNull(secondaryPool, primaryPool)); + Pair fullSnapPathAndDirPath = getFullSnapshotOrCheckpointPathAndDirPathOnCorrectStorage(primaryPool, secondaryPool, snapshotName, volume, false); + + String snapshotPath = fullSnapPathAndDirPath.first(); + String directoryPath = fullSnapPathAndDirPath.second(); + String relativePath = directoryPath + File.separator + snapshotName; + + String convertResult = convertBaseFileToSnapshotFileInStorageDir(ObjectUtils.defaultIfNull(secondaryPool, primaryPool), diskPath, snapshotPath, directoryPath, volume, cmd.getWait()); + + validateConvertResult(convertResult, snapshotPath); + + return createSnapshotToAndUpdatePathAndSize(secondaryPool == null ? snapshotPath : relativePath, snapshotPath); + } + + private CreateObjectAnswer takeClvmVolumeSnapshotOfStoppedVm(KVMPhysicalDisk disk, String snapshotName) { + /* VM is not running, create a snapshot by ourself */ + final Script command = new Script(_manageSnapshotPath, _cmdsTimeout, logger); + command.add(MANAGE_SNAPSTHOT_CREATE_OPTION, disk.getPath()); + command.add(NAME_OPTION, snapshotName); + final String result = command.execute(); + if (result != null) { + String message = String.format("Failed to manage snapshot [%s] due to: [%s].", snapshotName, result); + logger.debug(message); + return new CreateObjectAnswer(message); + } + return null; + } + + /** + * For RBD we can't use libvirt to do our snapshotting or any Bash scripts. + * libvirt also wants to store the memory contents of the Virtual Machine, + * but that's not possible with RBD since there is no way to store the memory + * contents in RBD. + *

+ * So we rely on the Java bindings for RBD to create our snapshot + *

+ * This snapshot might not be 100% consistent due to writes still being in the + * memory of the Virtual Machine, but if the VM runs a kernel which supports + * barriers properly (>2.6.32) this won't be any different then pulling the power + * cord out of a running machine. + */ + private void takeRbdVolumeSnapshotOfStoppedVm(KVMStoragePool primaryPool, KVMPhysicalDisk disk, String snapshotName) { + try { + Rados r = radosConnect(primaryPool); + + final IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); + final Rbd rbd = new Rbd(io); + final RbdImage image = rbd.open(disk.getName()); + + logger.debug("Attempting to create RBD snapshot {}@{}", disk.getName(), snapshotName); + image.snapCreate(snapshotName); + + rbd.close(image); + r.ioCtxDestroy(io); + } catch (final Exception e) { + logger.error("A RBD snapshot operation on [{}] failed. The error was: {}", disk.getName(), e.getMessage(), e); + } + } + + /** + * Retrieves the disk label to take snapshot; + * @param disks List of VM's disks; + * @param diskPath Path of the disk to take snapshot; + * @param vm VM in which disks are; + * @return the label to take snapshot. If the disk path is not found in VM's XML, it will throw a CloudRuntimeException. + * @throws org.libvirt.LibvirtException if the disk is not found + */ + protected String getDiskLabelToSnapshot(List disks, String diskPath, Domain vm) throws LibvirtException { + logger.debug("Searching disk label of disk with path [{}] on VM [{}].", diskPath, vm.getName()); + for (DiskDef disk : disks) { + String diskDefPath = disk.getDiskPath(); + + if (StringUtils.isEmpty(diskDefPath)) { + continue; + } + + if (!diskDefPath.contains(diskPath)) { + continue; + } + logger.debug("Found disk label [{}] for volume with path [{}] on VM [{}].", disk.getDiskLabel(), diskPath, disk.getDiskLabel()); + + return disk.getDiskLabel(); + } + + throw new CloudRuntimeException(String.format("VM [%s] has no disk with path [%s]. VM's XML [%s].", vm.getName(), diskPath, vm.getXMLDesc(0))); + } + + /** + * Gets the fully qualified path of the snapshot or checkpoint and the directory path. If a secondary pool is informed, the path will be on the secondary pool, + * otherwise, the path will be on the primary pool. + * @param primaryPool Primary pool definition, the path returned will be here if no secondary pool is informed; + * @param secondaryPool Secondary pool definition. If informed, the primary pool will be ignored and the path returned will be on the secondary pool; + * @param snapshotName Name of the snapshot; + * @param volume Volume that is being snapshot; + * @param checkpoint Whether to return a path for a snapshot or a snapshot's checkpoint; + * @return Fully qualified path and the directory path of the snapshot/checkpoint. + * */ + private Pair getFullSnapshotOrCheckpointPathAndDirPathOnCorrectStorage(KVMStoragePool primaryPool, KVMStoragePool secondaryPool, String snapshotName, + VolumeObjectTO volume, boolean checkpoint) { + String fullSnapshotPath; + String dirPath; + + if (secondaryPool == null) { + fullSnapshotPath = getSnapshotOrCheckpointPathInPrimaryStorage(primaryPool.getLocalPath(), snapshotName, checkpoint); + dirPath = checkpoint ? TemplateConstants.DEFAULT_CHECKPOINT_ROOT_DIR : TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR; + } else { + Pair fullPathAndDirectoryPath = getSnapshotOrCheckpointPathAndDirectoryPathInSecondaryStorage(secondaryPool.getLocalPath(), snapshotName, + volume.getAccountId(), volume.getVolumeId(), checkpoint); + + fullSnapshotPath = fullPathAndDirectoryPath.first(); + dirPath = fullPathAndDirectoryPath.second(); + } + return new Pair<>(fullSnapshotPath, dirPath); + } + protected void deleteFullVmSnapshotAfterConvertingItToExternalDiskSnapshot(Domain vm, String snapshotName, VolumeObjectTO volume, String vmName) throws LibvirtException { logger.debug(String.format("Deleting full VM snapshot [%s] of VM [%s] as we already converted it to an external disk snapshot of the volume [%s].", snapshotName, vmName, volume)); @@ -1935,16 +2426,20 @@ protected void manuallyDeleteUnusedSnapshotFile(boolean isLibvirtSupportingFlagD /** * Creates the snapshot directory in the primary storage, if it does not exist; then, converts the base file (VM's old writing file) to the snapshot directory. - * @param primaryPool Storage to create folder, if not exists; - * @param baseFile Base file of VM, which will be converted; + * @param pool Storage to create folder, if not exists; + * @param baseFile Base file of VM, which will be converted; * @param snapshotPath Path to convert the base file; + * @param snapshotFolder Folder where the snapshot will be converted to; + * @param volume Volume being snapshot, used for logging only; + * @param wait timeout; * @return null if the conversion occurs successfully or an error message that must be handled. */ - protected String convertBaseFileToSnapshotFileInPrimaryStorageDir(KVMStoragePool primaryPool, String baseFile, String snapshotPath, VolumeObjectTO volume, int wait) { + + protected String convertBaseFileToSnapshotFileInStorageDir(KVMStoragePool pool, String baseFile, String snapshotPath, String snapshotFolder, VolumeObjectTO volume, int wait) { try { - logger.debug(String.format("Trying to convert volume [%s] (%s) to snapshot [%s].", volume, baseFile, snapshotPath)); + logger.debug("Trying to convert volume [{}] ({}) to snapshot [{}].", volume, baseFile, snapshotPath); - primaryPool.createFolder(TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR); + pool.createFolder(snapshotFolder); QemuImgFile srcFile = new QemuImgFile(baseFile); srcFile.setFormat(PhysicalDiskFormat.QCOW2); @@ -1962,14 +2457,26 @@ protected String convertBaseFileToSnapshotFileInPrimaryStorageDir(KVMStoragePool } } + protected String getSnapshotOrCheckpointPathInPrimaryStorage(String primaryStoragePath, String snapshotName, boolean checkpoint) { + String rootDir = checkpoint ? TemplateConstants.DEFAULT_CHECKPOINT_ROOT_DIR : TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR; + return String.format("%s%s%s%s%s", primaryStoragePath, File.separator, rootDir, File.separator, snapshotName); + } + /** - * Retrieves the path of the snapshot on primary storage snapshot's dir. - * @param primaryStoragePath Path of the primary storage; + * Retrieves the path of the snapshot or snapshot's checkpoint on secondary storage snapshot's dir. + * @param secondaryStoragePath Path of the secondary storage; * @param snapshotName Snapshot name; - * @return the path of the snapshot in primary storage snapshot's dir. + * @param accountId accountId; + * @param volumeId volumeId; + * @param checkpoint Whether to return a path for a snapshot or a snapshot's checkpoint; + * @return the path of the snapshot or snapshot's checkpoint in secondary storage and the snapshot's dir. */ - protected String getSnapshotPathInPrimaryStorage(String primaryStoragePath, String snapshotName) { - return String.format("%s%s%s%s%s", primaryStoragePath, File.separator, TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR, File.separator, snapshotName); + protected Pair getSnapshotOrCheckpointPathAndDirectoryPathInSecondaryStorage(String secondaryStoragePath, String snapshotName, long accountId, long volumeId, boolean checkpoint) { + String rootDir = checkpoint ? TemplateConstants.DEFAULT_CHECKPOINT_ROOT_DIR : TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR; + String snapshotParentDirectories = String.format("%s%s%s%s%s", rootDir, File.separator, accountId, File.separator, volumeId); + String fullSnapPath = String.format("%s%s%s%s%s", secondaryStoragePath, File.separator, snapshotParentDirectories, File.separator, snapshotName); + + return new Pair<>(fullSnapPath, snapshotParentDirectories); } /** @@ -2318,6 +2825,9 @@ public Answer deleteSnapshot(final DeleteCommand cmd) { logger.info(String.format("Deleting snapshot (id=%s, name=%s, path=%s, storage type=%s) on primary storage", snapshotTO.getId(), snapshotTO.getName(), snapshotTO.getPath(), primaryPool.getType())); deleteSnapshotFile(snapshotTO); + if (snapshotTO.isKvmIncrementalSnapshot()) { + deleteCheckpoint(snapshotTO); + } } else { logger.warn("Operation not implemented for storage pool type of " + primaryPool.getType().toString()); throw new InternalErrorException("Operation not implemented for storage pool type of " + primaryPool.getType().toString()); @@ -2339,6 +2849,23 @@ public Answer deleteSnapshot(final DeleteCommand cmd) { } } + /** + * Deletes the checkpoint dump if it exists. And deletes the checkpoint definition on the VM if it is running. + * */ + protected void deleteCheckpoint(SnapshotObjectTO snapshotTO) throws IOException { + if (snapshotTO.getCheckpointPath() != null) { + Files.deleteIfExists(Path.of(snapshotTO.getCheckpointPath())); + } + + String vmName = snapshotTO.getVmName(); + if (vmName == null) { + return; + } + String checkpointName = snapshotTO.getPath().substring(snapshotTO.getPath().lastIndexOf(File.separator) + 1); + + Script.runSimpleBashScript(String.format(CHECKPOINT_DELETE_COMMAND, vmName, checkpointName)); + } + /** * Deletes the snapshot's file. * @throws CloudRuntimeException If can't delete the snapshot file. diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index b80accd60181..71864bf623a9 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -26,6 +26,7 @@ import java.util.UUID; import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.qemu.QemuImageOptions; import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.cloudstack.utils.qemu.QemuImgException; @@ -1379,13 +1380,13 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt } else { destFile = new QemuImgFile(destPath, destFormat); try { - qemu.convert(srcFile, destFile); + qemu.convert(srcFile, destFile, null, null, new QemuImageOptions(srcFile.getFormat(), srcFile.getFileName(), null), null, false, true); Map destInfo = qemu.info(destFile); Long virtualSize = Long.parseLong(destInfo.get(QemuImg.VIRTUAL_SIZE)); newDisk.setVirtualSize(virtualSize); newDisk.setSize(virtualSize); - } catch (QemuImgException | LibvirtException e) { - logger.error("Failed to convert " + srcFile.getFileName() + " to " + destFile.getFileName() + " the error was: " + e.getMessage()); + } catch (QemuImgException e) { + logger.error("Failed to convert [{}] to [{}] due to: [{}].", srcFile.getFileName(), destFile.getFileName(), e.getMessage(), e); newDisk = null; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java index 52adc59cbe7b..bdc785b2eb7d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java @@ -295,6 +295,11 @@ public Map getDetails() { return null; } + @Override + public String getLocalPathFor(String relativePath) { + return String.format("%s%s%s", getLocalPath(), File.separator, relativePath); + } + @Override public boolean isPoolSupportHA() { return type == StoragePoolType.NetworkFilesystem; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java index bc2f072f7192..4a31d9a53d9b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java @@ -17,6 +17,7 @@ package com.cloud.hypervisor.kvm.storage; +import java.io.File; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -203,6 +204,11 @@ public Map getDetails() { return this.details; } + @Override + public String getLocalPathFor(String relativePath) { + return String.format("%s%s%s", getLocalPath(), File.separator, relativePath); + } + @Override public boolean isPoolSupportHA() { return false; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java index 77f21910da6c..8794c50ff5ce 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java @@ -17,6 +17,7 @@ package com.cloud.hypervisor.kvm.storage; +import java.io.File; import java.util.List; import java.util.Map; @@ -214,6 +215,11 @@ public Map getDetails() { return this.details; } + @Override + public String getLocalPathFor(String relativePath) { + return String.format("%s%s%s", getLocalPath(), File.separator, relativePath); + } + @Override public boolean isPoolSupportHA() { return false; diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java index bc123b294ce7..d6949d73f8bf 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java @@ -51,6 +51,7 @@ public class QemuImg { public static final String ENCRYPT_KEY_SECRET = "encrypt.key-secret"; public static final String TARGET_ZERO_FLAG = "--target-is-zero"; public static final long QEMU_2_10 = 2010000; + public static final long QEMU_5_10 = 5010000; /* The qemu-img binary. We expect this to be in $PATH */ public String _qemuImgPath = "qemu-img"; @@ -381,6 +382,37 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, */ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, final Map options, final List qemuObjects, final QemuImageOptions srcImageOpts, final String snapshotName, final boolean forceSourceFormat) throws QemuImgException { + convert(srcFile, destFile, options, qemuObjects, srcImageOpts, snapshotName, forceSourceFormat, false); + } + + /** + * Converts an image from source to destination. + * + * This method is a facade for 'qemu-img convert' and converts a disk image or snapshot into a disk image with the specified filename and format. + * + * @param srcFile + * The source file. + * @param destFile + * The destination file. + * @param options + * Options for the conversion. Takes a Map with key value + * pairs which are passed on to qemu-img without validation. + * @param qemuObjects + * Pass qemu Objects to create - see objects in the qemu main page. + * @param srcImageOpts + * pass qemu --image-opts to convert. + * @param snapshotName + * If it is provided, conversion uses it as parameter. + * @param forceSourceFormat + * If true, specifies the source format in the conversion command. + * @param keepBitmaps + * If true, copies the bitmaps to the destination image. + * @return void + */ + public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, + final Map options, final List qemuObjects, final QemuImageOptions srcImageOpts, final String snapshotName, final boolean forceSourceFormat, + boolean keepBitmaps) throws QemuImgException { + Script script = new Script(_qemuImgPath, timeout); if (StringUtils.isNotBlank(snapshotName)) { String qemuPath = Script.runSimpleBashScript(getQemuImgPathScript); @@ -430,6 +462,10 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, script.add(srcFile.getFileName()); } + if (this.version >= QEMU_5_10 && keepBitmaps) { + script.add("--bitmaps"); + } + script.add(destFile.getFileName()); final String result = script.execute(); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index ecb34adc6eda..edd3501e3528 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -61,6 +62,7 @@ import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.linux.CPUStat; import org.apache.cloudstack.utils.linux.MemStat; @@ -238,6 +240,13 @@ public class LibvirtComputingResourceTest { @Mock LibvirtDomainXMLParser parserMock; + @Mock + DiskTO diskToMock; + + @Mock + private VolumeObjectTO volumeObjectToMock; + + @Spy private LibvirtComputingResource libvirtComputingResourceSpy = Mockito.spy(new LibvirtComputingResource()); @@ -5316,8 +5325,8 @@ public void testStartCommand() throws Exception { try (MockedStatic sshHelperMockedStatic = Mockito.mockStatic(SshHelper.class)) { sshHelperMockedStatic.when(() -> SshHelper.scpTo( Mockito.anyString(), Mockito.anyInt(), - Mockito.anyString(), Mockito.any(File.class), nullable(String.class), Mockito.anyString(), - Mockito.any(String[].class), Mockito.anyString())).thenAnswer(invocation -> null); + Mockito.anyString(), any(File.class), nullable(String.class), Mockito.anyString(), + any(String[].class), Mockito.anyString())).thenAnswer(invocation -> null); final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance(); assertNotNull(wrapper); @@ -5396,8 +5405,8 @@ public void testStartCommandIsolationEc2() throws Exception { try (MockedStatic sshHelperMockedStatic = Mockito.mockStatic(SshHelper.class)) { sshHelperMockedStatic.when(() -> SshHelper.scpTo( Mockito.anyString(), Mockito.anyInt(), - Mockito.anyString(), Mockito.any(File.class), nullable(String.class), Mockito.anyString(), - Mockito.any(String[].class), Mockito.anyString())).thenAnswer(invocation -> null); + Mockito.anyString(), any(File.class), nullable(String.class), Mockito.anyString(), + any(String[].class), Mockito.anyString())).thenAnswer(invocation -> null); final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance(); assertNotNull(wrapper); @@ -5435,8 +5444,12 @@ public void testStartCommandHostMemory() { when(vmSpec.getNics()).thenReturn(nics); when(vmSpec.getType()).thenReturn(VirtualMachine.Type.User); when(vmSpec.getName()).thenReturn(vmName); + when(vmSpec.getDisks()).thenReturn(new DiskTO[]{diskToMock}); + when(diskToMock.getData()).thenReturn(new VolumeObjectTO()); when(libvirtComputingResourceMock.createVMFromSpec(vmSpec)).thenReturn(vmDef); + when(libvirtComputingResourceMock.recreateCheckpointsOnVm(any(), any(), any())).thenReturn(true); + when(libvirtComputingResourceMock.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); try { when(libvirtUtilitiesHelper.getConnectionByType(vmDef.getHvsType())).thenReturn(conn); @@ -5647,7 +5660,7 @@ public void testKnownCommand() { public void testAddExtraConfigComponentEmptyExtraConfig() { libvirtComputingResourceMock = new LibvirtComputingResource(); libvirtComputingResourceMock.addExtraConfigComponent(new HashMap<>(), vmDef); - Mockito.verify(vmDef, never()).addComp(Mockito.any()); + Mockito.verify(vmDef, never()).addComp(any()); } @Test @@ -5658,7 +5671,7 @@ public void testAddExtraConfigComponentNotEmptyExtraConfig() { extraConfig.put("extraconfig-2", "value2"); extraConfig.put("extraconfig-3", "value3"); libvirtComputingResourceMock.addExtraConfigComponent(extraConfig, vmDef); - Mockito.verify(vmDef, times(1)).addComp(Mockito.any()); + Mockito.verify(vmDef, times(1)).addComp(any()); } public void validateGetCurrentMemAccordingToMemBallooningWithoutMemBalooning(){ @@ -5877,7 +5890,7 @@ public void defineResourceNetworkInterfacesTestUseProperties() { try (MockedStatic ignored = Mockito.mockStatic(AgentPropertiesFileHandler.class); MockedStatic netUtilsMockedStatic = Mockito.mockStatic(NetUtils.class)) { - Mockito.when(AgentPropertiesFileHandler.getPropertyValue(Mockito.any())).thenReturn("cloudbr15", + Mockito.when(AgentPropertiesFileHandler.getPropertyValue(any())).thenReturn("cloudbr15", "cloudbr28"); Mockito.when(NetUtils.getNetworkInterface(Mockito.anyString())).thenReturn(networkInterfaceMock1, @@ -5952,7 +5965,7 @@ public void testGetHaproxyStatsMethod() throws Exception { (mock, context) -> { doNothing().when(mock).add(Mockito.anyString()); when(mock.execute()).thenReturn(null); - when(mock.execute(Mockito.any())).thenReturn(null); + when(mock.execute(any())).thenReturn(null); }); MockedConstruction ignored = Mockito.mockConstruction(OneLineParser.class, (mock, context) -> { when(mock.getLine()).thenReturn("result"); @@ -6008,7 +6021,7 @@ public void testNetworkUsageMethod1() throws Exception { (mock, context) -> { doNothing().when(mock).add(Mockito.anyString()); when(mock.execute()).thenReturn(null); - when(mock.execute(Mockito.any())).thenReturn(null); + when(mock.execute(any())).thenReturn(null); }); MockedConstruction ignored2 = Mockito.mockConstruction(OneLineParser.class, (mock, context) -> {when(mock.getLine()).thenReturn("result");})) { @@ -6030,7 +6043,7 @@ public void testNetworkUsageMethod2() throws Exception { (mock, context) -> { doNothing().when(mock).add(Mockito.anyString()); when(mock.execute()).thenReturn(null); - when(mock.execute(Mockito.any())).thenReturn(null); + when(mock.execute(any())).thenReturn(null); }); MockedConstruction ignored2 = Mockito.mockConstruction(OneLineParser.class, (mock, context) -> {when(mock.getLine()).thenReturn("result");})) { @@ -6050,7 +6063,7 @@ public void getVmsToSetMemoryBalloonStatsPeriodTestLibvirtError() throws Libvirt List result = libvirtComputingResourceSpy.getVmsToSetMemoryBalloonStatsPeriod(connMock); - Mockito.verify(loggerMock).error(Mockito.anyString(), (Throwable) Mockito.any()); + Mockito.verify(loggerMock).error(Mockito.anyString(), (Throwable) any()); Assert.assertTrue(result.isEmpty()); } @@ -6140,7 +6153,7 @@ public void setupMemoryBalloonStatsPeriodTestMemBalloonPropertyDisabled() throws Mockito.when(parserMock.parseDomainXML(Mockito.anyString())).thenReturn(true); Mockito.when(parserMock.getMemBalloon()).thenReturn(memBalloonDef); try (MockedStatic