diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index bff7078fd9f9..e70acee229df 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -441,3 +441,9 @@ iscsi.session.cleanup.enabled=false # Wait(in seconds) during agent reconnections. When no value is set then default value of 5s will be used #backoff.seconds= + +# Timeout (in seconds) to wait for the snapshot reversion to complete. +# revert.snapshot.timeout=10800 + +# Timeout (in seconds) to wait for the incremental snapshot to complete. +# incremental.snapshot.timeout=10800 diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index feb1845d84b2..695376216732 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -818,6 +818,16 @@ public Property getWorkers() { */ public static final Property SSL_HANDSHAKE_TIMEOUT = new Property<>("ssl.handshake.timeout", 30, Integer.class); + /** + * Timeout (in seconds) to wait for the incremental snapshot to complete. + * */ + public static final Property INCREMENTAL_SNAPSHOT_TIMEOUT = new Property<>("incremental.snapshot.timeout", 10800); + + /** + * Timeout (in seconds) to wait for the snapshot reversion to complete. + * */ + public static final Property REVERT_SNAPSHOT_TIMEOUT = new Property<>("revert.snapshot.timeout", 10800); + public static class Property { private String name; private T defaultValue; diff --git a/api/src/main/java/com/cloud/storage/Snapshot.java b/api/src/main/java/com/cloud/storage/Snapshot.java index fc919e442b2e..c0a7b812ed9e 100644 --- a/api/src/main/java/com/cloud/storage/Snapshot.java +++ b/api/src/main/java/com/cloud/storage/Snapshot.java @@ -48,7 +48,7 @@ public boolean equals(String snapshotType) { } public enum State { - Allocated, Creating, CreatedOnPrimary, BackingUp, BackedUp, Copying, Destroying, Destroyed, + Allocated, Creating, CreatedOnPrimary, BackingUp, BackedUp, Copying, Destroying, Destroyed, Hidden, //it's a state, user can't see the snapshot from ui, while the snapshot may still exist on the storage Error; diff --git a/core/src/main/java/com/cloud/agent/api/ConvertSnapshotAnswer.java b/core/src/main/java/com/cloud/agent/api/ConvertSnapshotAnswer.java new file mode 100644 index 000000000000..c19a061f736c --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ConvertSnapshotAnswer.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.cloud.agent.api; + +import org.apache.cloudstack.storage.to.SnapshotObjectTO; + +public class ConvertSnapshotAnswer extends Answer { + + private SnapshotObjectTO snapshotObjectTO; + + public ConvertSnapshotAnswer(SnapshotObjectTO snapshotObjectTO) { + super(null); + this.snapshotObjectTO = snapshotObjectTO; + } + + public SnapshotObjectTO getSnapshotObjectTO() { + return snapshotObjectTO; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/ConvertSnapshotCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertSnapshotCommand.java new file mode 100644 index 000000000000..e456d4b6b122 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ConvertSnapshotCommand.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.cloud.agent.api; + +import org.apache.cloudstack.storage.to.SnapshotObjectTO; + +public class ConvertSnapshotCommand extends Command { + + public static final String TEMP_SNAPSHOT_NAME = "_temp"; + + SnapshotObjectTO snapshotObjectTO; + + public SnapshotObjectTO getSnapshotObjectTO() { + return snapshotObjectTO; + } + + public ConvertSnapshotCommand(SnapshotObjectTO snapshotObjectTO) { + this.snapshotObjectTO = snapshotObjectTO; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/RecreateCheckpointsCommand.java b/core/src/main/java/com/cloud/agent/api/RecreateCheckpointsCommand.java new file mode 100644 index 000000000000..154b611be98f --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/RecreateCheckpointsCommand.java @@ -0,0 +1,49 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import org.apache.cloudstack.storage.to.VolumeObjectTO; + +import java.util.List; + +public class RecreateCheckpointsCommand extends Command { + + private List volumes; + + private String vmName; + + public RecreateCheckpointsCommand(List volumes, String vmName) { + this.volumes = volumes; + this.vmName = vmName; + } + + public List getDisks() { + return volumes; + } + + public String getVmName() { + return vmName; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/RemoveBitmapCommand.java b/core/src/main/java/com/cloud/agent/api/RemoveBitmapCommand.java new file mode 100644 index 000000000000..90b62e3f17b9 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/RemoveBitmapCommand.java @@ -0,0 +1,46 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api; + +import org.apache.cloudstack.storage.to.SnapshotObjectTO; + +public class RemoveBitmapCommand extends Command { + + private SnapshotObjectTO snapshotObjectTO; + + private boolean isVmRunning; + + public RemoveBitmapCommand(SnapshotObjectTO snapshotObjectTO, boolean isVmRunning) { + this.snapshotObjectTO = snapshotObjectTO; + this.isVmRunning = isVmRunning; + } + + @Override + public boolean executeInSequence() { + return true; + } + + public SnapshotObjectTO getSnapshotObjectTO() { + return snapshotObjectTO; + } + + public boolean isVmRunning() { + return isVmRunning; + } +} diff --git a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java index 7d8225462cab..588852101206 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java @@ -142,7 +142,7 @@ protected Answer execute(CreateObjectCommand cmd) { } return new CreateObjectAnswer("not supported type"); } catch (Exception e) { - logger.debug("Failed to create object: " + data.getObjectType() + ": " + e.toString()); + logger.error("Failed to create object [{}] due to [{}].", data.getObjectType(), e.getMessage(), e); return new CreateObjectAnswer(e.toString()); } } diff --git a/core/src/main/java/com/cloud/storage/template/TemplateConstants.java b/core/src/main/java/com/cloud/storage/template/TemplateConstants.java index d6622bed73ef..349997da1b7a 100644 --- a/core/src/main/java/com/cloud/storage/template/TemplateConstants.java +++ b/core/src/main/java/com/cloud/storage/template/TemplateConstants.java @@ -24,7 +24,7 @@ public final class TemplateConstants { public static final String DEFAULT_SNAPSHOT_ROOT_DIR = "snapshots"; public static final String DEFAULT_VOLUME_ROOT_DIR = "volumes"; public static final String DEFAULT_TMPLT_FIRST_LEVEL_DIR = "tmpl/"; - + public static final String DEFAULT_CHECKPOINT_ROOT_DIR = "checkpoints"; public static final String DEFAULT_SYSTEM_VM_TEMPLATE_PATH = "template/tmpl/1/"; public static final int DEFAULT_TMPLT_COPY_PORT = 80; diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java index 76b93909b8c1..0c3bb99e75ca 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java @@ -35,12 +35,16 @@ public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO { private VolumeObjectTO volume; private String parentSnapshotPath; private DataStoreTO dataStore; + private DataStoreTO imageStore; + private boolean kvmIncrementalSnapshot = false; private String vmName; private String name; private HypervisorType hypervisorType; private long id; private boolean quiescevm; private String[] parents; + private DataStoreTO parentStore; + private String checkpointPath; private Long physicalSize = (long) 0; private long accountId; @@ -49,6 +53,11 @@ public SnapshotObjectTO() { } + @Override + public DataObjectType getObjectType() { + return DataObjectType.SNAPSHOT; + } + public SnapshotObjectTO(SnapshotInfo snapshot) { this.path = snapshot.getPath(); this.setId(snapshot.getId()); @@ -59,27 +68,28 @@ public SnapshotObjectTO(SnapshotInfo snapshot) { this.setVmName(vol.getAttachedVmName()); } - SnapshotInfo parentSnapshot = snapshot.getParent(); - ArrayList parentsArry = new ArrayList(); - if (parentSnapshot != null) { - this.parentSnapshotPath = parentSnapshot.getPath(); - while(parentSnapshot != null) { - parentsArry.add(parentSnapshot.getPath()); - parentSnapshot = parentSnapshot.getParent(); - } - parents = parentsArry.toArray(new String[parentsArry.size()]); - ArrayUtils.reverse(parents); - } - this.dataStore = snapshot.getDataStore().getTO(); this.setName(snapshot.getName()); this.hypervisorType = snapshot.getHypervisorType(); this.quiescevm = false; - } - @Override - public DataObjectType getObjectType() { - return DataObjectType.SNAPSHOT; + this.checkpointPath = snapshot.getCheckpointPath(); + this.kvmIncrementalSnapshot = snapshot.isKvmIncrementalSnapshot(); + + SnapshotInfo parentSnapshot = snapshot.getParent(); + + if (parentSnapshot == null || (HypervisorType.KVM.equals(snapshot.getHypervisorType()) && !parentSnapshot.isKvmIncrementalSnapshot())) { + return; + } + + ArrayList parentsArray = new ArrayList<>(); + this.parentSnapshotPath = parentSnapshot.getPath(); + while (parentSnapshot != null) { + parentsArray.add(parentSnapshot.getPath()); + parentSnapshot = parentSnapshot.getParent(); + } + parents = parentsArray.toArray(new String[parentsArray.size()]); + ArrayUtils.reverse(parents); } @Override @@ -91,6 +101,30 @@ public void setDataStore(DataStoreTO store) { this.dataStore = store; } + public DataStoreTO getImageStore() { + return imageStore; + } + + public void setImageStore(DataStoreTO imageStore) { + this.imageStore = imageStore; + } + + public boolean isKvmIncrementalSnapshot() { + return kvmIncrementalSnapshot; + } + + public void setKvmIncrementalSnapshot(boolean kvmIncrementalSnapshot) { + this.kvmIncrementalSnapshot = kvmIncrementalSnapshot; + } + + public String getCheckpointPath() { + return checkpointPath; + } + + public void setCheckpointPath(String checkpointPath) { + this.checkpointPath = checkpointPath; + } + @Override public String getPath() { return this.path; @@ -178,6 +212,14 @@ public void setAccountId(long accountId) { this.accountId = accountId; } + public DataStoreTO getParentStore() { + return parentStore; + } + + public void setParentStore(DataStoreTO parentStore) { + this.parentStore = parentStore; + } + @Override public String toString() { return new StringBuilder("SnapshotTO[datastore=").append(dataStore).append("|volume=").append(volume).append("|path").append(path).append("]").toString(); diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java index 4d1d0bf90971..92f9ee497a44 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java @@ -33,6 +33,8 @@ import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import java.util.Arrays; +import java.util.List; +import java.util.Set; public class VolumeObjectTO extends DownloadableObjectTO implements DataTO { private String uuid; @@ -76,6 +78,8 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO { @LogLevel(LogLevel.Log4jLevel.Off) private byte[] passphrase; private String encryptFormat; + private List checkpointPaths; + private Set checkpointImageStoreUrls; public VolumeObjectTO() { @@ -122,6 +126,8 @@ public VolumeObjectTO(VolumeInfo volume) { this.passphrase = volume.getPassphrase(); this.encryptFormat = volume.getEncryptFormat(); this.followRedirects = volume.isFollowRedirects(); + this.checkpointPaths = volume.getCheckpointPaths(); + this.checkpointImageStoreUrls = volume.getCheckpointImageStoreUrls(); } public String getUuid() { @@ -397,4 +403,21 @@ public void clearPassphrase() { public boolean requiresEncryption() { return passphrase != null && passphrase.length > 0; } + + + public List getCheckpointPaths() { + return checkpointPaths; + } + + public void setCheckpointPaths(List checkpointPaths) { + this.checkpointPaths = checkpointPaths; + } + + public Set getCheckpointImageStoreUrls() { + return checkpointImageStoreUrls; + } + + public void setCheckpointImageStoreUrls(Set checkpointImageStoreUrls) { + this.checkpointImageStoreUrls = checkpointImageStoreUrls; + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index 7950dda4d68e..8f0e16b3e7c2 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -23,6 +23,7 @@ import java.util.Set; import com.cloud.exception.ResourceAllocationException; +import com.cloud.utils.Pair; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; @@ -179,4 +180,9 @@ DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachin * Unmanage VM volumes */ void unmanageVolumes(long vmId); + + /** + * Retrieves the volume's checkpoints paths to be used in the KVM processor. If there are no checkpoints, it will return an empty list. + */ + Pair, Set> getVolumeCheckpointPathsAndImageStoreUrls(long volumeId, HypervisorType hypervisorType); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java index 4f2a69bc7719..7cf2a593db4a 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java @@ -18,6 +18,8 @@ */ package org.apache.cloudstack.engine.subsystem.api.storage; +import com.cloud.hypervisor.Hypervisor; + import java.util.List; public interface EndPointSelector { @@ -39,6 +41,8 @@ public interface EndPointSelector { EndPoint select(DataObject object, StorageAction action, boolean encryptionSupportRequired); + EndPoint selectRandom(long zoneId, Hypervisor.HypervisorType hypervisorType); + List selectAll(DataStore store); List findAllEndpointsForScope(DataStore store); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java index 611d1247c49d..ce9f4a67a732 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java @@ -32,7 +32,8 @@ enum State { Migrated("The object has been migrated"), Destroying("Template is destroying"), Destroyed("Template is destroyed"), - Failed("Failed to download template"); + Failed("Failed to download template"), + Hidden("The object is hidden from the user"); String _description; private State(String description) { diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java index 3213484694eb..d47aa5865a0f 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java @@ -32,6 +32,10 @@ public interface SnapshotInfo extends DataObject, Snapshot { String getPath(); + DataStore getImageStore(); + + void setImageStore(DataStore imageStore); + SnapshotInfo getChild(); List getChildren(); @@ -56,6 +60,14 @@ public interface SnapshotInfo extends DataObject, Snapshot { void markBackedUp() throws CloudRuntimeException; + String getCheckpointPath(); + + void setCheckpointPath(String checkpointPath); + + void setKvmIncrementalSnapshot(boolean isKvmIncrementalSnapshot); + + boolean isKvmIncrementalSnapshot(); + Snapshot getSnapshotVO(); long getAccountId(); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java index d2e085fe90cf..d7b6b2ec75b7 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java @@ -25,8 +25,12 @@ public interface SnapshotService { SnapshotResult takeSnapshot(SnapshotInfo snapshot); + DataStore findSnapshotImageStore(SnapshotInfo snapshot); + SnapshotInfo backupSnapshot(SnapshotInfo snapshot); + SnapshotInfo convertSnapshot(SnapshotInfo snapshotInfo); + boolean deleteSnapshot(SnapshotInfo snapshot); boolean revertSnapshot(SnapshotInfo snapshot); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StorageAction.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StorageAction.java index dc93f4aa41e1..037676f81899 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StorageAction.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StorageAction.java @@ -22,6 +22,8 @@ public enum StorageAction { TAKESNAPSHOT, BACKUPSNAPSHOT, DELETESNAPSHOT, + CONVERTSNAPSHOT, + REMOVEBITMAP, MIGRATEVOLUME, DELETEVOLUME } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java index 19b852bd9126..8b0171870765 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java @@ -26,6 +26,9 @@ import com.cloud.storage.Volume; import com.cloud.vm.VirtualMachine; +import java.util.List; +import java.util.Set; + public interface VolumeInfo extends DownloadableDataInfo, Volume { boolean isAttachedVM(); @@ -96,4 +99,8 @@ public interface VolumeInfo extends DownloadableDataInfo, Volume { public byte[] getPassphrase(); Volume getVolume(); + + List getCheckpointPaths(); + + Set getCheckpointImageStoreUrls(); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index e3307caa54b6..e2e6f3c6f9d2 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -63,6 +63,7 @@ import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; @@ -83,6 +84,7 @@ import org.apache.cloudstack.reservation.dao.ReservationDao; import org.apache.cloudstack.resource.ResourceCleanupService; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.utils.cache.SingleCache; @@ -120,6 +122,7 @@ import com.cloud.agent.api.PrepareForMigrationCommand; import com.cloud.agent.api.RebootAnswer; import com.cloud.agent.api.RebootCommand; +import com.cloud.agent.api.RecreateCheckpointsCommand; import com.cloud.agent.api.ReplugNicAnswer; import com.cloud.agent.api.ReplugNicCommand; import com.cloud.agent.api.RestoreVMSnapshotAnswer; @@ -241,6 +244,7 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.ResourceLimitService; @@ -412,6 +416,16 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private SingleCache> vmIdsInProgressCache; + @Inject + private SnapshotDataStoreDao snapshotDataStoreDao; + + @Inject + private SnapshotManager snapshotManager; + + @Inject + private VolumeDataFactory volumeDataFactory; + + VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this); Map _vmGurus = new HashMap<>(); @@ -2915,6 +2929,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy _networkMgr.commitNicForMigration(vmSrc, profile); volumeMgr.release(vm.getId(), srcHostId); _networkMgr.setHypervisorHostname(profile, dest, true); + recreateCheckpointsKvmOnVmAfterMigration(vm, dstHostId); updateVmPod(vm, dstHostId); } @@ -3364,6 +3379,7 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo _networkMgr.commitNicForMigration(vmSrc, profile); volumeMgr.release(vm.getId(), srcHostId); _networkMgr.setHypervisorHostname(profile, destination, true); + endSnapshotChainForVolumes(volumeToPoolMap, vm.getHypervisorType()); } work.setStep(Step.Done); @@ -3371,6 +3387,68 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo } } + protected void endSnapshotChainForVolumes(Map volumeToPoolMap, HypervisorType hypervisorType) { + Set volumes = volumeToPoolMap.keySet(); + volumes.forEach(volume -> { + Volume volumeOnDestination = _volsDao.findByPoolIdName(volumeToPoolMap.get(volume).getId(), volume.getName()); + snapshotManager.endSnapshotChainForVolume(volumeOnDestination.getId(), hypervisorType); + }); + } + + protected void recreateCheckpointsKvmOnVmAfterMigration(VMInstanceVO vm, long hostId) { + if (!HypervisorType.KVM.equals(vm.getHypervisorType())) { + logger.debug("Will not recreate checkpoint on VM as it is not running on KVM, thus it is not needed."); + return; + } + + List volumes = getVmVolumesWithCheckpointsToRecreate(vm); + + if (volumes.isEmpty()) { + logger.debug("Will not recreate checkpoints on VM as its volumes do not have any checkpoints associated with them."); + return; + } + + RecreateCheckpointsCommand recreateCheckpointsCommand = new RecreateCheckpointsCommand(volumes, vm.getInstanceName()); + Answer answer = null; + try { + logger.debug(String.format("Recreating the volume checkpoints with URLs [%s] of volumes [%s] on %s as part of the migration process.", volumes.stream().map(VolumeObjectTO::getCheckpointPaths).collect(Collectors.toList()), volumes, vm)); + answer = _agentMgr.send(hostId, recreateCheckpointsCommand); + } catch (AgentUnavailableException | OperationTimedoutException e) { + logger.error(String.format("Exception while sending command to host [%s] to recreate checkpoints with URLs [%s] of volumes [%s] on %s due to: [%s].", hostId, volumes.stream().map(VolumeObjectTO::getCheckpointPaths).collect(Collectors.toList()), volumes, vm, e.getMessage()), e); + throw new CloudRuntimeException(e); + } finally { + if (answer != null && answer.getResult()) { + logger.debug(String.format("Successfully recreated checkpoints on VM [%s].", vm)); + return; + } + + logger.debug(String.format("Migration on VM [%s] was successful; however, we weren't able to recreate the checkpoints on it. Marking the snapshot chain as ended." + + " Next snapshot will create a new snapshot chain.", vm)); + + volumes.forEach(volumeObjectTO -> snapshotManager.endSnapshotChainForVolume(volumeObjectTO.getId(), HypervisorType.KVM)); + } + } + + + protected List getVmVolumesWithCheckpointsToRecreate(VMInstanceVO vm) { + List vmVolumes = _volsDao.findByInstance(vm.getId()); + List volumes = new ArrayList<>(); + + for (VolumeVO volume : vmVolumes) { + Pair, Set> volumeCheckpointPathsAndImageStoreUrls = volumeMgr.getVolumeCheckpointPathsAndImageStoreUrls(volume.getId(), HypervisorType.KVM); + if (volumeCheckpointPathsAndImageStoreUrls.first().isEmpty()) { + continue; + } + VolumeObjectTO volumeTo = new VolumeObjectTO(); + volumeTo.setCheckpointPaths(volumeCheckpointPathsAndImageStoreUrls.first()); + volumeTo.setCheckpointImageStoreUrls(volumeCheckpointPathsAndImageStoreUrls.second()); + volumeTo.setPath(volume.getPath()); + volumes.add(volumeTo); + } + return volumes; + } + + @Override public VirtualMachineTO toVmTO(final VirtualMachineProfile profile) { final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(profile.getVirtualMachine().getHypervisorType()); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index db0119febde7..10c9f2fa59cd 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -42,6 +42,7 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.user.AccountManager; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; @@ -81,8 +82,10 @@ import org.apache.cloudstack.secret.dao.PassphraseDao; import org.apache.cloudstack.snapshot.SnapshotHelper; import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; @@ -198,6 +201,8 @@ public enum UserVmCloneType { @Inject protected PrimaryDataStoreDao _storagePoolDao = null; @Inject + protected ImageStoreDao imageStoreDao; + @Inject protected TemplateDataStoreDao _vmTemplateStoreDao = null; @Inject protected VolumeDao _volumeDao; @@ -574,6 +579,11 @@ public VolumeInfo createVolumeFromSnapshot(Volume volume, Snapshot snapshot, Use throw e; } + boolean kvmIncrementalSnapshot = SnapshotManager.kvmIncrementalSnapshot.valueIn(_hostDao.findClusterIdByVolumeInfo(snapInfo.getBaseVolume())); + if (kvmIncrementalSnapshot && DataStoreRole.Image.equals(dataStoreRole)) { + snapInfo = snapshotHelper.convertSnapshotIfNeeded(snapInfo); + } + // don't try to perform a sync if the DataStoreRole of the snapshot is equal to DataStoreRole.Primary if (!DataStoreRole.Primary.equals(dataStoreRole) || kvmSnapshotOnlyInPrimaryStorage) { try { @@ -1974,8 +1984,27 @@ public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws Sto _vmCloneSettingDao.persist(vmCloneSettingVO); } } + } + } + @Override + public Pair, Set> getVolumeCheckpointPathsAndImageStoreUrls(long volumeId, HypervisorType hypervisorType) { + List checkpointPaths = new ArrayList<>(); + Set imageStoreIds = new HashSet<>(); + Set imageStoreUrls = new HashSet<>(); + if (HypervisorType.KVM.equals(hypervisorType)) { + List snapshotDataStoreVos = _snapshotDataStoreDao.listReadyByVolumeIdAndCheckpointPathNotNull(volumeId); + snapshotDataStoreVos.forEach(snapshotDataStoreVO -> { + checkpointPaths.add(snapshotDataStoreVO.getKvmCheckpointPath()); + if (DataStoreRole.Image.equals(snapshotDataStoreVO.getRole())) { + imageStoreIds.add(snapshotDataStoreVO.getDataStoreId()); + } + }); + imageStoreUrls = imageStoreIds.stream().map(id -> imageStoreDao.findById(id).getUrl()).collect(Collectors.toSet()); + logger.debug(String.format("Found [%s] snapshots [%s] that have checkpoints for volume with id [%s].", snapshotDataStoreVos.size(), snapshotDataStoreVos, volumeId)); } + + return new Pair<>(checkpointPaths, imageStoreUrls); } private void handleCheckAndRepairVolume(Volume vol, Long hostId) { diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index 47f9b9f33e27..4a451e4e2d4c 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -46,6 +46,7 @@ import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.commons.collections.MapUtils; import org.junit.After; import org.junit.Assert; @@ -85,7 +86,9 @@ import com.cloud.deploy.DeploymentPlanningManager; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; +import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; @@ -114,6 +117,7 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.AccountVO; @@ -153,6 +157,9 @@ public class VirtualMachineManagerImplTest { @Mock private ServiceOfferingVO serviceOfferingMock; + @Mock + private SnapshotManager snapshotManagerMock; + @Mock private DiskOfferingVO diskOfferingMock; @@ -1304,4 +1311,74 @@ public void testUpdateVmMetadataManufacturerAndProductCustomManufacturer() { Assert.assertEquals(manufacturer, to.getMetadataManufacturer()); Assert.assertEquals(product, to.getMetadataProductName()); } + + @Test + public void recreateCheckpointsKvmOnVmAfterMigrationTestReturnIfNotKvm() { + Mockito.doReturn(HypervisorType.VMware).when(vmInstanceMock).getHypervisorType(); + + virtualMachineManagerImpl.recreateCheckpointsKvmOnVmAfterMigration(vmInstanceMock, 0); + + Mockito.verify(volumeDaoMock, Mockito.never()).findByInstance(Mockito.anyLong()); + } + + @Test + public void recreateCheckpointsKvmOnVmAfterMigrationTestReturnIfVolumesDoNotHaveCheckpoints() throws OperationTimedoutException, AgentUnavailableException { + Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); + Mockito.doReturn(new ArrayList()).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); + + virtualMachineManagerImpl.recreateCheckpointsKvmOnVmAfterMigration(vmInstanceMock, 0); + + Mockito.verify(agentManagerMock, Mockito.never()).send(Mockito.anyLong(), (Command) any()); + } + + @Test (expected = CloudRuntimeException.class) + public void recreateCheckpointsKvmOnVmAfterMigrationTestAgentUnavailableThrowsCloudRuntimeExceptionAndEndsSnapshotChains() throws OperationTimedoutException, AgentUnavailableException { + Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); + Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); + + Mockito.doThrow(new AgentUnavailableException(0)).when(agentManagerMock).send(Mockito.anyLong(), (Command) any()); + Mockito.doNothing().when(snapshotManagerMock).endSnapshotChainForVolume(Mockito.anyLong(), Mockito.any()); + + virtualMachineManagerImpl.recreateCheckpointsKvmOnVmAfterMigration(vmInstanceMock, 0); + + Mockito.verify(snapshotManagerMock, Mockito.times(1)).endSnapshotChainForVolume(Mockito.anyLong(),any()); + } + + @Test (expected = CloudRuntimeException.class) + public void recreateCheckpointsKvmOnVmAfterMigrationTestOperationTimedoutExceptionThrowsCloudRuntimeExceptionAndEndsSnapshotChains() throws OperationTimedoutException, AgentUnavailableException { + Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); + Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); + + Mockito.doThrow(new OperationTimedoutException(null, 0, 0, 0, false)).when(agentManagerMock).send(Mockito.anyLong(), (Command) any()); + Mockito.doNothing().when(snapshotManagerMock).endSnapshotChainForVolume(Mockito.anyLong(), Mockito.any()); + + virtualMachineManagerImpl.recreateCheckpointsKvmOnVmAfterMigration(vmInstanceMock, 0); + + Mockito.verify(snapshotManagerMock, Mockito.times(1)).endSnapshotChainForVolume(Mockito.anyLong(),any()); + } + + @Test + public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationFails() throws OperationTimedoutException, AgentUnavailableException { + Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); + Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); + + Mockito.doReturn(new com.cloud.agent.api.Answer(null, false, null)).when(agentManagerMock).send(Mockito.anyLong(), (Command) any()); + Mockito.doNothing().when(snapshotManagerMock).endSnapshotChainForVolume(Mockito.anyLong(), Mockito.any()); + + virtualMachineManagerImpl.recreateCheckpointsKvmOnVmAfterMigration(vmInstanceMock, 0); + + Mockito.verify(snapshotManagerMock, Mockito.times(1)).endSnapshotChainForVolume(Mockito.anyLong(),any()); + } + + @Test + public void recreateCheckpointsKvmOnVmAfterMigrationTestRecreationSucceeds() throws OperationTimedoutException, AgentUnavailableException { + Mockito.doReturn(HypervisorType.KVM).when(vmInstanceMock).getHypervisorType(); + Mockito.doReturn(List.of(new VolumeObjectTO())).when(virtualMachineManagerImpl).getVmVolumesWithCheckpointsToRecreate(Mockito.any()); + + Mockito.doReturn(new com.cloud.agent.api.Answer(null, true, null)).when(agentManagerMock).send(Mockito.anyLong(), (Command) any()); + + virtualMachineManagerImpl.recreateCheckpointsKvmOnVmAfterMigration(vmInstanceMock, 0); + + Mockito.verify(snapshotManagerMock, Mockito.never()).endSnapshotChainForVolume(Mockito.anyLong(),any()); + } } diff --git a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java index a37845c86c2f..8cfc2ff200ab 100644 --- a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java +++ b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java @@ -18,13 +18,17 @@ import java.util.ArrayList; import java.util.Date; +import java.util.List; +import java.util.Set; import com.cloud.hypervisor.Hypervisor; import com.cloud.offering.DiskOffering; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage; import com.cloud.storage.Volume; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; +import com.cloud.utils.Pair; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; @@ -32,6 +36,10 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.commons.lang3.ObjectUtils; import org.junit.Assert; import org.junit.Before; @@ -66,6 +74,13 @@ public class VolumeOrchestratorTest { @Mock protected VolumeDao volumeDao; + @Mock + private SnapshotDataStoreDao snapshotDataStoreDaoMock; + + @Mock + private ImageStoreDao imageStoreDaoMock; + + @Spy @InjectMocks private VolumeOrchestrator volumeOrchestrator = new VolumeOrchestrator(); @@ -208,4 +223,47 @@ public void testImportVolume() { Mockito.verify(volume, Mockito.times(1)).setChainInfo(chainInfo); Mockito.verify(volume, Mockito.times(1)).setState(Volume.State.Ready); } + + @Test + public void getVolumeCheckpointPathsAndImageStoreUrlsTestReturnEmptyListsIfNotKVM() { + Pair, Set> result = volumeOrchestrator.getVolumeCheckpointPathsAndImageStoreUrls(0, Hypervisor.HypervisorType.VMware); + + Assert.assertTrue(result.first().isEmpty()); + Assert.assertTrue(result.second().isEmpty()); + } + + @Test + public void getVolumeCheckpointPathsAndImageStoreUrlsTestReturnCheckpointIfKVM() { + SnapshotDataStoreVO snapshotDataStoreVO = new SnapshotDataStoreVO(); + snapshotDataStoreVO.setKvmCheckpointPath("Test"); + snapshotDataStoreVO.setRole(DataStoreRole.Primary); + + Mockito.doReturn(List.of(snapshotDataStoreVO)).when(snapshotDataStoreDaoMock).listReadyByVolumeIdAndCheckpointPathNotNull(Mockito.anyLong()); + + Pair, Set> result = volumeOrchestrator.getVolumeCheckpointPathsAndImageStoreUrls(0, Hypervisor.HypervisorType.KVM); + + Assert.assertEquals("Test", result.first().get(0)); + Assert.assertTrue(result.second().isEmpty()); + } + + @Test + public void getVolumeCheckpointPathsAndImageStoreUrlsTestReturnCheckpointIfKVMAndImageStore() { + SnapshotDataStoreVO snapshotDataStoreVO = new SnapshotDataStoreVO(); + snapshotDataStoreVO.setKvmCheckpointPath("Test"); + snapshotDataStoreVO.setRole(DataStoreRole.Image); + snapshotDataStoreVO.setDataStoreId(13); + + Mockito.doReturn(List.of(snapshotDataStoreVO)).when(snapshotDataStoreDaoMock).listReadyByVolumeIdAndCheckpointPathNotNull(Mockito.anyLong()); + + ImageStoreVO imageStoreVO = new ImageStoreVO(); + imageStoreVO.setUrl("URL"); + Mockito.doReturn(imageStoreVO).when(imageStoreDaoMock).findById(Mockito.anyLong()); + + Pair, Set> result = volumeOrchestrator.getVolumeCheckpointPathsAndImageStoreUrls(0, Hypervisor.HypervisorType.KVM); + + Assert.assertEquals("Test", result.first().get(0)); + Assert.assertTrue(result.second().contains("URL")); + Assert.assertEquals(1, result.second().size()); + } + } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index 1b3a5f21ac32..4e07e6f5c370 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -31,6 +31,7 @@ import com.cloud.utils.Pair; import com.cloud.utils.db.GenericDao; import com.cloud.utils.fsm.StateDao; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; /** * Data Access Object for server @@ -218,4 +219,6 @@ List findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(final Long List listDistinctArchTypes(final Long clusterId); List listByIds(final List ids); + + Long findClusterIdByVolumeInfo(VolumeInfo volumeInfo); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 43dcd60e38b0..7cda0a367aa2 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -35,6 +35,8 @@ import javax.inject.Inject; import javax.persistence.TableGenerator; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; import org.apache.commons.collections.CollectionUtils; @@ -77,6 +79,7 @@ import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.UpdateBuilder; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.commons.lang3.ObjectUtils; @DB @TableGenerator(name = "host_req_sq", table = "op_host", pkColumnName = "id", valueColumnName = "sequence", allocationSize = 1) @@ -1853,4 +1856,24 @@ public List listByIds(List ids) { sc.setParameters("id", ids.toArray()); return search(sc, null); } + + + @Override + public Long findClusterIdByVolumeInfo(VolumeInfo volumeInfo) { + VirtualMachine virtualMachine = volumeInfo.getAttachedVM(); + if (virtualMachine == null) { + return null; + } + + Long hostId = ObjectUtils.defaultIfNull(virtualMachine.getHostId(), virtualMachine.getLastHostId()); + Host host = findById(hostId); + + if (host == null) { + logger.warn(String.format("VM [%s] has null host on DB, either this VM was never started, or there is some inconsistency on the DB.", virtualMachine.getUuid())); + return null; + } + + return host.getClusterId(); + } + } diff --git a/engine/schema/src/main/java/com/cloud/storage/SnapshotVO.java b/engine/schema/src/main/java/com/cloud/storage/SnapshotVO.java index 19c67a91e2c8..4a504333344f 100644 --- a/engine/schema/src/main/java/com/cloud/storage/SnapshotVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/SnapshotVO.java @@ -284,6 +284,6 @@ public Class getEntityType() { public String toString() { return String.format("Snapshot %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields( - this, "id", "uuid", "name", "volumeId", "version")); + this, "id", "uuid", "name", "volumeId", "version", "state")); } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index 4cd29b465eeb..f0072a414fb4 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -19,6 +19,7 @@ import java.util.Date; import java.util.List; +import com.cloud.hypervisor.Hypervisor; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; @@ -46,12 +47,26 @@ public interface SnapshotDataStoreDao extends GenericDao listBySnapshot(long snapshotId, DataStoreRole role); + SnapshotDataStoreVO findParent(DataStoreRole role, Long storeId, Long zoneId, Long volumeId, boolean kvmIncrementalSnapshot, Hypervisor.HypervisorType hypervisorType); + + SnapshotDataStoreVO findBySnapshotIdAndDataStoreRoleAndState(long snapshotId, DataStoreRole role, ObjectInDataStoreStateMachine.State state); + + List listReadyByVolumeIdAndCheckpointPathNotNull(long volumeId); + + SnapshotDataStoreVO findOneBySnapshotId(long snapshotId, long zoneId); + + List listBySnapshotId(long snapshotId); + + List listBySnapshotAndDataStoreRole(long snapshotId, DataStoreRole role); + + List listExtractedSnapshotsBeforeDate(Date beforeDate); List listReadyBySnapshot(long snapshotId, DataStoreRole role); SnapshotDataStoreVO findBySourceSnapshot(long snapshotId, DataStoreRole role); + List findBySnapshotIdAndNotInDestroyedHiddenState(long snapshotId); + List listDestroyed(long storeId); List findBySnapshotId(long snapshotId); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index 5bf67eb38819..2b064be6b60a 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -58,26 +58,42 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase searchFilteringStoreIdEqStoreRoleEqStateNeqRefCntNeq; protected SearchBuilder searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq; private SearchBuilder stateSearch; - private SearchBuilder idStateNeqSearch; + private SearchBuilder idStateNinSearch; protected SearchBuilder snapshotVOSearch; private SearchBuilder snapshotCreatedSearch; private SearchBuilder dataStoreAndInstallPathSearch; private SearchBuilder storeAndSnapshotIdsSearch; private SearchBuilder storeSnapshotDownloadStatusSearch; + private SearchBuilder searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull; + private SearchBuilder searchFilterStateAndDownloadUrlNotNullAndDownloadUrlCreatedBefore; + private SearchBuilder searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq; + protected static final List HYPERVISORS_SUPPORTING_SNAPSHOTS_CHAINING = List.of(Hypervisor.HypervisorType.XenServer); @Inject protected SnapshotDao snapshotDao; + @Inject + protected ImageStoreDao imageStoreDao; + private static final String FIND_OLDEST_OR_LATEST_SNAPSHOT = "select store_id, store_role, snapshot_id from cloud.snapshot_store_ref where " + " store_role = ? and volume_id = ? and state = 'Ready'" + " order by created %s " + " limit 1"; + private static final String FIND_SNAPSHOT_IN_ZONE = "SELECT ssr.* FROM " + + "snapshot_store_ref ssr, snapshots s " + + "WHERE ssr.snapshot_id=? AND ssr.snapshot_id = s.id AND s.data_center_id=?;"; + @Override public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); @@ -119,10 +135,10 @@ public boolean configure(String name, Map params) throws Configu stateSearch.done(); - idStateNeqSearch = createSearchBuilder(); - idStateNeqSearch.and(SNAPSHOT_ID, idStateNeqSearch.entity().getSnapshotId(), SearchCriteria.Op.EQ); - idStateNeqSearch.and(STATE, idStateNeqSearch.entity().getState(), SearchCriteria.Op.NEQ); - idStateNeqSearch.done(); + idStateNinSearch = createSearchBuilder(); + idStateNinSearch.and(SNAPSHOT_ID, idStateNinSearch.entity().getSnapshotId(), SearchCriteria.Op.EQ); + idStateNinSearch.and(STATE, idStateNinSearch.entity().getState(), SearchCriteria.Op.NOTIN); + idStateNinSearch.done(); snapshotVOSearch = snapshotDao.createSearchBuilder(); snapshotVOSearch.and(VOLUME_ID, snapshotVOSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); @@ -151,6 +167,26 @@ public boolean configure(String name, Map params) throws Configu storeSnapshotDownloadStatusSearch.and("downloadState", storeSnapshotDownloadStatusSearch.entity().getDownloadState(), SearchCriteria.Op.IN); storeSnapshotDownloadStatusSearch.done(); + searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull = createSearchBuilder(); + searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull.and(VOLUME_ID, searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull.entity().getVolumeId(), SearchCriteria.Op.EQ); + searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull.and(STATE, searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull.entity().getState(), SearchCriteria.Op.EQ); + searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull.and(STORE_ROLE, searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull.entity().getRole(), SearchCriteria.Op.EQ); + searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull.and(KVM_CHECKPOINT_PATH, searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull.entity().getKvmCheckpointPath(), SearchCriteria.Op.NNULL); + searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull.and(STORE_ID, searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull.entity().getDataStoreId(), SearchCriteria.Op.IN); + searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull.done(); + + searchFilterStateAndDownloadUrlNotNullAndDownloadUrlCreatedBefore = createSearchBuilder(); + searchFilterStateAndDownloadUrlNotNullAndDownloadUrlCreatedBefore.and(STATE, searchFilterStateAndDownloadUrlNotNullAndDownloadUrlCreatedBefore.entity().getState(), SearchCriteria.Op.EQ); + searchFilterStateAndDownloadUrlNotNullAndDownloadUrlCreatedBefore.and(DOWNLOAD_URL, searchFilterStateAndDownloadUrlNotNullAndDownloadUrlCreatedBefore.entity().getExtractUrl(), SearchCriteria.Op.NNULL); + searchFilterStateAndDownloadUrlNotNullAndDownloadUrlCreatedBefore.and(URL_CREATED_BEFORE, searchFilterStateAndDownloadUrlNotNullAndDownloadUrlCreatedBefore.entity().getExtractUrlCreated(), SearchCriteria.Op.LT); + searchFilterStateAndDownloadUrlNotNullAndDownloadUrlCreatedBefore.done(); + + searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq = createSearchBuilder(); + searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq.and(STATE, searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq.entity().getState(), SearchCriteria.Op.EQ); + searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq.and(VOLUME_ID, searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq.entity().getVolumeId(), SearchCriteria.Op.EQ); + searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq.and(STORE_ROLE, searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq.entity().getRole(), SearchCriteria.Op.EQ); + searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq.and(STORE_ID, searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq.entity().getDataStoreId(), SearchCriteria.Op.IN); + return true; } @@ -283,26 +319,86 @@ protected SnapshotDataStoreVO findOldestOrLatestSnapshotForVolume(long volumeId, @Override @DB public SnapshotDataStoreVO findParent(DataStoreRole role, Long storeId, Long volumeId) { - if (!isSnapshotChainingRequired(volumeId)) { + return findParent(role, storeId, null, volumeId, false, null); + } + + @Override + @DB + public SnapshotDataStoreVO findParent(DataStoreRole role, Long storeId, Long zoneId, Long volumeId, boolean kvmIncrementalSnapshot, Hypervisor.HypervisorType hypervisorType) { + if (!isSnapshotChainingRequired(volumeId, kvmIncrementalSnapshot)) { logger.trace(String.format("Snapshot chaining is not required for snapshots of volume [%s]. Returning null as parent.", volumeId)); return null; } - SearchCriteria sc = searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq.create(); + SearchCriteria sc; + if (kvmIncrementalSnapshot && Hypervisor.HypervisorType.KVM.equals(hypervisorType)) { + sc = searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull.create(); + } else { + sc = searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEq.create(); + } + sc.setParameters(VOLUME_ID, volumeId); - sc.setParameters(STORE_ROLE, role.toString()); + if (role != null) { + sc.setParameters(STORE_ROLE, role.toString()); + } sc.setParameters(STATE, ObjectInDataStoreStateMachine.State.Ready.name()); - sc.setParameters(STORE_ID, storeId); + if (storeId != null) { + sc.setParameters(STORE_ID, new Long[]{storeId}); + } else if (zoneId != null) { + List imageStores = imageStoreDao.listStoresByZoneId(zoneId); + Object[] imageStoreIds = imageStores.stream().map(ImageStoreVO::getId).toArray(); + sc.setParameters(STORE_ID, imageStoreIds); + } List snapshotList = listBy(sc, new Filter(SnapshotDataStoreVO.class, CREATED, false, null, null)); - if (CollectionUtils.isNotEmpty(snapshotList)) { - return snapshotList.get(0); + if (CollectionUtils.isEmpty(snapshotList)) { + return null; + } + + SnapshotDataStoreVO parent = snapshotList.get(0); + + if (kvmIncrementalSnapshot && parent.getKvmCheckpointPath() == null && Hypervisor.HypervisorType.KVM.equals(hypervisorType)) { + return null; + } + + return parent; + } + + @Override + public SnapshotDataStoreVO findBySnapshotIdAndDataStoreRoleAndState(long snapshotId, DataStoreRole role, State state) { + SearchCriteria sc = createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, role); + sc.setParameters(STATE, state); + return findOneBy(sc); + } + + @Override + public SnapshotDataStoreVO findOneBySnapshotId(long snapshotId, long zoneId) { + try (TransactionLegacy transactionLegacy = TransactionLegacy.currentTxn()) { + try (PreparedStatement preparedStatement = transactionLegacy.prepareStatement(FIND_SNAPSHOT_IN_ZONE)) { + preparedStatement.setLong(1, snapshotId); + preparedStatement.setLong(2, zoneId); + + try (ResultSet resultSet = preparedStatement.executeQuery()) { + if (resultSet.next()) { + return toEntityBean(resultSet, false); + } + } + } + } catch (SQLException e) { + logger.warn(String.format("Failed to find %s snapshot in zone %s due to [%s].", snapshotId, zoneId, e.getMessage()), e); } return null; } @Override - public List listBySnapshot(long snapshotId, DataStoreRole role) { + public List listBySnapshotId(long snapshotId) { + SearchCriteria sc = searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq.create(); + sc.setParameters(SNAPSHOT_ID, snapshotId); + return listBy(sc); + } + + @Override + public List listBySnapshotAndDataStoreRole(long snapshotId, DataStoreRole role) { SearchCriteria sc = createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, role); return listBy(sc); } @@ -340,9 +436,17 @@ public List findByVolume(long snapshotId, long volumeId, Da @Override public List findBySnapshotId(long snapshotId) { - SearchCriteria sc = idStateNeqSearch.create(); + SearchCriteria sc = idStateNinSearch.create(); sc.setParameters(SNAPSHOT_ID, snapshotId); - sc.setParameters(STATE, State.Destroyed); + sc.setParameters(STATE, State.Destroyed.name()); + return listBy(sc); + } + + @Override + public List findBySnapshotIdAndNotInDestroyedHiddenState(long snapshotId) { + SearchCriteria sc = idStateNinSearch.create(); + sc.setParameters(SNAPSHOT_ID, snapshotId); + sc.setParameters(STATE, State.Destroyed.name(), State.Hidden.name()); return listBy(sc); } @@ -485,13 +589,35 @@ protected SearchCriteria createSearchCriteriaBySnapshotIdAn return sc; } - protected boolean isSnapshotChainingRequired(long volumeId) { + protected boolean isSnapshotChainingRequired(long volumeId, boolean kvmIncrementalSnapshot) { SearchCriteria sc = snapshotVOSearch.create(); sc.setParameters(VOLUME_ID, volumeId); SnapshotVO snapshot = snapshotDao.findOneBy(sc); - return snapshot != null && HYPERVISORS_SUPPORTING_SNAPSHOTS_CHAINING.contains(snapshot.getHypervisorType()); + if (snapshot == null) { + return false; + } + + Hypervisor.HypervisorType hypervisorType = snapshot.getHypervisorType(); + return HYPERVISORS_SUPPORTING_SNAPSHOTS_CHAINING.contains(hypervisorType) || (Hypervisor.HypervisorType.KVM.equals(hypervisorType) && kvmIncrementalSnapshot); + } + + @Override + public List listReadyByVolumeIdAndCheckpointPathNotNull(long volumeId) { + SearchCriteria sc = searchFilteringStoreIdInVolumeIdEqStoreRoleEqStateEqKVMCheckpointNotNull.create(); + sc.setParameters(VOLUME_ID, volumeId); + sc.setParameters(STATE, State.Ready); + return listBy(sc); + } + + @Override + public List listExtractedSnapshotsBeforeDate(Date beforeDate) { + SearchCriteria sc = searchFilterStateAndDownloadUrlNotNullAndDownloadUrlCreatedBefore.create(); + sc.setParameters(URL_CREATED_BEFORE, beforeDate); + sc.setParameters(STATE, State.Ready); + + return listBy(sc); } @Override diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java index 7a466c1f5055..44eb7e6c02cb 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java @@ -29,6 +29,8 @@ import javax.persistence.Temporal; import javax.persistence.TemporalType; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -80,12 +82,18 @@ public class SnapshotDataStoreVO implements StateObject snapshotStatesAbleToDeleteSnapshot = Arrays.asList(Snapshot.State.Destroying, Snapshot.State.Destroyed, Snapshot.State.Error); + private final List snapshotStatesAbleToDeleteSnapshot = Arrays.asList(Snapshot.State.Destroying, Snapshot.State.Destroyed, Snapshot.State.Error, Snapshot.State.Hidden); public SnapshotDataStoreVO getSnapshotImageStoreRef(long snapshotId, long zoneId) { List snaps = snapshotStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image); @@ -161,13 +161,11 @@ public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { VolumeVO volume = volumeDao.findById(snapshot.getVolumeId()); if (oldestSnapshotOnPrimary != null) { if (oldestSnapshotOnPrimary.getDataStoreId() == volume.getPoolId() && oldestSnapshotOnPrimary.getId() != parentSnapshotOnPrimaryStore.getId()) { - int _deltaSnapshotMax = NumbersUtil.parseInt(configDao.getValue("snapshot.delta.max"), - SnapshotManager.DELTAMAX); - int deltaSnap = _deltaSnapshotMax; + int deltaSnap = SnapshotManager.snapshotDeltaMax.value(); int i; for (i = 1; i < deltaSnap; i++) { - Long prevBackupId = parentSnapshotOnBackupStore.getParentSnapshotId(); + long prevBackupId = parentSnapshotOnBackupStore.getParentSnapshotId(); if (prevBackupId == 0) { break; } @@ -177,11 +175,7 @@ public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { } } - if (i >= deltaSnap) { - fullBackup = true; - } else { - fullBackup = false; - } + fullBackup = i >= deltaSnap; } else if (oldestSnapshotOnPrimary.getId() != parentSnapshotOnPrimaryStore.getId()){ // if there is an snapshot entry for previousPool(primary storage) of migrated volume, delete it because CS created one more snapshot entry for current pool snapshotStoreDao.remove(oldestSnapshotOnPrimary.getId()); @@ -204,7 +198,10 @@ protected boolean deleteSnapshotChain(SnapshotInfo snapshot, String storageToStr SnapshotInfo child = snapshot.getChild(); if (child != null) { - logger.debug(String.format("Snapshot [%s] has child [%s], not deleting it on the storage [%s]", snapshotTo, child.getTO(), storageToString)); + logger.debug(String.format("Snapshot [%s] has child [%s], not deleting it on the storage [%s], will only set it as hidden.", snapshotTo, child.getTO(), storageToString)); + SnapshotDataStoreVO snapshotDataStoreVo = snapshotStoreDao.findByStoreSnapshot(snapshot.getDataStore().getRole(), snapshot.getDataStore().getId(), snapshot.getSnapshotId()); + snapshotDataStoreVo.setState(State.Hidden); + snapshotStoreDao.update(snapshotDataStoreVo.getId(), snapshotDataStoreVo); break; } @@ -213,6 +210,7 @@ protected boolean deleteSnapshotChain(SnapshotInfo snapshot, String storageToStr SnapshotInfo parent = snapshot.getParent(); boolean deleted = false; if (parent != null) { + logger.debug("Snapshot [{}] has parent [{}].", snapshot, parent); if (parent.getPath() != null && parent.getPath().equalsIgnoreCase(snapshot.getPath())) { //NOTE: if both snapshots share the same path, it's for xenserver's empty delta snapshot. We can't delete the snapshot on the backend, as parent snapshot still reference to it //Instead, mark it as destroyed in the db. @@ -226,6 +224,7 @@ protected boolean deleteSnapshotChain(SnapshotInfo snapshot, String storageToStr } if (!deleted) { + logger.debug("Deleting snapshot [{}].", snapshot); try { boolean r = snapshotSvr.deleteSnapshot(snapshot); if (r) { @@ -242,6 +241,7 @@ protected boolean deleteSnapshotChain(SnapshotInfo snapshot, String storageToStr } } catch (Exception e) { logger.error(String.format("Failed to delete snapshot [%s] on storage [%s] due to [%s].", snapshotTo, storageToString, e.getMessage()), e); + throw e; } } @@ -249,10 +249,26 @@ protected boolean deleteSnapshotChain(SnapshotInfo snapshot, String storageToStr } while (snapshot != null && snapshotStatesAbleToDeleteSnapshot.contains(snapshot.getState())); } catch (Exception e) { logger.error(String.format("Failed to delete snapshot [%s] on storage [%s] due to [%s].", snapshotTo, storageToString, e.getMessage()), e); + throw new CloudRuntimeException("Failed to delete snapshot chain."); } return result; } + private Long getRootSnapshotId(SnapshotVO snapshotVO) { + List snapshotDataStoreVOList = snapshotStoreDao.findBySnapshotId(snapshotVO.getSnapshotId()); + + long parentId = snapshotDataStoreVOList.stream(). + map(SnapshotDataStoreVO::getParentSnapshotId). + filter(parentSnapshotId -> parentSnapshotId != 0).findFirst().orElse(0L); + while (parentId != 0) { + snapshotDataStoreVOList = snapshotStoreDao.findBySnapshotId(parentId); + parentId = snapshotDataStoreVOList.stream(). + map(SnapshotDataStoreVO::getParentSnapshotId). + filter(parentSnapshotId -> parentSnapshotId != 0).findFirst().orElse(0L); + } + return snapshotDataStoreVOList.stream().map(SnapshotDataStoreVO::getSnapshotId).findFirst().orElse(0L); + } + @Override public boolean deleteSnapshot(Long snapshotId, Long zoneId) { SnapshotVO snapshotVO = snapshotDao.findById(snapshotId); @@ -316,6 +332,9 @@ protected boolean destroySnapshotEntriesAndFiles(SnapshotVO snapshotVo, Long zon } else { snapshotZoneDao.removeSnapshotFromZones(snapshotVo.getId()); } + + updateEndOfChainIfNeeded(snapshotVo); + if (CollectionUtils.isNotEmpty(retrieveSnapshotEntries(snapshotVo.getId(), null))) { return true; } @@ -323,6 +342,37 @@ protected boolean destroySnapshotEntriesAndFiles(SnapshotVO snapshotVo, Long zon return true; } + /** + * If using the KVM hypervisor and the snapshot was the end of a chain, will mark their parents as end of chain. + * */ + protected void updateEndOfChainIfNeeded(SnapshotVO snapshotVo) { + if (!HypervisorType.KVM.equals(snapshotVo.getHypervisorType())) { + return; + } + + SnapshotDataStoreVO snapshotDataStoreVo = snapshotStoreDao.findBySnapshotIdAndDataStoreRoleAndState(snapshotVo.getSnapshotId(), DataStoreRole.Image, State.Destroyed); + + if (snapshotDataStoreVo == null) { + snapshotDataStoreVo = snapshotStoreDao.findBySnapshotIdAndDataStoreRoleAndState(snapshotVo.getSnapshotId(), DataStoreRole.Primary, State.Destroyed); + } + + // Snapshot is hidden, no need to update endOfChain + if (snapshotDataStoreVo == null) { + return; + } + + if (!snapshotDataStoreVo.isEndOfChain() || snapshotDataStoreVo.getParentSnapshotId() <= 0) { + return; + } + + List parentSnapshotDataStoreVoList = findLastAliveAncestors(snapshotDataStoreVo.getParentSnapshotId()); + + for (SnapshotDataStoreVO parentSnapshotDatastoreVo : parentSnapshotDataStoreVoList) { + parentSnapshotDatastoreVo.setEndOfChain(true); + snapshotStoreDao.update(parentSnapshotDatastoreVo.getId(), parentSnapshotDatastoreVo); + } + } + /** * Updates the snapshot to {@link Snapshot.State#Destroyed}. */ @@ -331,17 +381,34 @@ protected void updateSnapshotToDestroyed(SnapshotVO snapshotVo) { snapshotDao.update(snapshotVo.getId(), snapshotVo); } + protected List findLastAliveAncestors(long snapshotId) { + List parentSnapshotDataStoreVoList = snapshotStoreDao.listBySnapshotId(snapshotId); + if (CollectionUtils.isEmpty(parentSnapshotDataStoreVoList)) { + return parentSnapshotDataStoreVoList; + } + if (parentSnapshotDataStoreVoList.stream().anyMatch(snapshotDataStoreVO -> State.Ready.equals(snapshotDataStoreVO.getState()))) { + return parentSnapshotDataStoreVoList; + } + return findLastAliveAncestors(parentSnapshotDataStoreVoList.get(0).getParentSnapshotId()); + } + protected boolean deleteSnapshotInfos(SnapshotVO snapshotVo, Long zoneId) { - List snapshotInfos = retrieveSnapshotEntries(snapshotVo.getId(), zoneId); + return Transaction.execute((TransactionCallback) status -> { + long rootSnapshotId = getRootSnapshotId(snapshotVo); + snapshotDao.acquireInLockTable(rootSnapshotId); - boolean result = false; - for (var snapshotInfo : snapshotInfos) { - if (BooleanUtils.toBooleanDefaultIfNull(deleteSnapshotInfo(snapshotInfo, snapshotVo), false)) { - result = true; - } - } + List snapshotInfos = retrieveSnapshotEntries(snapshotVo.getId(), zoneId); + logger.debug("Found {} snapshot references to delete.", snapshotInfos); - return result; + boolean result = false; + for (var snapshotInfo : snapshotInfos) { + if (BooleanUtils.toBooleanDefaultIfNull(deleteSnapshotInfo(snapshotInfo, snapshotVo), false)) { + result = true; + } + } + snapshotDao.releaseFromLockTable(rootSnapshotId); + return result; + }); } /** @@ -351,59 +418,31 @@ protected boolean deleteSnapshotInfos(SnapshotVO snapshotVo, Long zoneId) { protected Boolean deleteSnapshotInfo(SnapshotInfo snapshotInfo, SnapshotVO snapshotVo) { DataStore dataStore = snapshotInfo.getDataStore(); String storageToString = String.format("%s {uuid: \"%s\", name: \"%s\"}", dataStore.getRole().name(), dataStore.getUuid(), dataStore.getName()); - List snapshotStoreRefs = snapshotStoreDao.findBySnapshotId(snapshotVo.getId()); + List snapshotStoreRefs = snapshotStoreDao.findBySnapshotIdAndNotInDestroyedHiddenState(snapshotVo.getId()); boolean isLastSnapshotRef = CollectionUtils.isEmpty(snapshotStoreRefs) || snapshotStoreRefs.size() == 1; try { SnapshotObject snapshotObject = castSnapshotInfoToSnapshotObject(snapshotInfo); if (isLastSnapshotRef) { snapshotObject.processEvent(Snapshot.Event.DestroyRequested); } - if (!DataStoreRole.Primary.equals(dataStore.getRole())) { - verifyIfTheSnapshotIsBeingUsedByAnyVolume(snapshotObject); - if (deleteSnapshotChain(snapshotInfo, storageToString)) { - logger.debug(String.format("%s was deleted on %s. We will mark the snapshot as destroyed.", snapshotVo, storageToString)); - } else { - logger.debug(String.format("%s was not deleted on %s; however, we will mark the snapshot as destroyed for future garbage collecting.", snapshotVo, - storageToString)); - } - snapshotStoreDao.updateDisplayForSnapshotStoreRole(snapshotVo.getId(), dataStore.getId(), dataStore.getRole(), false); - if (isLastSnapshotRef) { - snapshotObject.processEvent(Snapshot.Event.OperationSucceeded); - } - return true; - } else if (deleteSnapshotInPrimaryStorage(snapshotInfo, snapshotVo, storageToString, snapshotObject, isLastSnapshotRef)) { - snapshotStoreDao.updateDisplayForSnapshotStoreRole(snapshotVo.getId(), dataStore.getId(), dataStore.getRole(), false); - return true; + verifyIfTheSnapshotIsBeingUsedByAnyVolume(snapshotObject); + if (deleteSnapshotChain(snapshotInfo, storageToString)) { + logger.debug(String.format("%s was deleted on %s. We will mark the snapshot as destroyed.", snapshotVo, storageToString)); + } else { + logger.debug(String.format("%s was not deleted on %s; however, we will mark the snapshot as hidden for future garbage collecting.", snapshotVo, + storageToString)); } - logger.debug(String.format("Failed to delete %s on %s.", snapshotVo, storageToString)); + snapshotStoreDao.updateDisplayForSnapshotStoreRole(snapshotVo.getId(), dataStore.getId(), dataStore.getRole(), false); if (isLastSnapshotRef) { - snapshotObject.processEvent(Snapshot.Event.OperationFailed); + snapshotObject.processEvent(Snapshot.Event.OperationSucceeded); } + return true; } catch (NoTransitionException ex) { logger.warn(String.format("Failed to delete %s on %s due to %s.", snapshotVo, storageToString, ex.getMessage()), ex); } return false; } - protected boolean deleteSnapshotInPrimaryStorage(SnapshotInfo snapshotInfo, SnapshotVO snapshotVo, - String storageToString, SnapshotObject snapshotObject, boolean isLastSnapshotRef) throws NoTransitionException { - try { - if (snapshotSvr.deleteSnapshot(snapshotInfo)) { - String msg = String.format("%s was deleted on %s.", snapshotVo, storageToString); - if (isLastSnapshotRef) { - msg = String.format("%s We will mark the snapshot as destroyed.", msg); - snapshotObject.processEvent(Snapshot.Event.OperationSucceeded); - } - logger.debug(msg); - return true; - } - } catch (CloudRuntimeException ex) { - logger.warn(String.format("Unable do delete snapshot %s on %s due to [%s]. The reference will be marked as 'Destroying' for future garbage collecting.", - snapshotVo, storageToString, ex.getMessage()), ex); - } - return false; - } - protected void verifyIfTheSnapshotIsBeingUsedByAnyVolume(SnapshotObject snapshotObject) throws NoTransitionException { List volumesFromSnapshot = _volumeDetailsDaoImpl.findDetails("SNAPSHOT_ID", String.valueOf(snapshotObject.getSnapshotId()), null); if (CollectionUtils.isEmpty(volumesFromSnapshot)) { diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index a3964bd461ec..178d42f5c743 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -41,7 +41,6 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.to.SnapshotObjectTO; -import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -65,8 +64,11 @@ public class SnapshotObject implements SnapshotInfo { protected Logger logger = LogManager.getLogger(getClass()); private SnapshotVO snapshot; private DataStore store; + private DataStore imageStore; private Object payload; private Boolean fullBackup; + private String checkpointPath; + private boolean kvmIncrementalSnapshot = false; private String url; @Inject protected SnapshotDao snapshotDao; @@ -111,12 +113,14 @@ public DataStore getStore() { @Override public SnapshotInfo getParent() { - + logger.trace("Searching for parents of snapshot [{}], in store [{}] with role [{}].", snapshot.getSnapshotId(), store.getId(), store.getRole()); SnapshotDataStoreVO snapStoreVO = snapshotStoreDao.findByStoreSnapshot(store.getRole(), store.getId(), snapshot.getId()); - Long parentId = null; if (snapStoreVO != null) { - parentId = snapStoreVO.getParentSnapshotId(); - if (parentId != null && parentId != 0) { + long parentId = snapStoreVO.getParentSnapshotId(); + if (parentId != 0) { + if (HypervisorType.KVM.equals(snapshot.getHypervisorType())) { + return getCorrectIncrementalParent(parentId); + } return snapshotFactory.getSnapshot(parentId, store); } } @@ -124,6 +128,30 @@ public SnapshotInfo getParent() { return null; } + /** + * Returns the snapshotInfo of the passed snapshot parentId. Will search for the snapshot reference which has a checkpoint path. If none is found, throws an exception. + * */ + protected SnapshotInfo getCorrectIncrementalParent(long parentId) { + List parentSnapshotDatastoreVos = snapshotStoreDao.findBySnapshotId(parentId); + + if (parentSnapshotDatastoreVos.isEmpty()) { + return null; + } + + logger.debug("Found parent snapshot references {}, will filter to just one.", parentSnapshotDatastoreVos); + + SnapshotDataStoreVO parent = parentSnapshotDatastoreVos.stream().filter(snapshotDataStoreVO -> snapshotDataStoreVO.getKvmCheckpointPath() != null) + .findFirst(). + orElseThrow(() -> new CloudRuntimeException(String.format("Could not find snapshot parent with id [%s]. None of the records have a checkpoint path.", parentId))); + + SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(parentId, parent.getDataStoreId(), parent.getRole()); + snapshotInfo.setKvmIncrementalSnapshot(parent.getKvmCheckpointPath() != null); + + logger.debug("Filtered snapshot references {} to just {}.", parentSnapshotDatastoreVos, parent); + + return snapshotInfo; + } + @Override public SnapshotInfo getChild() { QueryBuilder sc = QueryBuilder.create(SnapshotDataStoreVO.class); @@ -216,6 +244,16 @@ public DataStore getDataStore() { return store; } + @Override + public DataStore getImageStore() { + return imageStore; + } + + @Override + public void setImageStore(DataStore imageStore) { + this.imageStore = imageStore; + } + @Override public Long getSize() { return snapshot.getSize(); @@ -453,6 +491,26 @@ public Boolean getFullBackup() { return fullBackup; } + @Override + public String getCheckpointPath() { + return checkpointPath; + } + + @Override + public void setCheckpointPath(String checkpointPath) { + this.checkpointPath = checkpointPath; + } + + @Override + public void setKvmIncrementalSnapshot(boolean isKvmIncrementalSnapshot) { + this.kvmIncrementalSnapshot = isKvmIncrementalSnapshot; + } + + @Override + public boolean isKvmIncrementalSnapshot() { + return kvmIncrementalSnapshot; + } + @Override public boolean delete() { if (store != null) { @@ -468,8 +526,7 @@ public Class getEntityType() { @Override public String toString() { - return String.format("SnapshotObject %s", - ReflectionToStringBuilderUtils.reflectOnlySelectedFields( - this, "snapshot", "store")); + return String.format("%s, dataStoreId %s, imageStore id %s, checkpointPath %s.", snapshot, store != null? store.getId() : 0, + imageStore != null ? imageStore.getId() : 0, checkpointPath); } } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index 2173aba3f056..567df1262f62 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -22,6 +22,14 @@ import javax.inject.Inject; +import com.cloud.agent.api.ConvertSnapshotAnswer; +import com.cloud.agent.api.ConvertSnapshotCommand; +import com.cloud.agent.api.RemoveBitmapCommand; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.Volume; +import com.cloud.storage.snapshot.SnapshotManager; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionService; @@ -38,6 +46,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageAction; import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; @@ -50,6 +59,7 @@ import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; @@ -57,6 +67,8 @@ import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -101,6 +113,8 @@ public class SnapshotServiceImpl implements SnapshotService { EndPointSelector epSelector; @Inject ConfigurationDao _configDao; + @Inject + HostDao hostDao; @Inject private HeuristicRuleHelper heuristicRuleHelper; @@ -232,27 +246,27 @@ protected Void createSnapshotAsyncCallback(AsyncCallbackDispatcher future = new AsyncCallFuture(); try { - CreateSnapshotContext context = new CreateSnapshotContext(null, snap.getBaseVolume(), snapshotOnPrimary, future); + CreateSnapshotContext context = new CreateSnapshotContext(null, snap.getBaseVolume(), snapshotOnPrimaryStorage, future); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createSnapshotAsyncCallback(null, null)).setContext(context); - PrimaryDataStoreDriver primaryStore = (PrimaryDataStoreDriver)snapshotOnPrimary.getDataStore().getDriver(); + PrimaryDataStoreDriver primaryStore = (PrimaryDataStoreDriver)snapshotOnPrimaryStorage.getDataStore().getDriver(); primaryStore.takeSnapshot(snapshot, caller); } catch (Exception e) { logger.debug("Failed to take snapshot: {}", snapshot, e); @@ -281,22 +295,56 @@ public SnapshotResult takeSnapshot(SnapshotInfo snap) { try { result = future.get(); + + updateSnapSizeAndCheckpointPathIfPossible(result, snap); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_ON_PRIMARY, snap.getAccountId(), snap.getDataCenterId(), snap.getId(), - snap.getName(), null, null, snapshotOnPrimary.getSize(), snapshotOnPrimary.getSize(), snap.getClass().getName(), snap.getUuid()); + snap.getName(), null, null, snapshotOnPrimaryStorage.getSize(), snapshotOnPrimaryStorage.getSize(), snap.getClass().getName(), snap.getUuid()); return result; - } catch (InterruptedException e) { - logger.debug("Failed to create snapshot", e); - throw new CloudRuntimeException("Failed to create snapshot", e); - } catch (ExecutionException e) { - logger.debug("Failed to create snapshot", e); - throw new CloudRuntimeException("Failed to create snapshot", e); + } catch (InterruptedException | ExecutionException e) { + String message = String.format("Failed to create snapshot [%s] due to [%s].", snapshot, e.getMessage()); + logger.error(message, e); + throw new CloudRuntimeException(message, e); } } + /** + * Updates the snapshot physical size if the answer is an instance of CreateObjectAnswer and the returned physical size if bigger than 0. + * Also updates the checkpoint path if possible. + * */ + protected void updateSnapSizeAndCheckpointPathIfPossible(SnapshotResult result, SnapshotInfo snapshotInfo) { + SnapshotDataStoreVO snapshotStore; + Answer answer = result.getAnswer(); + + if (!answer.getResult() || !(answer instanceof CreateObjectAnswer)) { + return; + } + + SnapshotInfo resultSnapshot = result.getSnapshot(); + if (snapshotInfo.getImageStore() != null) { + snapshotInfo.getImageStore().create(resultSnapshot); + snapshotStore = _snapshotStoreDao.findBySnapshotIdAndDataStoreRoleAndState(resultSnapshot.getSnapshotId(), DataStoreRole.Image, ObjectInDataStoreStateMachine.State.Allocated); + } else { + snapshotStore = _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Primary, resultSnapshot.getDataStore().getId(), resultSnapshot.getSnapshotId()); + } + + SnapshotObjectTO snapshotObjectTo = (SnapshotObjectTO) ((CreateObjectAnswer) answer).getData(); + + Long physicalSize = snapshotObjectTo.getPhysicalSize(); + if (NumberUtils.compare(physicalSize, 0L) > 0) { + snapshotStore.setPhysicalSize(physicalSize); + } + + snapshotStore.setKvmCheckpointPath(snapshotObjectTo.getCheckpointPath()); + _snapshotStoreDao.update(snapshotStore.getId(), snapshotStore); + } + + // if a snapshot has parent snapshot, the new snapshot should be stored in // the same store as its parent since // we are taking delta snapshot - private DataStore findSnapshotImageStore(SnapshotInfo snapshot) { + @Override + public DataStore findSnapshotImageStore(SnapshotInfo snapshot) { Boolean fullSnapshot = true; Boolean snapshotFullBackup = snapshot.getFullBackup(); if (snapshotFullBackup != null) { @@ -339,6 +387,49 @@ protected DataStore getImageStoreForSnapshot(Long dataCenterId, SnapshotInfo sna return imageStore; } + /** + * Converts a given snapshot that is on the secondary storage. The original and its backing chains will be maintained, the converted snapshot must be later deleted if not used. + * The original purpose of this method is to work with KVM incremental snapshots, copying the snapshot to a temporary location and consolidating the snapshot chain. + * @param snapshotInfo The snapshot to be converted + * @return the snapshotInfo given with the updated path. This should not be persisted on the DB, otherwise the original snapshot will be lost. + * */ + @Override + public SnapshotInfo convertSnapshot(SnapshotInfo snapshotInfo) { + SnapshotObject snapObj = (SnapshotObject)snapshotInfo; + + logger.debug("Converting snapshot [%s].", snapObj); + Answer answer = null; + try { + snapObj.processEvent(Snapshot.Event.BackupToSecondary); + + SnapshotObjectTO snapshotObjectTO = (SnapshotObjectTO) snapshotInfo.getTO(); + ConvertSnapshotCommand cmd = new ConvertSnapshotCommand(snapshotObjectTO); + + EndPoint ep = epSelector.select(snapshotInfo, StorageAction.CONVERTSNAPSHOT); + + answer = ep.sendMessage(cmd); + + if (answer != null && answer.getResult()) { + snapObj.setPath(((ConvertSnapshotAnswer) answer).getSnapshotObjectTO().getPath()); + return snapObj; + } + } catch (NoTransitionException e) { + logger.debug("Failed to change snapshot {} state.", snapObj.getUuid(), e); + } finally { + try { + if (answer != null && answer.getResult()) { + snapObj.processEvent(Snapshot.Event.OperationSucceeded); + } else { + snapObj.processEvent(Snapshot.Event.OperationNotPerformed); + } + } catch (NoTransitionException ex) { + logger.debug("Failed to change snapshot {} state.", snapObj.getUuid(), ex); + } + } + + throw new CloudRuntimeException(String.format("Failed to convert snapshot [%s]%s.", snapObj.getUuid(), answer != null ? String.format(" due to [%s]", answer.getDetails()) : "")); + } + @Override public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { SnapshotObject snapObj = (SnapshotObject)snapshot; @@ -404,7 +495,7 @@ protected Void copySnapshotAsyncCallback(AsyncCallbackDispatcher future = new AsyncCallFuture(); DeleteSnapshotContext context = new DeleteSnapshotContext(null, snapInfo, future); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); @@ -551,6 +653,25 @@ public boolean deleteSnapshot(SnapshotInfo snapInfo) { return false; } + protected void deleteBitmap (SnapshotInfo snapshotInfo) { + Volume baseVol = snapshotInfo.getBaseVolume(); + if (baseVol == null || !Volume.State.Ready.equals(baseVol.getState())) { + return; + } + + VirtualMachine attachedVM = snapshotInfo.getBaseVolume().getAttachedVM(); + + RemoveBitmapCommand cmd = new RemoveBitmapCommand((SnapshotObjectTO) snapshotInfo.getTO(), + attachedVM != null && attachedVM.getState().equals(VirtualMachine.State.Running)); + EndPoint ep = epSelector.select(snapshotInfo, StorageAction.REMOVEBITMAP); + + Answer answer = ep.sendMessage(cmd); + if (!answer.getResult()) { + logger.error("Unable to remove bitmap associated with snapshot {} due to {}.", answer.getDetails()); + throw new CloudRuntimeException(String.format("Unable to remove bitmap associated with snapshot [%s].", snapshotInfo.getName())); + } + } + @Override public boolean revertSnapshot(SnapshotInfo snapshot) { PrimaryDataStore store = null; diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java index 09e5c85b770f..53f98c18f1be 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; +import com.cloud.hypervisor.Hypervisor; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; @@ -77,7 +78,7 @@ public class DefaultSnapshotStrategyTest { VolumeDetailsDao volumeDetailsDaoMock; @Mock - SnapshotService snapshotServiceMock; + private SnapshotDataStoreVO snapshotDataStoreVOMock; @Mock SnapshotZoneDao snapshotZoneDaoMock; @@ -88,6 +89,9 @@ public class DefaultSnapshotStrategyTest { @Mock DataStoreManager dataStoreManager; + @Mock + SnapshotService snapshotService; + List mockSnapshotInfos = new ArrayList<>(); @Before @@ -106,6 +110,49 @@ public void validateRetrieveSnapshotEntries() { Assert.assertTrue(result.contains(snapshotInfo2Mock)); } + @Test + public void updateEndOfChainIfNeededTestNotKvm() { + Mockito.doReturn(Hypervisor.HypervisorType.VMware).when(snapshotVoMock).getHypervisorType(); + + defaultSnapshotStrategySpy.updateEndOfChainIfNeeded(snapshotVoMock); + + Mockito.verify(snapshotDataStoreDao, Mockito.never()).findBySnapshotIdAndDataStoreRoleAndState(Mockito.anyLong(), Mockito.any(), Mockito.any()); + } + + @Test + public void updateEndOfChainIfNeededTestKvmAndIsNotEndOfChain() { + Mockito.doReturn(Hypervisor.HypervisorType.KVM).when(snapshotVoMock).getHypervisorType(); + Mockito.doReturn(2L).when(snapshotVoMock).getSnapshotId(); + + SnapshotDataStoreVO snapshotDataStoreVO = new SnapshotDataStoreVO(); + snapshotDataStoreVO.setEndOfChain(false); + Mockito.doReturn(snapshotDataStoreVO).when(snapshotDataStoreDao).findBySnapshotIdAndDataStoreRoleAndState(2, DataStoreRole.Image, ObjectInDataStoreStateMachine.State.Destroyed); + + defaultSnapshotStrategySpy.updateEndOfChainIfNeeded(snapshotVoMock); + + Mockito.verify(snapshotDataStoreDao, Mockito.never()).update(Mockito.anyLong(), Mockito.any()); + } + + @Test + public void updateEndOfChainIfNeededTestKvmAndIsEndOfChain() { + Mockito.doReturn(Hypervisor.HypervisorType.KVM).when(snapshotVoMock).getHypervisorType(); + Mockito.doReturn(2L).when(snapshotVoMock).getSnapshotId(); + + SnapshotDataStoreVO snapshotDataStoreVO = new SnapshotDataStoreVO(); + snapshotDataStoreVO.setEndOfChain(true); + snapshotDataStoreVO.setParentSnapshotId(8); + Mockito.doReturn(snapshotDataStoreVO).when(snapshotDataStoreDao).findBySnapshotIdAndDataStoreRoleAndState(2, DataStoreRole.Image, ObjectInDataStoreStateMachine.State.Destroyed); + + Mockito.doReturn(ObjectInDataStoreStateMachine.State.Ready).when(snapshotDataStoreVOMock).getState(); + Mockito.doReturn(List.of(snapshotDataStoreVOMock)).when(snapshotDataStoreDao).listBySnapshotId(8); + + defaultSnapshotStrategySpy.updateEndOfChainIfNeeded(snapshotVoMock); + + Mockito.verify(snapshotDataStoreDao, Mockito.times(1)).listBySnapshotId(Mockito.anyLong()); + Mockito.verify(snapshotDataStoreVOMock).setEndOfChain(true); + Mockito.verify(snapshotDataStoreDao, Mockito.times(1)).update(Mockito.anyLong(), Mockito.any()); + } + @Test public void validateUpdateSnapshotToDestroyed() { Mockito.doReturn(true).when(snapshotDaoMock).update(Mockito.anyLong(), Mockito.any()); @@ -146,37 +193,12 @@ public void deleteSnapshotInfoTestReturnTrueIfCanDeleteTheSnapshotOnPrimaryStora Mockito.doReturn(dataStoreMock).when(snapshotInfo1Mock).getDataStore(); Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); - Mockito.doReturn(true).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Primary); boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); Assert.assertTrue(result); } - @Test - public void deleteSnapshotInfoTestReturnFalseIfCannotDeleteTheSnapshotOnPrimaryStorage() throws NoTransitionException { - Mockito.doReturn(dataStoreMock).when(snapshotInfo1Mock).getDataStore(); - Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); - Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); - Mockito.doReturn(false).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Primary); - - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); - Assert.assertFalse(result); - } - - @Test - public void deleteSnapshotInfoTestReturnFalseIfDeleteSnapshotOnPrimaryStorageThrowsACloudRuntimeException() throws NoTransitionException { - Mockito.doReturn(dataStoreMock).when(snapshotInfo1Mock).getDataStore(); - Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); - Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); - Mockito.doThrow(CloudRuntimeException.class).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Primary); - - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); - Assert.assertFalse(result); - } - @Test public void deleteSnapshotInfoTestReturnTrueIfCanDeleteTheSnapshotChainForSecondaryStorage() throws NoTransitionException { Mockito.doReturn(dataStoreMock).when(snapshotInfo1Mock).getDataStore(); @@ -233,31 +255,6 @@ public void verifyIfTheSnapshotIsBeingUsedByAnyVolumeTestDetailsIsNotEmptyThrowC defaultSnapshotStrategySpy.verifyIfTheSnapshotIsBeingUsedByAnyVolume(snapshotObjectMock); } - @Test - public void deleteSnapshotInPrimaryStorageTestReturnTrueIfDeleteReturnsTrue() throws NoTransitionException { - Mockito.doReturn(true).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); - Assert.assertTrue(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, snapshotObjectMock, true)); - } - - @Test - public void deleteSnapshotInPrimaryStorageTestReturnTrueIfDeleteNotLastRefReturnsTrue() throws NoTransitionException { - Mockito.doReturn(true).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Assert.assertTrue(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, snapshotObjectMock, false)); - } - - @Test - public void deleteSnapshotInPrimaryStorageTestReturnFalseIfDeleteReturnsFalse() throws NoTransitionException { - Mockito.doReturn(false).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, null, true)); - } - - @Test - public void deleteSnapshotInPrimaryStorageTestReturnFalseIfDeleteThrowsException() throws NoTransitionException { - Mockito.doThrow(CloudRuntimeException.class).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); - Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInPrimaryStorage(null, null, null, null, true)); - } - @Test public void testGetSnapshotImageStoreRefNull() { SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java index 6d59b6f36e0a..e32725ebb23d 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java @@ -18,7 +18,9 @@ */ package org.apache.cloudstack.storage.snapshot; +import com.cloud.agent.api.Answer; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ImageStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; @@ -30,7 +32,11 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.storage.command.CreateObjectAnswer; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -61,7 +67,7 @@ public class SnapshotServiceImplTest { HeuristicRuleHelper heuristicRuleHelperMock; @Mock - SnapshotInfo snapshotMock; + SnapshotInfo snapshotInfoMock; @Mock VolumeInfo volumeInfoMock; @@ -69,12 +75,34 @@ public class SnapshotServiceImplTest { @Mock DataStoreManager dataStoreManagerMock; + @Mock + private SnapshotResult snapshotResultMock; + + @Mock + private CreateObjectAnswer createObjectAnswerMock; + + @Mock + private ImageStore imageStoreMock; + + @Mock + private SnapshotDataStoreVO snapshotDataStoreVoMock; + + @Mock + private SnapshotDataStoreDao snapshotDataStoreDaoMock; + + @Mock + private DataStore dataStoreMock; + + @Mock + private SnapshotObjectTO snapshotObjectTOMock; + + private static final long DUMMY_ID = 1L; @Test public void testRevertSnapshotWithNoPrimaryStorageEntry() throws Exception { - Mockito.when(snapshotMock.getId()).thenReturn(DUMMY_ID); - Mockito.when(snapshotMock.getVolumeId()).thenReturn(DUMMY_ID); + Mockito.when(snapshotInfoMock.getId()).thenReturn(DUMMY_ID); + Mockito.when(snapshotInfoMock.getVolumeId()).thenReturn(DUMMY_ID); Mockito.when(_snapshotFactory.getSnapshotOnPrimaryStore(1L)).thenReturn(null); Mockito.when(volFactory.getVolume(DUMMY_ID, DataStoreRole.Primary)).thenReturn(volumeInfoMock); @@ -89,7 +117,7 @@ public void testRevertSnapshotWithNoPrimaryStorageEntry() throws Exception { Mockito.when(mock.get()).thenReturn(result); Mockito.when(result.isFailed()).thenReturn(false); })) { - Assert.assertTrue(snapshotService.revertSnapshot(snapshotMock)); + Assert.assertTrue(snapshotService.revertSnapshot(snapshotInfoMock)); } } @@ -97,9 +125,9 @@ public void testRevertSnapshotWithNoPrimaryStorageEntry() throws Exception { public void getImageStoreForSnapshotTestShouldListFreeImageStoresWithNoHeuristicRule() { Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))). thenReturn(null); - Mockito.when(snapshotMock.getDataCenterId()).thenReturn(DUMMY_ID); + Mockito.when(snapshotInfoMock.getDataCenterId()).thenReturn(DUMMY_ID); - snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock); + snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotInfoMock); Mockito.verify(dataStoreManagerMock, Mockito.times(1)).getImageStoreWithFreeCapacity(Mockito.anyLong()); } @@ -110,8 +138,123 @@ public void getImageStoreForSnapshotTestShouldReturnImageStoreReturnedByTheHeuri Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))). thenReturn(dataStore); - snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock); + snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotInfoMock); Mockito.verify(dataStoreManagerMock, Mockito.times(0)).getImageStoreWithFreeCapacity(Mockito.anyLong()); } + + @Test + public void updateSnapSizeAndCheckpointPathIfPossibleTestResultIsFalse() { + Mockito.doReturn(createObjectAnswerMock).when(snapshotResultMock).getAnswer(); + Mockito.doReturn(false).when(createObjectAnswerMock).getResult(); + + snapshotService.updateSnapSizeAndCheckpointPathIfPossible(snapshotResultMock, snapshotInfoMock); + + Mockito.verify(snapshotInfoMock, Mockito.never()).getImageStore(); + } + + @Test + public void updateSnapSizeAndCheckpointPathIfPossibleTestResultIsTrueAnswerIsNotCreateObjectAnswer() { + Answer answer = new Answer(null, true, null); + + Mockito.doReturn(answer).when(snapshotResultMock).getAnswer(); + + snapshotService.updateSnapSizeAndCheckpointPathIfPossible(snapshotResultMock, snapshotInfoMock); + + Mockito.verify(snapshotInfoMock, Mockito.never()).getImageStore(); + } + + @Test + public void updateSnapSizeAndCheckpointPathIfPossibleTestResultIsTrueAnswerIsCreateObjectAnswerAndImageStoreIsNotNullAndPhysicalSizeIsZero() { + Mockito.doReturn(createObjectAnswerMock).when(snapshotResultMock).getAnswer(); + Mockito.doReturn(true).when(createObjectAnswerMock).getResult(); + + Mockito.doReturn(snapshotInfoMock).when(snapshotResultMock).getSnapshot(); + + Mockito.doReturn(dataStoreMock).when(snapshotInfoMock).getImageStore(); + + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findBySnapshotIdAndDataStoreRoleAndState(Mockito.anyLong(), Mockito.any(), Mockito.any()); + + Mockito.doReturn(snapshotObjectTOMock).when(createObjectAnswerMock).getData(); + Mockito.doReturn("checkpath").when(snapshotObjectTOMock).getCheckpointPath(); + Mockito.doReturn(0L).when(snapshotObjectTOMock).getPhysicalSize(); + + snapshotService.updateSnapSizeAndCheckpointPathIfPossible(snapshotResultMock, snapshotInfoMock); + + Mockito.verify(snapshotDataStoreVoMock).setKvmCheckpointPath("checkpath"); + Mockito.verify(snapshotDataStoreVoMock, Mockito.never()).setPhysicalSize(Mockito.anyLong()); + + Mockito.verify(snapshotDataStoreDaoMock).update(Mockito.anyLong(), Mockito.any()); + } + + @Test + public void updateSnapSizeAndCheckpointPathIfPossibleTestResultIsTrueAnswerIsCreateObjectAnswerAndImageStoreIsNotNullAndPhysicalSizeGreaterThanZero() { + Mockito.doReturn(createObjectAnswerMock).when(snapshotResultMock).getAnswer(); + Mockito.doReturn(true).when(createObjectAnswerMock).getResult(); + + Mockito.doReturn(snapshotInfoMock).when(snapshotResultMock).getSnapshot(); + + Mockito.doReturn(dataStoreMock).when(snapshotInfoMock).getImageStore(); + + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findBySnapshotIdAndDataStoreRoleAndState(Mockito.anyLong(), Mockito.any(), Mockito.any()); + + Mockito.doReturn(snapshotObjectTOMock).when(createObjectAnswerMock).getData(); + Mockito.doReturn("checkpath").when(snapshotObjectTOMock).getCheckpointPath(); + Mockito.doReturn(1000L).when(snapshotObjectTOMock).getPhysicalSize(); + + snapshotService.updateSnapSizeAndCheckpointPathIfPossible(snapshotResultMock, snapshotInfoMock); + + Mockito.verify(snapshotDataStoreVoMock).setKvmCheckpointPath("checkpath"); + Mockito.verify(snapshotDataStoreVoMock).setPhysicalSize(1000L); + + Mockito.verify(snapshotDataStoreDaoMock).update(Mockito.anyLong(), Mockito.any()); + } + + @Test + public void updateSnapSizeAndCheckpointPathIfPossibleTestResultIsTrueAnswerIsCreateObjectAnswerAndImageStoreIsNullAndPhysicalSizeIsZero() { + Mockito.doReturn(createObjectAnswerMock).when(snapshotResultMock).getAnswer(); + Mockito.doReturn(true).when(createObjectAnswerMock).getResult(); + + Mockito.doReturn(snapshotInfoMock).when(snapshotResultMock).getSnapshot(); + + Mockito.doReturn(null).when(snapshotInfoMock).getImageStore(); + Mockito.doReturn(dataStoreMock).when(snapshotInfoMock).getDataStore(); + + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findByStoreSnapshot(Mockito.any(), Mockito.anyLong(), Mockito.anyLong()); + + Mockito.doReturn(snapshotObjectTOMock).when(createObjectAnswerMock).getData(); + Mockito.doReturn("checkpath").when(snapshotObjectTOMock).getCheckpointPath(); + Mockito.doReturn(0L).when(snapshotObjectTOMock).getPhysicalSize(); + + snapshotService.updateSnapSizeAndCheckpointPathIfPossible(snapshotResultMock, snapshotInfoMock); + + Mockito.verify(snapshotDataStoreVoMock).setKvmCheckpointPath("checkpath"); + Mockito.verify(snapshotDataStoreVoMock, Mockito.never()).setPhysicalSize(Mockito.anyLong()); + + Mockito.verify(snapshotDataStoreDaoMock).update(Mockito.anyLong(), Mockito.any()); + } + + @Test + public void updateSnapSizeAndCheckpointPathIfPossibleTestResultIsTrueAnswerIsCreateObjectAnswerAndImageStoreIsNullAndPhysicalSizeGreaterThanZero() { + Mockito.doReturn(createObjectAnswerMock).when(snapshotResultMock).getAnswer(); + Mockito.doReturn(true).when(createObjectAnswerMock).getResult(); + + Mockito.doReturn(snapshotInfoMock).when(snapshotResultMock).getSnapshot(); + + Mockito.doReturn(null).when(snapshotInfoMock).getImageStore(); + Mockito.doReturn(dataStoreMock).when(snapshotInfoMock).getDataStore(); + + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findByStoreSnapshot(Mockito.any(), Mockito.anyLong(), Mockito.anyLong()); + + Mockito.doReturn(snapshotObjectTOMock).when(createObjectAnswerMock).getData(); + Mockito.doReturn("checkpath").when(snapshotObjectTOMock).getCheckpointPath(); + Mockito.doReturn(1000L).when(snapshotObjectTOMock).getPhysicalSize(); + + snapshotService.updateSnapSizeAndCheckpointPathIfPossible(snapshotResultMock, snapshotInfoMock); + + Mockito.verify(snapshotDataStoreVoMock).setKvmCheckpointPath("checkpath"); + Mockito.verify(snapshotDataStoreVoMock).setPhysicalSize(1000L); + + Mockito.verify(snapshotDataStoreDaoMock).update(Mockito.anyLong(), Mockito.any()); + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java index 570a47a752c6..e5c8c69ddf55 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java @@ -18,6 +18,10 @@ import javax.inject.Inject; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.ImageStore; +import com.cloud.storage.snapshot.SnapshotManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.springframework.stereotype.Component; @@ -84,6 +88,10 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { SnapshotDao snapshotDao; @Inject VolumeDao volumeDao; + + @Inject + HostDao hostDao; + protected StateMachine2 stateMachines; public ObjectInDataStoreManagerImpl() { @@ -113,6 +121,7 @@ public ObjectInDataStoreManagerImpl() { stateMachines.addTransition(State.Migrating, Event.MigrationSucceeded, State.Destroyed); stateMachines.addTransition(State.Migrating, Event.OperationSuccessed, State.Ready); stateMachines.addTransition(State.Migrating, Event.OperationFailed, State.Ready); + stateMachines.addTransition(State.Hidden, Event.DestroyRequested, State.Destroying); } @Override @@ -130,14 +139,17 @@ public DataObject create(DataObject obj, DataStore dataStore) { ss.setVolumeId(snapshotInfo.getVolumeId()); ss.setSize(snapshotInfo.getSize()); // this is the virtual size of snapshot in primary storage. ss.setPhysicalSize(snapshotInfo.getSize()); // this physical size will get updated with actual size once the snapshot backup is done. - SnapshotDataStoreVO snapshotDataStoreVO = snapshotDataStoreDao.findParent(dataStore.getRole(), dataStore.getId(), snapshotInfo.getVolumeId()); - if (snapshotDataStoreVO != null) { + Long clusterId = hostDao.findClusterIdByVolumeInfo(snapshotInfo.getBaseVolume()); + SnapshotDataStoreVO parentSnapshotDataStoreVO = findParent(dataStore, clusterId, snapshotInfo); + if (parentSnapshotDataStoreVO != null) { //Double check the snapshot is removed or not - SnapshotVO parentSnap = snapshotDao.findById(snapshotDataStoreVO.getSnapshotId()); - if (parentSnap != null) { - ss.setParentSnapshotId(snapshotDataStoreVO.getSnapshotId()); + SnapshotVO parentSnap = snapshotDao.findById(parentSnapshotDataStoreVO.getSnapshotId()); + if (parentSnap != null && !parentSnapshotDataStoreVO.isEndOfChain()) { + ss.setParentSnapshotId(parentSnapshotDataStoreVO.getSnapshotId()); + } else if (parentSnapshotDataStoreVO.isEndOfChain()) { + logger.debug("Snapshot [{}] will begin a new chain, as the last one has finished.", ss.getSnapshotId()); } else { - logger.debug("find inconsistent db for snapshot " + snapshotDataStoreVO.getSnapshotId()); + logger.debug("find inconsistent db for snapshot " + parentSnapshotDataStoreVO.getSnapshotId()); } } ss.setState(ObjectInDataStoreStateMachine.State.Allocated); @@ -179,9 +191,12 @@ public DataObject create(DataObject obj, DataStore dataStore) { ss.setRole(dataStore.getRole()); ss.setSize(snapshot.getSize()); ss.setVolumeId(snapshot.getVolumeId()); - SnapshotDataStoreVO snapshotDataStoreVO = snapshotDataStoreDao.findParent(dataStore.getRole(), dataStore.getId(), snapshot.getVolumeId()); - if (snapshotDataStoreVO != null) { + Long clusterId = hostDao.findClusterIdByVolumeInfo(snapshot.getBaseVolume()); + SnapshotDataStoreVO snapshotDataStoreVO = snapshotDataStoreDao.findParent(dataStore.getRole(), null, ((ImageStore)dataStore).getDataCenterId(), snapshot.getVolumeId(), SnapshotManager.kvmIncrementalSnapshot.valueIn(clusterId), snapshot.getHypervisorType()); + if (snapshotDataStoreVO != null && !snapshotDataStoreVO.isEndOfChain()) { ss.setParentSnapshotId(snapshotDataStoreVO.getSnapshotId()); + } else { + logger.debug("Snapshot [{}] will begin a new chain, as the last one has finished.", ss.getSnapshotId()); } ss.setInstallPath(TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR + "/" + snapshotDao.findById(obj.getId()).getAccountId() + "/" + snapshot.getVolumeId()); ss.setState(ObjectInDataStoreStateMachine.State.Allocated); @@ -201,6 +216,33 @@ public DataObject create(DataObject obj, DataStore dataStore) { return this.get(obj, dataStore, null); } + private SnapshotDataStoreVO findParent(DataStore dataStore, Long clusterId, SnapshotInfo snapshotInfo) { + boolean kvmIncrementalSnapshot = SnapshotManager.kvmIncrementalSnapshot.valueIn(clusterId); + SnapshotDataStoreVO snapshotDataStoreVO; + if (Hypervisor.HypervisorType.KVM.equals(snapshotInfo.getHypervisorType()) && kvmIncrementalSnapshot) { + snapshotDataStoreVO = snapshotDataStoreDao.findParent(null, null, null, snapshotInfo.getVolumeId(), + kvmIncrementalSnapshot, snapshotInfo.getHypervisorType()); + snapshotDataStoreVO = returnNullIfNotOnSameTypeOfStoreRole(snapshotInfo, snapshotDataStoreVO); + } else { + snapshotDataStoreVO = snapshotDataStoreDao.findParent(dataStore.getRole(), dataStore.getId(), null, snapshotInfo.getVolumeId(), + kvmIncrementalSnapshot, snapshotInfo.getHypervisorType()); + } + return snapshotDataStoreVO; + } + + private SnapshotDataStoreVO returnNullIfNotOnSameTypeOfStoreRole(SnapshotInfo snapshotInfo, SnapshotDataStoreVO snapshotDataStoreVO) { + if (snapshotDataStoreVO == null) { + return snapshotDataStoreVO; + } + if ((snapshotInfo.getImageStore() != null && !snapshotDataStoreVO.getRole().isImageStore()) || + (snapshotInfo.getImageStore() == null && snapshotDataStoreVO.getRole().isImageStore())) { + snapshotDataStoreVO.setEndOfChain(true); + snapshotDataStoreDao.update(snapshotDataStoreVO.getId(), snapshotDataStoreVO); + return null; + } + return snapshotDataStoreVO; + } + @Override public boolean delete(DataObject dataObj) { long objId = dataObj.getId(); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index 79be65888995..6febb2582590 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -461,45 +461,107 @@ public EndPoint select(DataObject object, StorageAction action) { @Override public EndPoint select(DataObject object, StorageAction action, boolean encryptionRequired) { - if (action == StorageAction.TAKESNAPSHOT) { - SnapshotInfo snapshotInfo = (SnapshotInfo)object; - if (snapshotInfo.getHypervisorType() == Hypervisor.HypervisorType.KVM) { - VolumeInfo volumeInfo = snapshotInfo.getBaseVolume(); - VirtualMachine vm = volumeInfo.getAttachedVM(); - if ((vm != null) && (vm.getState() == VirtualMachine.State.Running)) { - Long hostId = vm.getHostId(); - return getEndPointFromHostId(hostId); + switch (action) { + case DELETESNAPSHOT: + case TAKESNAPSHOT: + case CONVERTSNAPSHOT: { + SnapshotInfo snapshotInfo = (SnapshotInfo)object; + if (Hypervisor.HypervisorType.KVM.equals(snapshotInfo.getHypervisorType())) { + return getEndPointForSnapshotOperationsInKvm(snapshotInfo, encryptionRequired); } + break; } - } else if (action == StorageAction.MIGRATEVOLUME) { - VolumeInfo volume = (VolumeInfo)object; - if (volume.getHypervisorType() == Hypervisor.HypervisorType.Hyperv || volume.getHypervisorType() == Hypervisor.HypervisorType.VMware) { - VirtualMachine vm = volume.getAttachedVM(); - if ((vm != null) && (vm.getState() == VirtualMachine.State.Running)) { - Long hostId = vm.getHostId(); - return getEndPointFromHostId(hostId); - } + case REMOVEBITMAP: { + return getEndPointForBitmapRemoval(object, encryptionRequired); } - } else if (action == StorageAction.DELETEVOLUME) { - VolumeInfo volume = (VolumeInfo)object; - if (volume.getHypervisorType() == Hypervisor.HypervisorType.VMware) { - VirtualMachine vm = volume.getAttachedVM(); - if (vm != null) { - Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); - if (hostId != null) { + case MIGRATEVOLUME: { + VolumeInfo volume = (VolumeInfo) object; + if (volume.getHypervisorType() == Hypervisor.HypervisorType.Hyperv || volume.getHypervisorType() == Hypervisor.HypervisorType.VMware) { + VirtualMachine vm = volume.getAttachedVM(); + if ((vm != null) && (vm.getState() == VirtualMachine.State.Running)) { + Long hostId = vm.getHostId(); return getEndPointFromHostId(hostId); } } + break; + } + case DELETEVOLUME: { + VolumeInfo volume = (VolumeInfo) object; + if (volume.getHypervisorType() == Hypervisor.HypervisorType.VMware) { + VirtualMachine vm = volume.getAttachedVM(); + if (vm != null) { + Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); + if (hostId != null) { + return getEndPointFromHostId(hostId); + } + } + } + break; } } return select(object, encryptionRequired); } + protected EndPoint getEndPointForBitmapRemoval(DataObject object, boolean encryptionRequired) { + SnapshotInfo snapshotInfo = (SnapshotInfo)object; + VolumeInfo volumeInfo = snapshotInfo.getBaseVolume(); + + logger.debug("Selecting endpoint for bitmap removal of volume [{}].", volumeInfo.getUuid()); + if (volumeInfo.isAttachedVM()) { + VirtualMachine attachedVM = volumeInfo.getAttachedVM(); + if (attachedVM.getHostId() != null) { + return getEndPointFromHostId(attachedVM.getHostId()); + } else if (attachedVM.getLastHostId() != null) { + return getEndPointFromHostId(attachedVM.getLastHostId()); + } + } + return select(volumeInfo, encryptionRequired); + } + + protected EndPoint getEndPointForSnapshotOperationsInKvm(SnapshotInfo snapshotInfo, boolean encryptionRequired) { + VolumeInfo volumeInfo = snapshotInfo.getBaseVolume(); + DataStoreRole snapshotDataStoreRole = snapshotInfo.getDataStore().getRole(); + VirtualMachine vm = volumeInfo.getAttachedVM(); + + logger.debug("Selecting endpoint for operation on snapshot [{}] with encryptionRequired as [{}].", snapshotInfo, encryptionRequired); + if (vm == null) { + if (snapshotDataStoreRole == DataStoreRole.Image) { + return selectRandom(snapshotInfo.getDataCenterId(), Hypervisor.HypervisorType.KVM); + } else { + return select(snapshotInfo, encryptionRequired); + } + } + + if (vm.getState() == VirtualMachine.State.Running) { + return getEndPointFromHostId(vm.getHostId()); + } + + Long hostId = vm.getLastHostId(); + if (hostId != null) { + return getEndPointFromHostId(hostId); + } else if (snapshotDataStoreRole == DataStoreRole.Image) { + return selectRandom(snapshotInfo.getDataCenterId(), Hypervisor.HypervisorType.KVM); + } + + return select(snapshotInfo, encryptionRequired); + } + @Override public EndPoint select(Scope scope, Long storeId) { return findEndPointInScope(scope, findOneHostOnPrimaryStorage, storeId); } + @Override + public EndPoint selectRandom(long zoneId, Hypervisor.HypervisorType hypervisorType) { + List hostVOs = hostDao.listByDataCenterIdAndHypervisorType(zoneId, hypervisorType); + + if (hostVOs.isEmpty()) { + return null; + } + Collections.shuffle(hostVOs); + return RemoteHostEndPoint.getHypervisorHostEndPoint(hostVOs.get(0)); + } + @Override public List selectAll(DataStore store) { List endPoints = new ArrayList(); diff --git a/engine/storage/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java b/engine/storage/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java index cea6ac29f6ef..ac7240275d9c 100644 --- a/engine/storage/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java +++ b/engine/storage/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java @@ -88,7 +88,7 @@ public void isSnapshotChainingRequiredTestSnapshotIsNullReturnFalse() { snapshotDataStoreDaoImplSpy.snapshotVOSearch = searchBuilderMock; Mockito.doReturn(searchCriteriaMock).when(searchBuilderMock).create(); Mockito.doReturn(null).when(snapshotDaoMock).findOneBy(Mockito.any()); - Assert.assertFalse(snapshotDataStoreDaoImplSpy.isSnapshotChainingRequired(2)); + Assert.assertFalse(snapshotDataStoreDaoImplSpy.isSnapshotChainingRequired(2, false)); } @Test @@ -99,7 +99,7 @@ public void isSnapshotChainingRequiredTestSnapshotIsNotNullReturnAccordingHyperv for (Hypervisor.HypervisorType hypervisorType : Hypervisor.HypervisorType.values()) { Mockito.doReturn(hypervisorType).when(snapshotVoMock).getHypervisorType(); - boolean result = snapshotDataStoreDaoImplSpy.isSnapshotChainingRequired(2); + boolean result = snapshotDataStoreDaoImplSpy.isSnapshotChainingRequired(2, false); if (SnapshotDataStoreDaoImpl.HYPERVISORS_SUPPORTING_SNAPSHOTS_CHAINING.contains(hypervisorType)) { Assert.assertTrue(result); diff --git a/engine/storage/src/test/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelectorTest.java b/engine/storage/src/test/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelectorTest.java new file mode 100644 index 000000000000..9f01ab162ba7 --- /dev/null +++ b/engine/storage/src/test/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelectorTest.java @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.endpoint; + + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.DataStoreRole; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultEndPointSelectorTest { + + @Mock + private VirtualMachine virtualMachineMock; + + @Mock + private VolumeInfo volumeInfoMock; + + @Mock + private SnapshotInfo snapshotInfoMock; + + @Mock + private DataStore datastoreMock; + + @Spy + private DefaultEndPointSelector defaultEndPointSelectorSpy; + + @Before + public void setup() { + Mockito.doReturn(volumeInfoMock).when(snapshotInfoMock).getBaseVolume(); + } + + @Test + public void getEndPointForBitmapRemovalTestVolumeIsNotAttached() { + Mockito.doReturn(false).when(volumeInfoMock).isAttachedVM(); + Mockito.doReturn(null).when(defaultEndPointSelectorSpy).select(volumeInfoMock, false); + + defaultEndPointSelectorSpy.getEndPointForBitmapRemoval(snapshotInfoMock, false); + + Mockito.verify(defaultEndPointSelectorSpy, Mockito.times(1)).select(volumeInfoMock, false); + } + + @Test + public void getEndPointForBitmapRemovalTestVolumeIsAttachedHostIdIsSet() { + Mockito.doReturn(true).when(volumeInfoMock).isAttachedVM(); + Mockito.doReturn(virtualMachineMock).when(volumeInfoMock).getAttachedVM(); + long hostId = 12L; + Mockito.doReturn(hostId).when(virtualMachineMock).getHostId(); + + Mockito.doReturn(null).when(defaultEndPointSelectorSpy).getEndPointFromHostId(hostId); + + defaultEndPointSelectorSpy.getEndPointForBitmapRemoval(snapshotInfoMock, false); + + Mockito.verify(defaultEndPointSelectorSpy, Mockito.times(1)).getEndPointFromHostId(hostId); + } + + @Test + public void getEndPointForBitmapRemovalTestVolumeIsAttachedLastHostIdIsSet() { + Mockito.doReturn(true).when(volumeInfoMock).isAttachedVM(); + Mockito.doReturn(virtualMachineMock).when(volumeInfoMock).getAttachedVM(); + + Mockito.doReturn(null).when(virtualMachineMock).getHostId(); + long lastHostId = 13L; + Mockito.doReturn(lastHostId).when(virtualMachineMock).getLastHostId(); + + Mockito.doReturn(null).when(defaultEndPointSelectorSpy).getEndPointFromHostId(lastHostId); + + defaultEndPointSelectorSpy.getEndPointForBitmapRemoval(snapshotInfoMock, false); + + Mockito.verify(defaultEndPointSelectorSpy, Mockito.times(1)).getEndPointFromHostId(lastHostId); + } + + @Test + public void getEndPointForBitmapRemovalTestVolumeIsAttachedNoHostIsSet() { + Mockito.doReturn(true).when(volumeInfoMock).isAttachedVM(); + Mockito.doReturn(virtualMachineMock).when(volumeInfoMock).getAttachedVM(); + + Mockito.doReturn(null).when(virtualMachineMock).getHostId(); + Mockito.doReturn(null).when(virtualMachineMock).getLastHostId(); + + Mockito.doReturn(null).when(defaultEndPointSelectorSpy).select(volumeInfoMock, false); + + defaultEndPointSelectorSpy.getEndPointForBitmapRemoval(snapshotInfoMock, false); + + Mockito.verify(defaultEndPointSelectorSpy, Mockito.times(1)).select(volumeInfoMock, false); + } + + @Test + public void getEndPointForSnapshotOperationsInKvmTestVolumeIsNotAttachedToVMAndSnapshotOnPrimary() { + Mockito.doReturn(null).when(volumeInfoMock).getAttachedVM(); + Mockito.doReturn(datastoreMock).when(snapshotInfoMock).getDataStore(); + Mockito.doReturn(DataStoreRole.Primary).when(datastoreMock).getRole(); + Mockito.doReturn(null).when(defaultEndPointSelectorSpy).select(snapshotInfoMock, false); + + defaultEndPointSelectorSpy.getEndPointForSnapshotOperationsInKvm(snapshotInfoMock, false); + + Mockito.verify(defaultEndPointSelectorSpy, Mockito.times(1)).select(snapshotInfoMock, false); + } + + @Test + public void getEndPointForSnapshotOperationsInKvmTestVolumeIsNotAttachedToVMAndSnapshotOnSecondary() { + Mockito.doReturn(null).when(volumeInfoMock).getAttachedVM(); + Mockito.doReturn(datastoreMock).when(snapshotInfoMock).getDataStore(); + Mockito.doReturn(DataStoreRole.Image).when(datastoreMock).getRole(); + long zoneId = 1L; + Mockito.doReturn(zoneId).when(snapshotInfoMock).getDataCenterId(); + + Mockito.doReturn(null).when(defaultEndPointSelectorSpy).selectRandom(zoneId, Hypervisor.HypervisorType.KVM); + + defaultEndPointSelectorSpy.getEndPointForSnapshotOperationsInKvm(snapshotInfoMock, false); + + Mockito.verify(defaultEndPointSelectorSpy, Mockito.times(1)).selectRandom(zoneId, Hypervisor.HypervisorType.KVM); + } + + @Test + public void getEndPointForSnapshotOperationsInKvmTestVolumeAttachedToRunningVm() { + Mockito.doReturn(virtualMachineMock).when(volumeInfoMock).getAttachedVM(); + Mockito.doReturn(datastoreMock).when(snapshotInfoMock).getDataStore(); + Mockito.doReturn(VirtualMachine.State.Running).when(virtualMachineMock).getState(); + long hostId = 12L; + Mockito.doReturn(hostId).when(virtualMachineMock).getHostId(); + + Mockito.doReturn(null).when(defaultEndPointSelectorSpy).getEndPointFromHostId(hostId); + + defaultEndPointSelectorSpy.getEndPointForSnapshotOperationsInKvm(snapshotInfoMock, false); + + Mockito.verify(defaultEndPointSelectorSpy, Mockito.times(1)).getEndPointFromHostId(hostId); + } + + @Test + public void getEndPointForSnapshotOperationsInKvmTestVolumeAttachedToStoppedVmAndLastHostIdIsSet() { + Mockito.doReturn(virtualMachineMock).when(volumeInfoMock).getAttachedVM(); + Mockito.doReturn(datastoreMock).when(snapshotInfoMock).getDataStore(); + Mockito.doReturn(VirtualMachine.State.Stopped).when(virtualMachineMock).getState(); + long hostId = 13L; + Mockito.doReturn(hostId).when(virtualMachineMock).getLastHostId(); + + Mockito.doReturn(null).when(defaultEndPointSelectorSpy).getEndPointFromHostId(hostId); + + defaultEndPointSelectorSpy.getEndPointForSnapshotOperationsInKvm(snapshotInfoMock, false); + + Mockito.verify(defaultEndPointSelectorSpy, Mockito.times(1)).getEndPointFromHostId(hostId); + } + + @Test + public void getEndPointForSnapshotOperationsInKvmTestVolumeAttachedToStoppedVmAndLastHostIdIsNotSetAndSnapshotIsOnSecondary() { + Mockito.doReturn(virtualMachineMock).when(volumeInfoMock).getAttachedVM(); + Mockito.doReturn(datastoreMock).when(snapshotInfoMock).getDataStore(); + Mockito.doReturn(DataStoreRole.Image).when(datastoreMock).getRole(); + Mockito.doReturn(VirtualMachine.State.Stopped).when(virtualMachineMock).getState(); + Mockito.doReturn(null).when(virtualMachineMock).getLastHostId(); + long zoneId = 1L; + Mockito.doReturn(zoneId).when(snapshotInfoMock).getDataCenterId(); + Mockito.doReturn(null).when(defaultEndPointSelectorSpy).selectRandom(zoneId, Hypervisor.HypervisorType.KVM); + + defaultEndPointSelectorSpy.getEndPointForSnapshotOperationsInKvm(snapshotInfoMock, false); + + Mockito.verify(defaultEndPointSelectorSpy, Mockito.times(1)).selectRandom(zoneId, Hypervisor.HypervisorType.KVM); + } + + + @Test + public void getEndPointForSnapshotOperationsInKvmTestVolumeAttachedToStoppedVmAndLastHostIdIsNotSetAndSnapshotIsOnPrimary() { + Mockito.doReturn(virtualMachineMock).when(volumeInfoMock).getAttachedVM(); + Mockito.doReturn(datastoreMock).when(snapshotInfoMock).getDataStore(); + Mockito.doReturn(DataStoreRole.Primary).when(datastoreMock).getRole(); + Mockito.doReturn(VirtualMachine.State.Stopped).when(virtualMachineMock).getState(); + Mockito.doReturn(null).when(virtualMachineMock).getLastHostId(); + Mockito.doReturn(null).when(defaultEndPointSelectorSpy).select(snapshotInfoMock, false); + + defaultEndPointSelectorSpy.getEndPointForSnapshotOperationsInKvm(snapshotInfoMock, false); + + Mockito.verify(defaultEndPointSelectorSpy, Mockito.times(1)).select(snapshotInfoMock, false); + } + +} diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index 4a9f34c9f565..31aaf7dc856a 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -24,9 +24,11 @@ import com.cloud.dc.VsphereStoragePolicyVO; import com.cloud.dc.dao.VsphereStoragePolicyDao; import com.cloud.storage.StorageManager; +import com.cloud.utils.Pair; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.secret.dao.PassphraseDao; import org.apache.cloudstack.secret.PassphraseVO; import com.cloud.service.dao.ServiceOfferingDetailsDao; @@ -82,6 +84,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; @@ -114,6 +117,9 @@ public class VolumeObject implements VolumeInfo { VsphereStoragePolicyDao vsphereStoragePolicyDao; @Inject PassphraseDao passphraseDao; + @Inject + VolumeOrchestrationService + orchestrationService; private Object payload; private MigrationOptions migrationOptions; @@ -121,6 +127,9 @@ public class VolumeObject implements VolumeInfo { private String vSphereStoragePolicyId; private boolean followRedirects; + private List checkpointPaths; + private Set checkpointImageStoreUrls; + private final List volumeStatesThatShouldNotTransitWhenDataStoreRoleIsImage = Arrays.asList(Volume.State.Migrating, Volume.State.Uploaded, Volume.State.Copying, Volume.State.Expunged); @@ -136,6 +145,9 @@ public VolumeObject() { protected void configure(DataStore dataStore, VolumeVO volumeVO) { this.volumeVO = volumeVO; this.dataStore = dataStore; + Pair, Set> volumeCheckPointPathsAndImageStoreUrls = orchestrationService.getVolumeCheckpointPathsAndImageStoreUrls(volumeVO.getId(), getHypervisorType()); + this.checkpointPaths = volumeCheckPointPathsAndImageStoreUrls.first(); + this.checkpointImageStoreUrls = volumeCheckPointPathsAndImageStoreUrls.second(); } public static VolumeObject getVolumeObject(DataStore dataStore, VolumeVO volumeVO) { @@ -945,6 +957,16 @@ public boolean isFollowRedirects() { return followRedirects; } + @Override + public List getCheckpointPaths() { + return checkpointPaths; + } + + @Override + public Set getCheckpointImageStoreUrls() { + return checkpointImageStoreUrls; + } + @Override public String toString() { return String.format("VolumeObject %s", diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 26bef607c9b2..f296a32e2c0c 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -497,6 +497,9 @@ public Void deleteVolumeCallback(AsyncCallbackDispatcher mapCapabilities = primaryDataStore.getDriver().getCapabilities(); @@ -507,10 +510,10 @@ public Void deleteVolumeCallback(AsyncCallbackDispatcher snapshotDataStoreVOList = _snapshotStoreDao.findBySnapshotId(snapshotDataStoreVO.getSnapshotId()); + for (SnapshotDataStoreVO snapshotStore : snapshotDataStoreVOList) { + if (DataStoreRole.Image.equals(snapshotStore.getRole())) { + _snapshotStoreDao.remove(snapshotDataStoreVO.getId()); + return; + } + } + + snapshotApiService.deleteSnapshot(snapshotDataStoreVO.getSnapshotId(), null); + } + @Override public boolean cloneVolume(long volumeId, long baseVolId) { // TODO Auto-generated method stub diff --git a/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeObjectTest.java b/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeObjectTest.java index 58f47a5db64e..54aaecb0c097 100644 --- a/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeObjectTest.java +++ b/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeObjectTest.java @@ -21,12 +21,14 @@ import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.exception.ConcurrentOperationException; +import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.Storage; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; import java.util.Arrays; @@ -38,6 +40,7 @@ import java.util.Set; import java.util.function.Function; import junit.framework.TestCase; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.storage.command.CopyCmdAnswer; @@ -82,6 +85,9 @@ public class VolumeObjectTest extends TestCase{ @Mock ObjectInDataStoreManager objectInDataStoreManagerMock; + @Mock + VolumeOrchestrationService orchestrationServiceMock; + Set> diskOfferingVoMethodsWithLongReturn = new HashSet<>(); List objectInDataStoreStateMachineEvents = Arrays.asList(ObjectInDataStoreStateMachine.Event.values()); @@ -90,6 +96,9 @@ public class VolumeObjectTest extends TestCase{ @Before public void setup(){ + volumeObjectSpy.orchestrationService = orchestrationServiceMock; + Mockito.doReturn(new Pair<>(List.of(), Set.of())).when(orchestrationServiceMock).getVolumeCheckpointPathsAndImageStoreUrls(Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(Hypervisor.HypervisorType.KVM).when(volumeObjectSpy).getHypervisorType(); volumeObjectSpy.configure(dataStoreMock, volumeVoMock); volumeObjectSpy.volumeStoreDao = volumeDataStoreDaoMock; volumeObjectSpy.volumeDao = volumeDaoMock; @@ -362,7 +371,8 @@ public void validateSetVolumeFormatNullFormatAndSetFormatFalseDoNothing(){ volumeObjectTo.setFormat(null); volumeObjectSpy.setVolumeFormat(volumeObjectTo, false, volumeVoMock); - Mockito.verifyNoInteractions(volumeVoMock); + + Mockito.verify(volumeVoMock, Mockito.never()).setFormat(Mockito.any()); } @Test @@ -371,7 +381,8 @@ public void validateSetVolumeFormatNullFormatAndSetFormatTrueDoNothing(){ volumeObjectTo.setFormat(null); volumeObjectSpy.setVolumeFormat(volumeObjectTo, true, volumeVoMock); - Mockito.verifyNoInteractions(volumeVoMock); + + Mockito.verify(volumeVoMock, Mockito.never()).setFormat(Mockito.any()); } @Test @@ -385,7 +396,7 @@ public void validateSetVolumeFormatValidFormatAndSetFormatFalseDoNothing(){ volumeObjectSpy.setVolumeFormat(volumeObjectTo, false, volumeVoMock); }); - Mockito.verifyNoInteractions(volumeVoMock); + Mockito.verify(volumeVoMock, Mockito.never()).setFormat(Mockito.any()); } @Test diff --git a/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeServiceTest.java b/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeServiceTest.java index aa5ac3b9a76e..57d10dd800a1 100644 --- a/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeServiceTest.java +++ b/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeServiceTest.java @@ -178,7 +178,8 @@ public void validateExpungeSourceVolumeAfterMigrationVolumeApiResultFailedRetryE @Test public void validateCopyPoliciesBetweenVolumesAndDestroySourceVolumeAfterMigrationReturnTrueOrFalse() throws ExecutionException, InterruptedException{ - VolumeObject volumeObject = new VolumeObject(); + VolumeObject volumeObject = Mockito.mock(VolumeObject.class); + Mockito.doReturn(new VolumeVO() {}).when(volumeObject).getVolume(); volumeObject.configure(null, new VolumeVO() {}); Mockito.doNothing().when(snapshotManagerMock).copySnapshotPoliciesBetweenVolumes(Mockito.any(), Mockito.any()); @@ -196,8 +197,8 @@ public void validateCopyPoliciesBetweenVolumesAndDestroySourceVolumeAfterMigrati @Test (expected = Exception.class) public void validateCopyPoliciesBetweenVolumesAndDestroySourceVolumeAfterMigrationThrowAnyOtherException() throws ExecutionException, InterruptedException{ - VolumeObject volumeObject = new VolumeObject(); - volumeObject.configure(null, new VolumeVO() {}); + VolumeObject volumeObject = Mockito.mock(VolumeObject.class); + Mockito.doReturn(new VolumeVO() {}).when(volumeObject).getVolume(); volumeServiceImplSpy.copyPoliciesBetweenVolumesAndDestroySourceVolumeAfterMigration(ObjectInDataStoreStateMachine.Event.DestroyRequested, null, volumeObject, volumeObject, true); @@ -205,8 +206,8 @@ public void validateCopyPoliciesBetweenVolumesAndDestroySourceVolumeAfterMigrati @Test public void validateDestroySourceVolumeAfterMigrationReturnTrue() throws ExecutionException, InterruptedException{ - VolumeObject volumeObject = new VolumeObject(); - volumeObject.configure(null, new VolumeVO() {}); + VolumeObject volumeObject = Mockito.mock(VolumeObject.class); + Mockito.doReturn(new VolumeVO() {}).when(volumeObject).getVolume(); Mockito.doReturn(true).when(volumeDaoMock).updateUuid(Mockito.anyLong(), Mockito.anyLong()); Mockito.doNothing().when(volumeServiceImplSpy).destroyVolume(Mockito.anyLong()); @@ -221,10 +222,11 @@ public void validateDestroySourceVolumeAfterMigrationReturnTrue() throws Executi @Test public void validateDestroySourceVolumeAfterMigrationExpungeSourceVolumeAfterMigrationThrowExceptionReturnFalse() throws ExecutionException, InterruptedException{ - VolumeObject volumeObject = new VolumeObject(); VolumeVO vo = new VolumeVO() {}; vo.setPoolType(Storage.StoragePoolType.Filesystem); - volumeObject.configure(null, vo); + + VolumeObject volumeObject = Mockito.mock(VolumeObject.class); + Mockito.doReturn(vo).when(volumeObject).getVolume(); vo.setPoolId(1L); List exceptions = new ArrayList<>(Arrays.asList(new InterruptedException(), new ExecutionException() {})); diff --git a/plugins/hypervisors/kvm/pom.xml b/plugins/hypervisors/kvm/pom.xml index 08ad50ce2cb1..b2ed85c9bb3c 100644 --- a/plugins/hypervisors/kvm/pom.xml +++ b/plugins/hypervisors/kvm/pom.xml @@ -83,6 +83,10 @@ ${project.version} compile + + org.json + json + diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 98c718576735..a632fd5adfd2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -51,7 +51,18 @@ import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; import org.apache.cloudstack.command.CommandInfo; @@ -78,6 +89,7 @@ import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.commons.collections.MapUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.BooleanUtils; @@ -262,8 +274,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv /** * Machine type. */ - private static final String PC = isHostS390x() ? S390X_VIRTIO_DEVICE : "pc"; - private static final String VIRT = isHostS390x() ? S390X_VIRTIO_DEVICE : "virt"; + public static final String PC = isHostS390x() ? S390X_VIRTIO_DEVICE : "pc"; + public static final String VIRT = isHostS390x() ? S390X_VIRTIO_DEVICE : "virt"; /** * Possible devices to add to VM. @@ -352,6 +364,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String COMMANDS_LOG_PATH = "/usr/share/cloudstack-agent/tmp/commands"; + public static final String CHECKPOINT_CREATE_COMMAND = "virsh checkpoint-create --domain %s --xmlfile %s --redefine"; + + public static final String CHECKPOINT_DELETE_COMMAND = "virsh checkpoint-delete --domain %s --checkpointname %s --metadata"; + private String modifyVlanPath; private String versionStringPath; private String patchScriptPath; @@ -414,6 +430,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private final static long HYPERVISOR_QEMU_VERSION_SUPPORTS_IO_URING = 5000000; private final static long HYPERVISOR_QEMU_VERSION_IDE_DISCARD_FIXED = 7000000; + private static final int MINIMUM_LIBVIRT_VERSION_FOR_INCREMENTAL_SNAPSHOT = 7006000; + + private static final int MINIMUM_QEMU_VERSION_FOR_INCREMENTAL_SNAPSHOT = 6001000; + protected HypervisorType hypervisorType; protected String hypervisorURI; protected long hypervisorLibvirtVersion; @@ -539,11 +559,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String CGROUP_V2 = "cgroup2fs"; - protected long getHypervisorLibvirtVersion() { + public long getHypervisorLibvirtVersion() { return hypervisorLibvirtVersion; } - protected long getHypervisorQemuVersion() { + public long getHypervisorQemuVersion() { return hypervisorQemuVersion; } @@ -1930,6 +1950,10 @@ boolean isDirectAttachedNetwork(final String type) { } public String startVM(final Connect conn, final String vmName, final String domainXML) throws LibvirtException, InternalErrorException { + return startVM(conn, vmName, domainXML, 0); + } + + public String startVM(final Connect conn, final String vmName, final String domainXML, int flags) throws LibvirtException, InternalErrorException { try { /* We create a transient domain here. When this method gets @@ -1957,7 +1981,7 @@ public String startVM(final Connect conn, final String vmName, final String doma } } - conn.domainCreateXML(domainXML, 0); + conn.domainCreateXML(domainXML, flags); } catch (final LibvirtException e) { throw e; } @@ -3006,7 +3030,7 @@ private boolean isUefiPropertieNotNull(String propertie) { return uefiProperties.getProperty(propertie) != null; } - private boolean isGuestAarch64() { + public boolean isGuestAarch64() { return AARCH64.equals(guestCpuArch); } @@ -4845,6 +4869,112 @@ protected long getMemoryFreeInKBs(Domain dm) throws LibvirtException { return freeMemory; } + public void removeCheckpointsOnVm(String vmName, String volumeUuid, List checkpointPaths) { + logger.debug("Removing checkpoints with paths [{}] of volume [{}] on VM [{}].", checkpointPaths, volumeUuid, vmName); + String checkpointName; + for (String checkpointPath : checkpointPaths) { + checkpointName = checkpointPath.substring(checkpointPath.lastIndexOf("/") + 1); + Script.runSimpleBashScript(String.format(CHECKPOINT_DELETE_COMMAND, vmName, checkpointName)); + } + logger.debug("Removed all checkpoints of volume [{}] on VM [{}].", volumeUuid, vmName); + } + + public boolean recreateCheckpointsOnVm(List volumes, String vmName, Connect conn) { + logger.debug("Trying to recreate checkpoints on VM [{}] with volumes [{}].", vmName, volumes); + try { + validateLibvirtAndQemuVersionForIncrementalSnapshots(); + } catch (CloudRuntimeException e) { + logger.warn("Will not recreate the checkpoints on VM as {}", e.getMessage(), e); + return false; + } + List diskDefs = getDisks(conn, vmName); + Map mapDiskToDiskDef = mapVolumeToDiskDef(volumes, diskDefs); + + for (VolumeObjectTO volume : volumes) { + if (CollectionUtils.isEmpty(volume.getCheckpointPaths())) { + continue; + } + Set storagePoolSet = connectToAllVolumeSnapshotSecondaryStorages(volume); + recreateCheckpointsOfDisk(vmName, volume, mapDiskToDiskDef); + disconnectAllVolumeSnapshotSecondaryStorages(storagePoolSet); + } + logger.debug("Successfully recreated all checkpoints on VM [{}].", vmName); + return true; + } + + public Set connectToAllVolumeSnapshotSecondaryStorages(VolumeObjectTO volumeObjectTO) { + return volumeObjectTO.getCheckpointImageStoreUrls().stream().map(uri -> getStoragePoolMgr().getStoragePoolByURI(uri)).collect(Collectors.toSet()); + } + + public void disconnectAllVolumeSnapshotSecondaryStorages(Set kvmStoragePools) { + kvmStoragePools.forEach(storage -> getStoragePoolMgr().deleteStoragePool(storage.getType(), storage.getUuid())); + } + + + protected void recreateCheckpointsOfDisk(String vmName, VolumeObjectTO volume, Map mapDiskToDiskDef) { + for (String path : volume.getCheckpointPaths()) { + DiskDef diskDef = mapDiskToDiskDef.get(volume); + if (diskDef != null) { + try { + updateDiskLabelOnXml(path, diskDef.getDiskLabel()); + } catch (ParserConfigurationException | IOException | SAXException | TransformerException | XPathExpressionException e) { + logger.error("Exception while parsing checkpoint XML with path [{}].", path, e); + throw new CloudRuntimeException(e); + } + } else { + logger.debug("Could not map [{}] to any disk definition. Will try to recreate snapshot without updating disk label.", volume); + } + + logger.trace("Recreating checkpoint with path [{}] on VM [{}].", path, vmName); + Script.runSimpleBashScript(String.format(CHECKPOINT_CREATE_COMMAND, vmName, path)); + } + } + + /** + * Changes the value of the disk label of the checkpoint XML found in {@code path} to {@code label}. This method assumes that the checkpoint only contains one disk. + * @param path the path to the checkpoint XML to be updated + * @param label the new label to be used for the disk + * */ + private void updateDiskLabelOnXml(String path, String label) throws ParserConfigurationException, IOException, SAXException, XPathExpressionException, TransformerException { + logger.trace("Updating checkpoint with path [{}] to use disk label [{}].", path, label); + + DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory(); + DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + Document doc = docBuilder.parse(new File(path)); + + XPath xPath = XPathFactory.newInstance().newXPath(); + Node diskNode = (Node) xPath.compile("/domaincheckpoint/disks/disk").evaluate(doc, XPathConstants.NODE); + diskNode.getAttributes().getNamedItem("name").setNodeValue(label); + + Transformer tf = TransformerFactory.newInstance().newTransformer(); + tf.setOutputProperty(OutputKeys.INDENT, "yes"); + tf.setOutputProperty(OutputKeys.METHOD, "xml"); + tf.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "4"); + + DOMSource domSource = new DOMSource(doc); + StreamResult sr = new StreamResult(new File(path)); + tf.transform(domSource, sr); + } + + protected Map mapVolumeToDiskDef(List volumeTos, List diskDefs) { + HashMap diskToDiskDefHashMap = new HashMap<>(); + for (VolumeObjectTO volumeTo : volumeTos) { + for (DiskDef diskDef : diskDefs) { + if (StringUtils.contains(diskDef.getDiskPath(), volumeTo.getPath())) { + diskToDiskDefHashMap.put(volumeTo, diskDef); + } + } + } + return diskToDiskDefHashMap; + } + + public void validateLibvirtAndQemuVersionForIncrementalSnapshots() { + if (getHypervisorLibvirtVersion() < MINIMUM_LIBVIRT_VERSION_FOR_INCREMENTAL_SNAPSHOT || getHypervisorQemuVersion() < MINIMUM_QEMU_VERSION_FOR_INCREMENTAL_SNAPSHOT) { + throw new CloudRuntimeException(String.format("Hypervisor version is insufficient, should have at least libvirt [%s] and qemu [%s] but we have [%s] and [%s].", + MINIMUM_LIBVIRT_VERSION_FOR_INCREMENTAL_SNAPSHOT, MINIMUM_QEMU_VERSION_FOR_INCREMENTAL_SNAPSHOT, getHypervisorLibvirtVersion(), getHypervisorQemuVersion())); + } + } + private boolean canBridgeFirewall(final String prvNic) { final Script cmd = new Script(securityGroupPath, timeout, LOGGER); cmd.add("can_bridge_firewall"); @@ -5810,4 +5940,12 @@ public VolumeOnStorageTO getVolumeOnStorage(PrimaryDataStoreTO primaryStore, Str } } } + + public String getHypervisorPath() { + return hypervisorPath; + } + public String getGuestCpuArch() { + return guestCpuArch; + } + } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtXMLParser.java index f5de9b754120..5acfb60ee47e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtXMLParser.java @@ -16,16 +16,23 @@ // under the License. package com.cloud.hypervisor.kvm.resource; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.StringReader; import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.w3c.dom.Document; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import org.xml.sax.helpers.DefaultHandler; @@ -65,6 +72,20 @@ public boolean parseDomainXML(String domXML) { return false; } + public static String getXml(Document doc) throws TransformerException { + TransformerFactory transformerFactory = ParserUtils.getSaferTransformerFactory(); + Transformer transformer = transformerFactory.newTransformer(); + + DOMSource source = new DOMSource(doc); + + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + StreamResult result = new StreamResult(byteArrayOutputStream); + + transformer.transform(source, result); + + return byteArrayOutputStream.toString(); + } + @Override public void characters(char[] ch, int start, int length) throws SAXException { } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertSnapshotCommandWrapper.java new file mode 100644 index 000000000000..6d7ac838a727 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertSnapshotCommandWrapper.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.ConvertSnapshotAnswer; +import com.cloud.agent.api.ConvertSnapshotCommand; +import com.cloud.agent.api.to.DataStoreTO; + +import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.properties.AgentProperties; +import com.cloud.agent.properties.AgentPropertiesFileHandler; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.resource.CommandWrapper; + +import com.cloud.resource.ResourceWrapper; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.libvirt.LibvirtException; + +import java.io.File; +import java.util.Set; + +@ResourceWrapper(handles = ConvertSnapshotCommand.class) +public class LibvirtConvertSnapshotCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(ConvertSnapshotCommand command, LibvirtComputingResource serverResource) { + SnapshotObjectTO snapshotObjectTO = command.getSnapshotObjectTO(); + DataStoreTO imageStore = snapshotObjectTO.getDataStore(); + + logger.debug(String.format("Converting snapshot [%s] in image store [%s].", snapshotObjectTO.getId(), imageStore.getUuid())); + + if (!(imageStore instanceof NfsTO)) { + return new Answer(command, false, "Image Store must be NFS."); + } + NfsTO nfsImageStore = (NfsTO)imageStore; + + String secondaryStoragePoolUrl = nfsImageStore.getUrl(); + + Set storagePoolSet = null; + KVMStoragePool secondaryStorage = null; + try { + secondaryStorage = serverResource.getStoragePoolMgr().getStoragePoolByURI(secondaryStoragePoolUrl); + storagePoolSet = serverResource.connectToAllVolumeSnapshotSecondaryStorages(snapshotObjectTO.getVolume()); + + String snapshotRelativePath = snapshotObjectTO.getPath(); + String snapshotPath = secondaryStorage.getLocalPathFor(snapshotRelativePath); + + String tempSnapshotPath = snapshotPath + ConvertSnapshotCommand.TEMP_SNAPSHOT_NAME; + + logger.debug(String.format("Converting snapshot [%s] to [%s]. The original snapshot is at [%s].", snapshotObjectTO.getId(), tempSnapshotPath, snapshotPath)); + + QemuImg qemuImg = new QemuImg(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.INCREMENTAL_SNAPSHOT_TIMEOUT) * 1000); + + QemuImgFile snapshot = new QemuImgFile(snapshotPath, QemuImg.PhysicalDiskFormat.QCOW2); + QemuImgFile tempSnapshot = new QemuImgFile(tempSnapshotPath, QemuImg.PhysicalDiskFormat.QCOW2); + + qemuImg.convert(snapshot, tempSnapshot); + + SnapshotObjectTO convertedSnapshot = new SnapshotObjectTO(); + convertedSnapshot.setPath(snapshotRelativePath + ConvertSnapshotCommand.TEMP_SNAPSHOT_NAME); + + final File snapFile = new File(tempSnapshotPath); + + if (!snapFile.exists()) { + return new Answer(command, false, "Failed to convert snapshot."); + } + + convertedSnapshot.setPhysicalSize(snapFile.length()); + logger.debug(String.format("Successfully converted snapshot [%s] to [%s].", snapshotObjectTO.getId(), tempSnapshotPath)); + + return new ConvertSnapshotAnswer(convertedSnapshot); + } catch (LibvirtException | QemuImgException ex) { + logger.error(String.format("Failed to convert snapshot [%s] due to %s.", snapshotObjectTO, ex.getMessage()), ex); + return new Answer(command, ex); + } finally { + if (secondaryStorage != null) { + serverResource.getStoragePoolMgr().deleteStoragePool(secondaryStorage.getType(), secondaryStorage.getUuid()); + } + if (storagePoolSet != null) { + serverResource.disconnectAllVolumeSnapshotSecondaryStorages(storagePoolSet); + } + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index 1ff7c3e618ab..0d0dcd96ffa2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -19,7 +19,6 @@ package com.cloud.hypervisor.kvm.resource.wrapper; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URISyntaxException; @@ -39,12 +38,9 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; +import com.cloud.hypervisor.kvm.resource.LibvirtXMLParser; import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.FilenameUtils; @@ -457,7 +453,7 @@ protected String updateVmSharesIfNeeded(MigrateCommand migrateCommand, String xm logger.info(String.format("VM [%s] will have CPU shares altered from [%s] to [%s] as part of migration because the cgroups version differs between hosts.", migrateCommand.getVmName(), currentShares, newVmCpuShares)); sharesNode.setTextContent(String.valueOf(newVmCpuShares)); - return getXml(document); + return LibvirtXMLParser.getXml(document); } /** @@ -527,7 +523,7 @@ protected String replaceDpdkInterfaces(String xmlDesc, Map dpdkP } } - return getXml(doc); + return LibvirtXMLParser.getXml(doc); } /** @@ -702,7 +698,7 @@ protected String replaceStorage(String xmlDesc, Map disks, String vmName) { @@ -795,7 +791,7 @@ protected String replaceCdromIsoPath(String xmlDesc, String vmName, String oldIs newChildSourceNode.setAttribute("file", newIsoVolumePath); diskNode.appendChild(newChildSourceNode); logger.debug(String.format("Replaced ISO path [%s] with [%s] in VM [%s] XML configuration.", oldIsoVolumePath, newIsoVolumePath, vmName)); - return getXml(doc); + return LibvirtXMLParser.getXml(doc); } } } @@ -804,7 +800,7 @@ protected String replaceCdromIsoPath(String xmlDesc, String vmName, String oldIs } } - return getXml(doc); + return LibvirtXMLParser.getXml(doc); } private String getPathFromSourceText(Set paths, String sourceText) { @@ -859,20 +855,6 @@ private String getSourceText(Node diskNode) { return null; } - private String getXml(Document doc) throws TransformerException { - TransformerFactory transformerFactory = ParserUtils.getSaferTransformerFactory(); - Transformer transformer = transformerFactory.newTransformer(); - - DOMSource source = new DOMSource(doc); - - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - StreamResult result = new StreamResult(byteArrayOutputStream); - - transformer.transform(source, result); - - return byteArrayOutputStream.toString(); - } - private String replaceDiskSourceFile(String xmlDesc, String isoPath, String vmName) throws IOException, SAXException, ParserConfigurationException, TransformerException { InputStream in = IOUtils.toInputStream(xmlDesc); @@ -895,7 +877,7 @@ private String replaceDiskSourceFile(String xmlDesc, String isoPath, String vmNa } } } - return getXml(doc); + return LibvirtXMLParser.getXml(doc); } private boolean findDiskNode(Document doc, NodeList devicesChildNodes, String vmName, String isoPath) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRecreateCheckpointsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRecreateCheckpointsCommandWrapper.java new file mode 100644 index 000000000000..6ac933fc0bbf --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRecreateCheckpointsCommandWrapper.java @@ -0,0 +1,48 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.RecreateCheckpointsCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.libvirt.LibvirtException; + +import java.util.List; + +@ResourceWrapper(handles = RecreateCheckpointsCommand.class) +public class LibvirtRecreateCheckpointsCommandWrapper extends CommandWrapper { + @Override + public Answer execute(RecreateCheckpointsCommand command, LibvirtComputingResource serverResource) { + String vmName = command.getVmName(); + List volumes = command.getDisks(); + + boolean result; + try { + result = serverResource.recreateCheckpointsOnVm(volumes, vmName, serverResource.getLibvirtUtilitiesHelper().getConnectionByVmName(vmName)); + } catch (LibvirtException e) { + logger.error(String.format("Failed to recreate checkpoints on VM [%s] due to %s", vmName, e.getMessage()), e); + return new Answer(command, e); + } + return new Answer(command, result, null); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRemoveBitmapCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRemoveBitmapCommandWrapper.java new file mode 100644 index 000000000000..0170d2fc2e4f --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRemoveBitmapCommandWrapper.java @@ -0,0 +1,130 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.RemoveBitmapCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.json.JSONArray; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; + +import org.json.JSONObject; + +import java.io.File; + + +@ResourceWrapper(handles = RemoveBitmapCommand.class) +public class LibvirtRemoveBitmapCommandWrapper extends CommandWrapper { + + private static final String QEMU_MONITOR_REMOVE_BITMAP_COMMAND = "{\"execute\": \"block-dirty-bitmap-remove\", \"arguments\":{\"node\":\"%s\",\"name\":\"%s\" }}"; + private static final String QEMU_MONITOR_QUERY_BLOCK_COMMAND = "{\"execute\": \"query-block\"}"; + + + @Override + public Answer execute(RemoveBitmapCommand command, LibvirtComputingResource resource) { + SnapshotObjectTO snapshotObjectTO = command.getSnapshotObjectTO(); + + try { + if (command.isVmRunning()) { + return removeBitmapForRunningVM(snapshotObjectTO, resource, command); + } + return removeBitmapForStoppedVM(snapshotObjectTO, resource, command); + } catch (LibvirtException | QemuImgException exception) { + logger.error("Exception while removing bitmap for volume [{}]. Caught exception is [{}].", snapshotObjectTO.getVolume().getName(), exception); + return new Answer(command, exception); + } + } + + protected Answer removeBitmapForRunningVM(SnapshotObjectTO snapshotObjectTO, LibvirtComputingResource resource, RemoveBitmapCommand cmd) throws LibvirtException { + Domain vm = resource.getDomain(resource.getLibvirtUtilitiesHelper().getConnection(), snapshotObjectTO.getVmName()); + String nodeName = getNodeName(vm, snapshotObjectTO); + logger.debug("Got [{}] as node-name for volume [{}] of VM [{}].", nodeName, snapshotObjectTO.getVolume().getName(), snapshotObjectTO.getVmName()); + if (nodeName == null) { + return new Answer(cmd, false, "Failed to get node-name to remove the bitmap."); + } + + String bitmapName = getBitmapName(snapshotObjectTO); + logger.debug("Removing bitmap [{}].", bitmapName); + vm.qemuMonitorCommand(String.format(QEMU_MONITOR_REMOVE_BITMAP_COMMAND, nodeName, bitmapName), 0); + return new Answer(cmd); + } + + protected Answer removeBitmapForStoppedVM(SnapshotObjectTO snapshotObjectTO, LibvirtComputingResource resource, Command cmd) throws LibvirtException, QemuImgException { + VolumeObjectTO volumeTo = snapshotObjectTO.getVolume(); + PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) volumeTo.getDataStore(); + + KVMStoragePool primaryPool = resource.getStoragePoolMgr().getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid()); + + QemuImg qemuImg = new QemuImg(cmd.getWait()); + QemuImgFile volume = new QemuImgFile(primaryPool.getLocalPath() + File.separator + volumeTo.getPath(), QemuImg.PhysicalDiskFormat.QCOW2); + + String bitmap = getBitmapName(snapshotObjectTO); + + logger.debug("Removing bitmap [{}] for volume [{}].", bitmap, volumeTo.getName()); + + try { + qemuImg.bitmap(QemuImg.BitmapOperation.Remove, volume, bitmap); + } catch (QemuImgException ex) { + if (!(ex.getMessage().contains("Dirty bitmap") || ex.getMessage().contains("not found"))) { + throw ex; + } + logger.warn("Could not delete dirty bitmap [{}] as it was not found. This will happen if the volume was migrated. If it is not the case, this should be reported.", bitmap); + } + return new Answer(cmd); + } + + + protected String getBitmapName(SnapshotObjectTO snapshotObjectTO) { + String[] splitPath = snapshotObjectTO.getPath().split(File.separator); + return splitPath[splitPath.length - 1]; + } + + protected String getNodeName(Domain vm, SnapshotObjectTO snapshotObjectTO) throws LibvirtException { + logger.debug("Getting nodeName to remove bitmap for volume [{}] of VM [{}].", snapshotObjectTO.getVolume().getName(), snapshotObjectTO.getVmName()); + String vmBlockInfo = vm.qemuMonitorCommand(QEMU_MONITOR_QUERY_BLOCK_COMMAND, 0); + logger.debug("Parsing [{}]", vmBlockInfo); + JSONObject jsonObj = new JSONObject(vmBlockInfo); + JSONArray returnArray = jsonObj.getJSONArray("return"); + + for (int i = 0; i < returnArray.length(); i++) { + JSONObject blockInfo = returnArray.getJSONObject(i); + if (!blockInfo.has("inserted")) { + continue; + } + JSONObject inserted = blockInfo.getJSONObject("inserted"); + String volumePath = inserted.getString("file"); + if (!volumePath.contains(snapshotObjectTO.getVolume().getPath())) { + continue; + } + return inserted.getString("node-name"); + } + return null; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertSnapshotCommandWrapper.java index d2f1ef827a15..5d76d140f229 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertSnapshotCommandWrapper.java @@ -20,14 +20,14 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import java.io.File; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.util.Set; import java.util.HashSet; import java.util.Arrays; +import com.cloud.agent.properties.AgentProperties; +import com.cloud.agent.properties.AgentPropertiesFileHandler; import org.apache.cloudstack.storage.command.RevertSnapshotCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.SnapshotObjectTO; @@ -53,6 +53,10 @@ import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.libvirt.LibvirtException; @ResourceWrapper(handles = RevertSnapshotCommand.class) public class LibvirtRevertSnapshotCommandWrapper extends CommandWrapper { @@ -80,7 +84,7 @@ public Answer execute(final RevertSnapshotCommand command, final LibvirtComputin String volumePath = volume.getPath(); String snapshotRelPath = snapshot.getPath(); - + KVMStoragePool secondaryStoragePool = null; try { KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); @@ -109,7 +113,6 @@ public Answer execute(final RevertSnapshotCommand command, final LibvirtComputin rbd.close(image); rados.ioCtxDestroy(io); } else { - KVMStoragePool secondaryStoragePool = null; if (snapshotImageStore != null && DataStoreRole.Primary != snapshotImageStore.getRole()) { secondaryStoragePool = storagePoolMgr.getStoragePoolByURI(snapshotImageStore.getUrl()); } @@ -125,7 +128,7 @@ public Answer execute(final RevertSnapshotCommand command, final LibvirtComputin return new Answer(command, false, result); } } else { - revertVolumeToSnapshot(snapshotOnPrimaryStorage, snapshot, snapshotImageStore, primaryPool, secondaryStoragePool); + revertVolumeToSnapshot(secondaryStoragePool, snapshotOnPrimaryStorage, snapshot, primaryPool, libvirtComputingResource); } } @@ -138,7 +141,12 @@ public Answer execute(final RevertSnapshotCommand command, final LibvirtComputin } catch (RbdException e) { logger.error("Failed to connect to revert snapshot due to RBD exception: ", e); return new Answer(command, false, e.toString()); + } finally { + if (secondaryStoragePool != null) { + libvirtComputingResource.getStoragePoolMgr().deleteStoragePool(secondaryStoragePool.getType(), secondaryStoragePool.getUuid()); + } } + } /** @@ -152,8 +160,8 @@ protected String getFullPathAccordingToStorage(KVMStoragePool kvmStoragePool, St /** * Reverts the volume to the snapshot. */ - protected void revertVolumeToSnapshot(SnapshotObjectTO snapshotOnPrimaryStorage, SnapshotObjectTO snapshotOnSecondaryStorage, DataStoreTO dataStoreTo, - KVMStoragePool kvmStoragePoolPrimary, KVMStoragePool kvmStoragePoolSecondary) { + protected void revertVolumeToSnapshot(KVMStoragePool kvmStoragePoolSecondary, SnapshotObjectTO snapshotOnPrimaryStorage, SnapshotObjectTO snapshotOnSecondaryStorage, + KVMStoragePool kvmStoragePoolPrimary, LibvirtComputingResource resource) { VolumeObjectTO volumeObjectTo = snapshotOnSecondaryStorage.getVolume(); String volumePath = getFullPathAccordingToStorage(kvmStoragePoolPrimary, volumeObjectTo.getPath()); @@ -161,13 +169,22 @@ protected void revertVolumeToSnapshot(SnapshotObjectTO snapshotOnPrimaryStorage, String snapshotPath = resultGetSnapshot.first(); SnapshotObjectTO snapshotToPrint = resultGetSnapshot.second(); + Set storagePoolSet = null; + if (kvmStoragePoolSecondary != null) { + storagePoolSet = resource.connectToAllVolumeSnapshotSecondaryStorages(volumeObjectTo); + } + logger.debug(String.format("Reverting volume [%s] to snapshot [%s].", volumeObjectTo, snapshotToPrint)); try { replaceVolumeWithSnapshot(volumePath, snapshotPath); logger.debug(String.format("Successfully reverted volume [%s] to snapshot [%s].", volumeObjectTo, snapshotToPrint)); - } catch (IOException ex) { + } catch (LibvirtException | QemuImgException ex) { throw new CloudRuntimeException(String.format("Unable to revert volume [%s] to snapshot [%s] due to [%s].", volumeObjectTo, snapshotToPrint, ex.getMessage()), ex); + } finally { + if (storagePoolSet != null) { + resource.disconnectAllVolumeSnapshotSecondaryStorages(storagePoolSet); + } } } @@ -208,9 +225,15 @@ protected Pair getSnapshot(SnapshotObjectTO snapshotOn /** * Replaces the current volume with the snapshot. - * @throws IOException If can't replace the current volume with the snapshot. + * @throws LibvirtException If can't replace the current volume with the snapshot. + * @throws QemuImgException If can't replace the current volume with the snapshot. */ - protected void replaceVolumeWithSnapshot(String volumePath, String snapshotPath) throws IOException { - Files.copy(Paths.get(snapshotPath), Paths.get(volumePath), StandardCopyOption.REPLACE_EXISTING); + protected void replaceVolumeWithSnapshot(String volumePath, String snapshotPath) throws LibvirtException, QemuImgException { + logger.debug(String.format("Replacing volume at [%s] with snapshot that is at [%s].", volumePath, snapshotPath)); + QemuImg qemuImg = new QemuImg(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REVERT_SNAPSHOT_TIMEOUT) * 1000); + QemuImgFile volumeImg = new QemuImgFile(volumePath, QemuImg.PhysicalDiskFormat.QCOW2); + QemuImgFile snapshotImg = new QemuImgFile(snapshotPath, QemuImg.PhysicalDiskFormat.QCOW2); + + qemuImg.convert(snapshotImg, volumeImg); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java index a174c9a6f14f..09e4ec380278 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java @@ -21,9 +21,13 @@ import java.io.File; import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import com.cloud.agent.resource.virtualnetwork.VRScripts; import com.cloud.utils.FileUtil; +import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.libvirt.Connect; import org.libvirt.DomainInfo.DomainState; import org.libvirt.LibvirtException; @@ -89,6 +93,12 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource libvirtComputingResource.applyDefaultNetworkRules(conn, vmSpec, false); + if (vmSpec.getType() == VirtualMachine.Type.User) { + List volumes = Arrays.stream(vmSpec.getDisks()).filter(diskTO -> diskTO.getData() instanceof VolumeObjectTO). + map(diskTO -> (VolumeObjectTO) diskTO.getData()).collect(Collectors.toList()); + libvirtComputingResource.recreateCheckpointsOnVm(volumes, vmName, conn); + } + // pass cmdline info to system vms if (vmSpec.getType() != VirtualMachine.Type.User || (vmSpec.getBootArgs() != null && (vmSpec.getBootArgs().contains(UserVmManager.CKS_NODE) || vmSpec.getBootArgs().contains(UserVmManager.SHAREDFSVM)))) { // try to patch and SSH into the systemvm for up to 5 minutes diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java index d7791310ff79..8dd2116e1235 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.hypervisor.kvm.storage; +import java.io.File; import java.util.List; import java.util.Map; @@ -100,6 +101,10 @@ default Long getUsedIops() { public Map getDetails(); + default String getLocalPathFor(String relativePath) { + return String.format("%s%s%s", getLocalPath(), File.separator, relativePath); + } + public boolean isPoolSupportHA(); public String getHearthBeatPath(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 36d9cad1796a..decbe9c013a2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -25,9 +25,11 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermissions; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -42,8 +44,17 @@ import java.util.stream.Collectors; import javax.naming.ConfigurationException; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; import com.cloud.agent.api.Command; +import com.cloud.hypervisor.kvm.resource.LibvirtXMLParser; import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.direct.download.DirectDownloadHelper; @@ -79,9 +90,12 @@ import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuObject; import org.apache.cloudstack.utils.qemu.QemuObject.EncryptFormat; +import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; @@ -143,7 +157,10 @@ import com.cloud.utils.script.Script; import com.cloud.utils.storage.S3.S3Utils; import com.cloud.vm.VmDetailConstants; - +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; public class KVMStorageProcessor implements StorageProcessor { protected Logger logger = LogManager.getLogger(getClass()); @@ -165,6 +182,47 @@ public class KVMStorageProcessor implements StorageProcessor { */ private final long waitDelayForVirshCommands = 1000L; + private int incrementalSnapshotTimeout; + + private static final String CHECKPOINT_XML_TEMP_DIR = "/tmp/cloudstack/checkpointXMLs"; + + private static final String BACKUP_XML_TEMP_DIR = "/tmp/cloudstack/backupXMLs"; + + private static final String BACKUP_BEGIN_COMMAND = "virsh backup-begin --domain %s --backupxml %s --checkpointxml %s"; + + private static final String BACKUP_XML = ""; + + private static final String INCREMENTAL_BACKUP_XML = "%s"; + + private static final String CHECKPOINT_XML = "%s"; + + private static final String CHECKPOINT_DUMP_XML_COMMAND = "virsh checkpoint-dumpxml --domain %s --checkpointname %s --no-domain"; + + private static final String DOMJOBINFO_COMPLETED_COMMAND = "virsh domjobinfo --domain %s --completed"; + + private static final String DOMJOBABORT_COMMAND = "virsh domjobabort --domain %s"; + + private static final String DUMMY_VM_XML = "\n" + + " %s\n" + + " 256\n" + + " 256\n" + + " 1\n" + + " \n" + + " hvm\n" + + " \n" + + " \n" + + " \n" + + " %s\n" + + " \n" + + " \n"+ + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + + public KVMStorageProcessor(final KVMStoragePoolManager storagePoolMgr, final LibvirtComputingResource resource) { this.storagePoolMgr = storagePoolMgr; this.resource = resource; @@ -191,6 +249,8 @@ public boolean configure(final String name, final Map params) th } _cmdsTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.CMDS_TIMEOUT) * 1000; + + incrementalSnapshotTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.INCREMENTAL_SNAPSHOT_TIMEOUT) * 1000; return true; } @@ -1572,6 +1632,8 @@ public Answer attachVolume(final AttachCommand cmd) { vol.getIopsReadRate(), vol.getIopsReadRateMax(), vol.getIopsReadRateMaxLength(), vol.getIopsWriteRate(), vol.getIopsWriteRateMax(), vol.getIopsWriteRateMaxLength(), volCacheMode, encryptDetails, disk.getDetails()); + resource.recreateCheckpointsOnVm(List.of((VolumeObjectTO) disk.getData()), vmName, conn); + return new AttachAnswer(disk); } catch (final LibvirtException e) { logger.debug(String.format("Failed to attach volume [id: %d, uuid: %s, name: %s, path: %s], due to ", @@ -1611,6 +1673,8 @@ public Answer dettachVolume(final DettachCommand cmd) { storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath()); + resource.removeCheckpointsOnVm(vmName, vol.getUuid(), vol.getCheckpointPaths()); + return new DettachAnswer(disk); } catch (final LibvirtException | InternalErrorException | CloudRuntimeException e) { logger.debug(String.format("Failed to detach volume [id: %d, uuid: %s, name: %s, path: %s], due to ", vol.getId(), vol.getUuid(), vol.getName(), vol.getPath()), e); @@ -1756,6 +1820,7 @@ public Answer createVolume(final CreateObjectCommand cmd) { public Answer createSnapshot(final CreateObjectCommand cmd) { final SnapshotObjectTO snapshotTO = (SnapshotObjectTO)cmd.getData(); final PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)snapshotTO.getDataStore(); + DataStoreTO imageStoreTo = snapshotTO.getImageStore(); final VolumeObjectTO volume = snapshotTO.getVolume(); final String snapshotName = UUID.randomUUID().toString(); final String vmName = volume.getVmName(); @@ -1773,101 +1838,45 @@ public Answer createSnapshot(final CreateObjectCommand cmd) { } } - if (state == DomainInfo.DomainState.VIR_DOMAIN_RUNNING && volume.requiresEncryption()) { + if (DomainInfo.DomainState.VIR_DOMAIN_RUNNING.equals(state) && volume.requiresEncryption()) { throw new CloudRuntimeException("VM is running, encrypted volume snapshots aren't supported"); } - final KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); + KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); + KVMStoragePool secondaryPool = imageStoreTo != null ? storagePoolMgr.getStoragePoolByURI(imageStoreTo.getUrl()) : null; - final KVMPhysicalDisk disk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), volume.getPath()); + KVMPhysicalDisk disk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), volume.getPath()); String diskPath = disk.getPath(); String snapshotPath = diskPath + File.separator + snapshotName; - if (state == DomainInfo.DomainState.VIR_DOMAIN_RUNNING && !primaryPool.isExternalSnapshot()) { - - validateAvailableSizeOnPoolToTakeVolumeSnapshot(primaryPool, disk); - - try { - snapshotPath = getSnapshotPathInPrimaryStorage(primaryPool.getLocalPath(), snapshotName); - - String diskLabel = takeVolumeSnapshot(resource.getDisks(conn, vmName), snapshotName, diskPath, vm); - String convertResult = convertBaseFileToSnapshotFileInPrimaryStorageDir(primaryPool, disk, snapshotPath, volume, cmd.getWait()); - - mergeSnapshotIntoBaseFile(vm, diskLabel, diskPath, snapshotName, volume, conn); - - validateConvertResult(convertResult, snapshotPath); - } catch (LibvirtException e) { - if (!e.getMessage().contains(LIBVIRT_OPERATION_NOT_SUPPORTED_MESSAGE)) { - throw e; - } - - logger.info(String.format("It was not possible to take live disk snapshot for volume [%s], in VM [%s], due to [%s]. We will take full snapshot of the VM" - + " and extract the disk instead. Consider upgrading your QEMU binary.", volume, vmName, e.getMessage())); - - takeFullVmSnapshotForBinariesThatDoesNotSupportLiveDiskSnapshot(vm, snapshotName, vmName); - primaryPool.createFolder(TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR); - extractDiskFromFullVmSnapshot(disk, volume, snapshotPath, snapshotName, vmName, vm); - } - - /* - * libvirt on RHEL6 doesn't handle resume event emitted from - * qemu - */ - vm = resource.getDomain(conn, vmName); - state = vm.getInfo().state; - if (state == DomainInfo.DomainState.VIR_DOMAIN_PAUSED) { - vm.resume(); + SnapshotObjectTO newSnapshot = new SnapshotObjectTO(); + if (DomainInfo.DomainState.VIR_DOMAIN_RUNNING.equals(state) && !primaryPool.isExternalSnapshot()) { + if (snapshotTO.isKvmIncrementalSnapshot()) { + newSnapshot = takeIncrementalVolumeSnapshotOfRunningVm(snapshotTO, primaryPool, secondaryPool, imageStoreTo != null ? imageStoreTo.getUrl() : null, snapshotName, volume, vm, conn, cmd.getWait()); + } else { + newSnapshot = takeFullVolumeSnapshotOfRunningVm(cmd, primaryPool, secondaryPool, disk, snapshotName, conn, vmName, diskPath, vm, volume, snapshotPath); } } else { - /** - * For RBD we can't use libvirt to do our snapshotting or any Bash scripts. - * libvirt also wants to store the memory contents of the Virtual Machine, - * but that's not possible with RBD since there is no way to store the memory - * contents in RBD. - * - * So we rely on the Java bindings for RBD to create our snapshot - * - * This snapshot might not be 100% consistent due to writes still being in the - * memory of the Virtual Machine, but if the VM runs a kernel which supports - * barriers properly (>2.6.32) this won't be any different then pulling the power - * cord out of a running machine. - */ if (primaryPool.getType() == StoragePoolType.RBD) { - try { - Rados r = radosConnect(primaryPool); - - final IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); - final Rbd rbd = new Rbd(io); - final RbdImage image = rbd.open(disk.getName()); - - logger.debug("Attempting to create RBD snapshot " + disk.getName() + "@" + snapshotName); - image.snapCreate(snapshotName); - - rbd.close(image); - r.ioCtxDestroy(io); - } catch (final Exception e) { - logger.error("A RBD snapshot operation on " + disk.getName() + " failed. The error was: " + e.getMessage()); - } + takeRbdVolumeSnapshotOfStoppedVm(primaryPool, disk, snapshotName); + newSnapshot.setPath(snapshotPath); } else if (primaryPool.getType() == StoragePoolType.CLVM) { - /* VM is not running, create a snapshot by ourself */ - final Script command = new Script(_manageSnapshotPath, _cmdsTimeout, logger); - command.add(MANAGE_SNAPSTHOT_CREATE_OPTION, disk.getPath()); - command.add(NAME_OPTION, snapshotName); - final String result = command.execute(); - if (result != null) { - logger.debug("Failed to manage snapshot: " + result); - return new CreateObjectAnswer("Failed to manage snapshot: " + result); - } + CreateObjectAnswer result = takeClvmVolumeSnapshotOfStoppedVm(disk, snapshotName); + if (result != null) return result; + newSnapshot.setPath(snapshotPath); } else { - snapshotPath = getSnapshotPathInPrimaryStorage(primaryPool.getLocalPath(), snapshotName); - String convertResult = convertBaseFileToSnapshotFileInPrimaryStorageDir(primaryPool, disk, snapshotPath, volume, cmd.getWait()); - validateConvertResult(convertResult, snapshotPath); + if (snapshotTO.isKvmIncrementalSnapshot()) { + newSnapshot = takeIncrementalVolumeSnapshotOfStoppedVm(snapshotTO, primaryPool, secondaryPool, imageStoreTo != null ? imageStoreTo.getUrl() : null, snapshotName, volume, conn, cmd.getWait()); + } else { + newSnapshot = takeFullVolumeSnapshotOfStoppedVm(cmd, primaryPool, secondaryPool, snapshotName, disk, volume); + } } } - final SnapshotObjectTO newSnapshot = new SnapshotObjectTO(); + if (secondaryPool != null) { + storagePoolMgr.deleteStoragePool(secondaryPool.getType(), secondaryPool.getUuid()); + } - newSnapshot.setPath(snapshotPath); return new CreateObjectAnswer(newSnapshot); } catch (CloudRuntimeException | LibvirtException | IOException ex) { String errorMsg = String.format("Failed take snapshot for volume [%s], in VM [%s], due to [%s].", volume, vmName, ex.getMessage()); @@ -1878,6 +1887,504 @@ public Answer createSnapshot(final CreateObjectCommand cmd) { } } + private SnapshotObjectTO createSnapshotToAndUpdatePathAndSize(String path, String fullPath) { + final File snapFile = new File(fullPath); + long size = 0; + + if (snapFile.exists()) { + size = snapFile.length(); + } + + SnapshotObjectTO snapshotObjectTo = new SnapshotObjectTO(); + + snapshotObjectTo.setPath(path); + snapshotObjectTo.setPhysicalSize(size); + + return snapshotObjectTo; + } + + private SnapshotObjectTO takeIncrementalVolumeSnapshotOfStoppedVm(SnapshotObjectTO snapshotObjectTO, KVMStoragePool primaryPool, KVMStoragePool secondaryPool, + String secondaryPoolUrl, String snapshotName, VolumeObjectTO volumeObjectTo, Connect conn, int wait) throws LibvirtException { + resource.validateLibvirtAndQemuVersionForIncrementalSnapshots(); + Domain vm = null; + logger.debug("Taking incremental volume snapshot of volume [{}]. Snapshot will be copied to [{}].", volumeObjectTo, + ObjectUtils.defaultIfNull(secondaryPool, primaryPool)); + try { + String vmName = String.format("DUMMY-VM-%s", snapshotName); + + String vmXml = getVmXml(primaryPool, volumeObjectTo, vmName); + + logger.debug("Creating dummy VM with volume [{}] to take an incremental snapshot of it.", volumeObjectTo); + resource.startVM(conn, vmName, vmXml, Domain.CreateFlags.PAUSED); + + vm = resource.getDomain(conn, vmName); + + resource.recreateCheckpointsOnVm(List.of(volumeObjectTo), vmName, conn); + + return takeIncrementalVolumeSnapshotOfRunningVm(snapshotObjectTO, primaryPool, secondaryPool, secondaryPoolUrl, snapshotName, volumeObjectTo, vm, conn, wait); + } catch (InternalErrorException | LibvirtException | CloudRuntimeException e) { + logger.error("Failed to take incremental volume snapshot of volume [{}] due to {}.", volumeObjectTo, e.getMessage(), e); + throw new CloudRuntimeException(e); + } finally { + if (vm != null) { + vm.destroy(); + } + } + } + + private String getVmXml(KVMStoragePool primaryPool, VolumeObjectTO volumeObjectTo, String vmName) { + String machine = resource.isGuestAarch64() ? LibvirtComputingResource.VIRT : LibvirtComputingResource.PC; + String cpuArch = resource.getGuestCpuArch() != null ? resource.getGuestCpuArch() : "x86_64"; + + return String.format(DUMMY_VM_XML, vmName, cpuArch, machine, resource.getHypervisorPath(), primaryPool.getLocalPathFor(volumeObjectTo.getPath())); + } + + private SnapshotObjectTO takeIncrementalVolumeSnapshotOfRunningVm(SnapshotObjectTO snapshotObjectTO, KVMStoragePool primaryPool, KVMStoragePool secondaryPool, + String secondaryPoolUrl, String snapshotName, VolumeObjectTO volumeObjectTo, Domain vm, Connect conn, int wait) { + logger.debug("Taking incremental volume snapshot of volume [{}] attached to running VM [{}]. Snapshot will be copied to [{}].", volumeObjectTo, volumeObjectTo.getVmName(), + ObjectUtils.defaultIfNull(secondaryPool, primaryPool)); + resource.validateLibvirtAndQemuVersionForIncrementalSnapshots(); + + Pair fullSnapshotPathAndDirPath = getFullSnapshotOrCheckpointPathAndDirPathOnCorrectStorage(primaryPool, secondaryPool, snapshotName, volumeObjectTo, false); + + String diskLabel; + String vmName; + try { + List disks = resource.getDisks(conn, vm.getName()); + diskLabel = getDiskLabelToSnapshot(disks, volumeObjectTo.getPath(), vm); + vmName = vm.getName(); + } catch (LibvirtException e) { + logger.error("Failed to get VM's disks or VM name due to: [{}].", e.getMessage(), e); + throw new CloudRuntimeException(e); + } + + String[] parents = snapshotObjectTO.getParents(); + String fullSnapshotPath = fullSnapshotPathAndDirPath.first(); + + String backupXml = generateBackupXml(volumeObjectTo, parents, diskLabel, fullSnapshotPath); + String checkpointXml = String.format(CHECKPOINT_XML, snapshotName, diskLabel); + + Path backupXmlPath = createFileAndWrite(backupXml, BACKUP_XML_TEMP_DIR, snapshotName); + Path checkpointXmlPath = createFileAndWrite(checkpointXml, CHECKPOINT_XML_TEMP_DIR, snapshotName); + + String backupCommand = String.format(BACKUP_BEGIN_COMMAND, vmName, backupXmlPath.toString(), checkpointXmlPath.toString()); + + createFolderOnCorrectStorage(primaryPool, secondaryPool, fullSnapshotPathAndDirPath); + + if (Script.runSimpleBashScript(backupCommand) == null) { + throw new CloudRuntimeException(String.format("Error backing up using backupXML [%s], checkpointXML [%s] for volume [%s].", backupXml, checkpointXml, + volumeObjectTo)); + } + + try { + waitForBackup(vmName); + } catch (CloudRuntimeException ex) { + cancelBackupJob(snapshotObjectTO); + throw ex; + } + + rebaseSnapshot(snapshotObjectTO, secondaryPool, secondaryPoolUrl, fullSnapshotPath, snapshotName, parents, wait); + + try { + Files.setPosixFilePermissions(Path.of(fullSnapshotPath), PosixFilePermissions.fromString("rw-r--r--")); + } catch (IOException ex) { + logger.warn("Failed to change permissions of snapshot [{}], snapshot download will not be possible.", snapshotName); + } + + String checkpointPath = dumpCheckpoint(primaryPool, secondaryPool, snapshotName, volumeObjectTo, vmName, parents); + + SnapshotObjectTO result = createSnapshotToAndUpdatePathAndSize(secondaryPool == null ? fullSnapshotPath : fullSnapshotPathAndDirPath.second() + File.separator + snapshotName, + fullSnapshotPath); + + result.setCheckpointPath(checkpointPath); + + return result; + } + + protected void createFolderOnCorrectStorage(KVMStoragePool primaryPool, KVMStoragePool secondaryPool, Pair fullSnapshotPathAndDirPath) { + if (secondaryPool == null) { + primaryPool.createFolder(fullSnapshotPathAndDirPath.second()); + } else { + secondaryPool.createFolder(fullSnapshotPathAndDirPath.second()); + } + } + + protected String generateBackupXml(VolumeObjectTO volumeObjectTo, String[] parents, String diskLabel, String fullSnapshotPath) { + if (parents == null) { + logger.debug("Snapshot of volume [{}] does not have a parent, taking a full snapshot.", volumeObjectTo); + return String.format(BACKUP_XML, diskLabel, fullSnapshotPath); + } else { + logger.debug("Snapshot of volume [{}] has parents [{}], taking an incremental snapshot.", volumeObjectTo, Arrays.toString(parents)); + String parentCheckpointName = getParentCheckpointName(parents); + return String.format(INCREMENTAL_BACKUP_XML, parentCheckpointName, diskLabel, fullSnapshotPath); + } + } + + private void waitForBackup(String vmName) throws CloudRuntimeException { + int timeout = incrementalSnapshotTimeout; + logger.debug("Waiting for backup of VM [{}] to finish, timeout is [{}].", vmName, timeout); + + String result; + + while (timeout > 0) { + result = checkBackupJob(vmName); + + if (result.contains("Completed") && result.contains("Backup")) { + return; + } + + timeout -= 10000; + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + throw new CloudRuntimeException(e); + } + } + + throw new CloudRuntimeException(String.format("Timeout while waiting for incremental snapshot for VM [%s] to finish.", vmName)); + } + + private void cancelBackupJob(SnapshotObjectTO snapshotObjectTO) { + Script.runSimpleBashScript(String.format(DOMJOBABORT_COMMAND, snapshotObjectTO.getVmName())); + + String result = checkBackupJob(snapshotObjectTO.getVmName()); + + if (result.contains("Backup") && result.contains("Cancelled")) { + logger.debug("Successfully canceled incremental snapshot job."); + } else { + logger.warn("Couldn't cancel the incremental snapshot job correctly. Job status is [{}].", result); + } + } + + private String checkBackupJob(String vmName) { + return Script.runSimpleBashScriptWithFullResult(String.format(DOMJOBINFO_COMPLETED_COMMAND, vmName), 10); + } + + protected void rebaseSnapshot(SnapshotObjectTO snapshotObjectTO, KVMStoragePool secondaryPool, String secondaryUrl, String snapshotPath, String snapshotName, String[] parents, int wait) { + if (parents == null) { + logger.debug("No need to rebase snapshot [{}], this snapshot has no parents, therefore it is the first on its backing chain.", snapshotName); + return; + } + String parentSnapshotPath; + + if (secondaryPool == null) { + parentSnapshotPath = parents[parents.length - 1]; + } else if (!secondaryUrl.equals(snapshotObjectTO.getParentStore().getUrl())) { + KVMStoragePool parentPool = storagePoolMgr.getStoragePoolByURI(snapshotObjectTO.getParentStore().getUrl()); + parentSnapshotPath = parentPool.getLocalPath() + File.separator + parents[parents.length - 1]; + storagePoolMgr.deleteStoragePool(parentPool.getType(), parentPool.getUuid()); + } else { + parentSnapshotPath = secondaryPool.getLocalPath() + File.separator + parents[parents.length - 1]; + } + + QemuImgFile snapshotFile = new QemuImgFile(snapshotPath); + QemuImgFile parentSnapshotFile = new QemuImgFile(parentSnapshotPath); + + logger.debug("Rebasing snapshot [{}] with parent [{}].", snapshotName, parentSnapshotPath); + + try { + QemuImg qemuImg = new QemuImg(wait); + qemuImg.rebase(snapshotFile, parentSnapshotFile, PhysicalDiskFormat.QCOW2.toString(), false); + } catch (LibvirtException | QemuImgException e) { + logger.error("Exception while rebasing incremental snapshot [{}] due to: [{}].", snapshotName, e.getMessage(), e); + throw new CloudRuntimeException(e); + } + } + + protected String getParentCheckpointName(String[] parents) { + String immediateParentPath = parents[parents.length - 1]; + return immediateParentPath.substring(immediateParentPath.lastIndexOf(File.separator) + 1); + } + + private Path createFileAndWrite(String content, String dir, String fileName) { + File dirFile = new File(dir); + if (!dirFile.exists()) { + dirFile.mkdirs(); + } + + Path filePath = Path.of(dirFile.getPath(), fileName); + try { + return Files.write(filePath, content.getBytes()); + } catch (IOException ex) { + String message = String.format("Error while writing file [%s].", filePath); + logger.error(message, ex); + throw new CloudRuntimeException(message, ex); + } + } + + private String dumpCheckpoint(KVMStoragePool primaryPool, KVMStoragePool secondaryPool, String snapshotName, VolumeObjectTO volumeObjectTo, String vmName, String[] snapshotParents) { + String result = Script.runSimpleBashScriptWithFullResult(String.format(CHECKPOINT_DUMP_XML_COMMAND, vmName, snapshotName), 10); + + String snapshotParent = null; + if (snapshotParents != null) { + String snapshotParentPath = snapshotParents[snapshotParents.length - 1]; + snapshotParent = snapshotParentPath.substring(snapshotParentPath.lastIndexOf(File.separator) + 1); + } + + return cleanupCheckpointXmlDumpCheckpointAndRedefine(result, primaryPool, secondaryPool, snapshotName, volumeObjectTo, snapshotParent, vmName); + } + + private String cleanupCheckpointXmlDumpCheckpointAndRedefine(String checkpointXml, KVMStoragePool primaryPool, KVMStoragePool secondaryPool, String snapshotName, VolumeObjectTO volumeObjectTo, String snapshotParent, String vmName) { + String updatedCheckpointXml; + try { + updatedCheckpointXml = updateCheckpointXml(checkpointXml, snapshotParent); + } catch (TransformerException | ParserConfigurationException | IOException | SAXException | + XPathExpressionException e) { + logger.error("Exception while parsing checkpoint XML [{}].", checkpointXml, e); + throw new CloudRuntimeException(e); + } + + Pair checkpointFullPathAndDirPath = getFullSnapshotOrCheckpointPathAndDirPathOnCorrectStorage(primaryPool, secondaryPool, snapshotName, volumeObjectTo, true); + + String fullPath = checkpointFullPathAndDirPath.first(); + String dirPath = checkpointFullPathAndDirPath.second(); + + KVMStoragePool workPool = ObjectUtils.defaultIfNull(secondaryPool, primaryPool); + workPool.createFolder(dirPath); + + logger.debug("Saving checkpoint of volume [{}], attached to VM [{}], referring to snapshot [{}] to path [{}].", volumeObjectTo, vmName, snapshotName, fullPath); + createFileAndWrite(updatedCheckpointXml, workPool.getLocalPath() + File.separator + dirPath, snapshotName); + + logger.debug("Redefining checkpoint on VM [{}].", vmName); + Script.runSimpleBashScript(String.format(LibvirtComputingResource.CHECKPOINT_CREATE_COMMAND, vmName, fullPath)); + + return fullPath; + } + + /** + * Updates the checkpoint XML, setting the parent to {@code snapshotParent} and removing any disks that were not backed up. + * @param checkpointXml checkpoint XML to be parsed + * @param snapshotParent snapshot parent + * */ + private String updateCheckpointXml(String checkpointXml, String snapshotParent) throws ParserConfigurationException, XPathExpressionException, IOException, SAXException, TransformerException { + logger.debug("Parsing checkpoint XML [{}].", checkpointXml); + + InputStream in = IOUtils.toInputStream(checkpointXml); + DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory(); + DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + Document doc = docBuilder.parse(in); + XPath xPath = XPathFactory.newInstance().newXPath(); + + updateParent(snapshotParent, doc, xPath); + + removeUnnecessaryDisks(doc, xPath); + + String finalXml = LibvirtXMLParser.getXml(doc); + + logger.debug("Checkpoint XML after parsing is [{}].", finalXml); + + return finalXml; + } + + /** + * Removes all the disk definitions on the checkpoint XML from disks that were not affected. + * @param checkpointXml the checkpoint XML to be updated. + * */ + private void removeUnnecessaryDisks(Document checkpointXml, XPath xPath) throws XPathExpressionException { + Node disksNode = (Node) xPath.compile("/domaincheckpoint/disks").evaluate(checkpointXml, XPathConstants.NODE); + NodeList disksNodeChildren = disksNode.getChildNodes(); + for (int j = 0; j < disksNodeChildren.getLength(); j++) { + Node diskNode = disksNodeChildren.item(j); + if (diskNode == null) { + continue; + } + if ("disk".equals(diskNode.getNodeName()) && "no".equals(diskNode.getAttributes().getNamedItem("checkpoint").getNodeValue())) { + disksNode.removeChild(diskNode); + logger.trace("Removing node [{}].", diskNode); + } + } + } + + /** + * Updates the parent on the {@code checkpointXml} to {@code snapshotParent}. If {@code snapshotParent} is null, removes the parent. + * @param checkpointXml the checkpoint XML to be updated + * @param snapshotParent the snapshot parent. Inform null if no parent. + * */ + private void updateParent(String snapshotParent, Document checkpointXml, XPath xPath) throws XPathExpressionException { + if (snapshotParent == null) { + Object parentNodeObject = xPath.compile("/domaincheckpoint/parent").evaluate(checkpointXml, XPathConstants.NODE); + if (parentNodeObject == null) { + return; + } + Node parentNode = (Node) parentNodeObject; + parentNode.getParentNode().removeChild(parentNode); + return; + } + + Node parentNameNode = (Node) xPath.compile("/domaincheckpoint/parent/name").evaluate(checkpointXml, XPathConstants.NODE); + parentNameNode.setTextContent(snapshotParent); + } + + /** + * If imageStore is not null, copy the snapshot directly to secondary storage, else, copy it to the primary storage. + * + * @return SnapshotObjectTO of the new snapshot. + * */ + private SnapshotObjectTO takeFullVolumeSnapshotOfRunningVm(CreateObjectCommand cmd, KVMStoragePool primaryPool, KVMStoragePool secondaryPool, KVMPhysicalDisk disk, String snapshotName, + Connect conn, String vmName, String diskPath, Domain vm, VolumeObjectTO volume, String snapshotPath) throws IOException, LibvirtException { + logger.debug("Taking full volume snapshot of volume [{}] attached to running VM [{}]. Snapshot will be copied to [{}].", volume, vmName, + ObjectUtils.defaultIfNull(secondaryPool, primaryPool)); + + validateAvailableSizeOnPoolToTakeVolumeSnapshot(primaryPool, disk); + String relativePath = null; + try { + String diskLabel = takeVolumeSnapshot(resource.getDisks(conn, vmName), snapshotName, diskPath, vm); + + Pair fullSnapPathAndDirPath = getFullSnapshotOrCheckpointPathAndDirPathOnCorrectStorage(primaryPool, secondaryPool, snapshotName, volume, false); + + snapshotPath = fullSnapPathAndDirPath.first(); + String directoryPath = fullSnapPathAndDirPath.second(); + relativePath = directoryPath + File.separator + snapshotName; + + String convertResult = convertBaseFileToSnapshotFileInStorageDir(ObjectUtils.defaultIfNull(secondaryPool, primaryPool), disk, snapshotPath, directoryPath, volume, cmd.getWait()); + + mergeSnapshotIntoBaseFile(vm, diskLabel, diskPath, snapshotName, volume, conn); + + validateConvertResult(convertResult, snapshotPath); + } catch (LibvirtException e) { + if (!e.getMessage().contains(LIBVIRT_OPERATION_NOT_SUPPORTED_MESSAGE)) { + throw e; + } + + logger.info("It was not possible to take live disk snapshot for volume [{}], in VM [{}], due to [{}]. We will take full snapshot of the VM" + + " and extract the disk instead. Consider upgrading your QEMU binary.", volume, vmName, e.getMessage()); + + takeFullVmSnapshotForBinariesThatDoesNotSupportLiveDiskSnapshot(vm, snapshotName, vmName); + primaryPool.createFolder(TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR); + extractDiskFromFullVmSnapshot(disk, volume, snapshotPath, snapshotName, vmName, vm); + } + + /* + * libvirt on RHEL6 doesn't handle resume event emitted from + * qemu + */ + vm = resource.getDomain(conn, vmName); + DomainInfo.DomainState state = vm.getInfo().state; + if (state == DomainInfo.DomainState.VIR_DOMAIN_PAUSED) { + vm.resume(); + } + + return createSnapshotToAndUpdatePathAndSize(secondaryPool == null ? snapshotPath : relativePath, snapshotPath); + } + + + private SnapshotObjectTO takeFullVolumeSnapshotOfStoppedVm(CreateObjectCommand cmd, KVMStoragePool primaryPool, KVMStoragePool secondaryPool, String snapshotName, KVMPhysicalDisk disk, VolumeObjectTO volume) throws IOException { + logger.debug("Taking full volume snapshot of volume [{}]. Snapshot will be copied to [{}].", volume, + ObjectUtils.defaultIfNull(secondaryPool, primaryPool)); + Pair fullSnapPathAndDirPath = getFullSnapshotOrCheckpointPathAndDirPathOnCorrectStorage(primaryPool, secondaryPool, snapshotName, volume, false); + + String snapshotPath = fullSnapPathAndDirPath.first(); + String directoryPath = fullSnapPathAndDirPath.second(); + String relativePath = directoryPath + File.separator + snapshotName; + + String convertResult = convertBaseFileToSnapshotFileInStorageDir(ObjectUtils.defaultIfNull(secondaryPool, primaryPool), disk, snapshotPath, directoryPath, volume, cmd.getWait()); + + validateConvertResult(convertResult, snapshotPath); + + return createSnapshotToAndUpdatePathAndSize(secondaryPool == null ? snapshotPath : relativePath, snapshotPath); + } + + private CreateObjectAnswer takeClvmVolumeSnapshotOfStoppedVm(KVMPhysicalDisk disk, String snapshotName) { + /* VM is not running, create a snapshot by ourself */ + final Script command = new Script(_manageSnapshotPath, _cmdsTimeout, logger); + command.add(MANAGE_SNAPSTHOT_CREATE_OPTION, disk.getPath()); + command.add(NAME_OPTION, snapshotName); + final String result = command.execute(); + if (result != null) { + String message = String.format("Failed to manage snapshot [%s] due to: [%s].", snapshotName, result); + logger.debug(message); + return new CreateObjectAnswer(message); + } + return null; + } + + /** + * For RBD we can't use libvirt to do our snapshotting or any Bash scripts. + * libvirt also wants to store the memory contents of the Virtual Machine, + * but that's not possible with RBD since there is no way to store the memory + * contents in RBD. + *

+ * So we rely on the Java bindings for RBD to create our snapshot + *

+ * This snapshot might not be 100% consistent due to writes still being in the + * memory of the Virtual Machine, but if the VM runs a kernel which supports + * barriers properly (>2.6.32) this won't be any different then pulling the power + * cord out of a running machine. + */ + private void takeRbdVolumeSnapshotOfStoppedVm(KVMStoragePool primaryPool, KVMPhysicalDisk disk, String snapshotName) { + try { + Rados r = radosConnect(primaryPool); + + final IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); + final Rbd rbd = new Rbd(io); + final RbdImage image = rbd.open(disk.getName()); + + logger.debug("Attempting to create RBD snapshot {}@{}", disk.getName(), snapshotName); + image.snapCreate(snapshotName); + + rbd.close(image); + r.ioCtxDestroy(io); + } catch (final Exception e) { + logger.error("A RBD snapshot operation on [{}] failed. The error was: {}", disk.getName(), e.getMessage(), e); + } + } + + /** + * Retrieves the disk label to take snapshot; + * @param disks List of VM's disks; + * @param diskPath Path of the disk to take snapshot; + * @param vm VM in which disks are; + * @return the label to take snapshot. If the disk path is not found in VM's XML, it will throw a CloudRuntimeException. + * @throws org.libvirt.LibvirtException if the disk is not found + */ + protected String getDiskLabelToSnapshot(List disks, String diskPath, Domain vm) throws LibvirtException { + logger.debug("Searching disk label of disk with path [{}] on VM [{}].", diskPath, vm.getName()); + for (DiskDef disk : disks) { + String diskDefPath = disk.getDiskPath(); + + if (StringUtils.isEmpty(diskDefPath)) { + continue; + } + + if (!diskDefPath.contains(diskPath)) { + continue; + } + logger.debug("Found disk label [{}] for volume with path [{}] on VM [{}].", disk.getDiskLabel(), diskPath, vm.getName()); + + return disk.getDiskLabel(); + } + + throw new CloudRuntimeException(String.format("VM [%s] has no disk with path [%s]. VM's XML [%s].", vm.getName(), diskPath, vm.getXMLDesc(0))); + } + + /** + * Gets the fully qualified path of the snapshot or checkpoint and the directory path. If a secondary pool is informed, the path will be on the secondary pool, + * otherwise, the path will be on the primary pool. + * @param primaryPool Primary pool definition, the path returned will be here if no secondary pool is informed; + * @param secondaryPool Secondary pool definition. If informed, the primary pool will be ignored and the path returned will be on the secondary pool; + * @param snapshotName Name of the snapshot; + * @param volume Volume that is being snapshot; + * @param checkpoint Whether to return a path for a snapshot or a snapshot's checkpoint; + * @return Fully qualified path and the directory path of the snapshot/checkpoint. + * */ + private Pair getFullSnapshotOrCheckpointPathAndDirPathOnCorrectStorage(KVMStoragePool primaryPool, KVMStoragePool secondaryPool, String snapshotName, + VolumeObjectTO volume, boolean checkpoint) { + String fullSnapshotPath; + String dirPath; + + if (secondaryPool == null) { + fullSnapshotPath = getSnapshotOrCheckpointPathInPrimaryStorage(primaryPool.getLocalPath(), snapshotName, checkpoint); + dirPath = checkpoint ? TemplateConstants.DEFAULT_CHECKPOINT_ROOT_DIR : TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR; + } else { + Pair fullPathAndDirectoryPath = getSnapshotOrCheckpointPathAndDirectoryPathInSecondaryStorage(secondaryPool.getLocalPath(), snapshotName, + volume.getAccountId(), volume.getVolumeId(), checkpoint); + + fullSnapshotPath = fullPathAndDirectoryPath.first(); + dirPath = fullPathAndDirectoryPath.second(); + } + return new Pair<>(fullSnapshotPath, dirPath); + } + protected void deleteFullVmSnapshotAfterConvertingItToExternalDiskSnapshot(Domain vm, String snapshotName, VolumeObjectTO volume, String vmName) throws LibvirtException { logger.debug(String.format("Deleting full VM snapshot [%s] of VM [%s] as we already converted it to an external disk snapshot of the volume [%s].", snapshotName, vmName, volume)); @@ -1978,18 +2485,22 @@ protected void manuallyDeleteUnusedSnapshotFile(boolean isLibvirtSupportingFlagD /** * Creates the snapshot directory in the primary storage, if it does not exist; then, converts the base file (VM's old writing file) to the snapshot directory. - * @param primaryPool Storage to create folder, if not exists; - * @param baseFile Base file of VM, which will be converted; + * @param pool Storage to create folder, if not exists; + * @param baseFile Base file of VM, which will be converted; * @param snapshotPath Path to convert the base file; + * @param snapshotFolder Folder where the snapshot will be converted to; + * @param volume Volume being snapshot, used for logging only; + * @param wait timeout; * @return null if the conversion occurs successfully or an error message that must be handled. */ - protected String convertBaseFileToSnapshotFileInPrimaryStorageDir(KVMStoragePool primaryPool, - KVMPhysicalDisk baseFile, String snapshotPath, VolumeObjectTO volume, int wait) { + + protected String convertBaseFileToSnapshotFileInStorageDir(KVMStoragePool pool, + KVMPhysicalDisk baseFile, String snapshotPath, String snapshotFolder, VolumeObjectTO volume, int wait) { try (KeyFile srcKey = new KeyFile(volume.getPassphrase())) { logger.debug( - String.format("Trying to convert volume [%s] (%s) to snapshot [%s].", volume, baseFile, snapshotPath)); + "Trying to convert volume [{}] ({}) to snapshot [{}].", volume, baseFile, snapshotPath); - primaryPool.createFolder(TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR); + pool.createFolder(snapshotFolder); convertTheBaseFileToSnapshot(baseFile, snapshotPath, wait, srcKey); } catch (QemuImgException | LibvirtException | IOException ex) { return String.format("Failed to convert %s snapshot of volume [%s] to [%s] due to [%s].", volume, baseFile, @@ -2022,14 +2533,26 @@ private void convertTheBaseFileToSnapshot(KVMPhysicalDisk baseFile, String snaps q.convert(srcFile, destFile, options, qemuObjects, qemuImageOpts, null, true); } + protected String getSnapshotOrCheckpointPathInPrimaryStorage(String primaryStoragePath, String snapshotName, boolean checkpoint) { + String rootDir = checkpoint ? TemplateConstants.DEFAULT_CHECKPOINT_ROOT_DIR : TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR; + return String.format("%s%s%s%s%s", primaryStoragePath, File.separator, rootDir, File.separator, snapshotName); + } + /** - * Retrieves the path of the snapshot on primary storage snapshot's dir. - * @param primaryStoragePath Path of the primary storage; + * Retrieves the path of the snapshot or snapshot's checkpoint on secondary storage snapshot's dir. + * @param secondaryStoragePath Path of the secondary storage; * @param snapshotName Snapshot name; - * @return the path of the snapshot in primary storage snapshot's dir. + * @param accountId accountId; + * @param volumeId volumeId; + * @param checkpoint Whether to return a path for a snapshot or a snapshot's checkpoint; + * @return the path of the snapshot or snapshot's checkpoint in secondary storage and the snapshot's dir. */ - protected String getSnapshotPathInPrimaryStorage(String primaryStoragePath, String snapshotName) { - return String.format("%s%s%s%s%s", primaryStoragePath, File.separator, TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR, File.separator, snapshotName); + protected Pair getSnapshotOrCheckpointPathAndDirectoryPathInSecondaryStorage(String secondaryStoragePath, String snapshotName, long accountId, long volumeId, boolean checkpoint) { + String rootDir = checkpoint ? TemplateConstants.DEFAULT_CHECKPOINT_ROOT_DIR : TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR; + String snapshotParentDirectories = String.format("%s%s%s%s%s", rootDir, File.separator, accountId, File.separator, volumeId); + String fullSnapPath = String.format("%s%s%s%s%s", secondaryStoragePath, File.separator, snapshotParentDirectories, File.separator, snapshotName); + + return new Pair<>(fullSnapPath, snapshotParentDirectories); } /** @@ -2276,6 +2799,7 @@ private KVMPhysicalDisk createVolumeFromSnapshotOnNFS(CopyCommand cmd, PrimaryDa KVMPhysicalDisk disk = storagePoolMgr.copyPhysicalDisk(snapshotDisk, path != null ? path : volUuid, primaryPool, cmd.getWaitInMillSeconds()); storagePoolMgr.disconnectPhysicalDisk(pool.getPoolType(), pool.getUuid(), path); + secondaryPool.delete(); return disk; } @@ -2378,6 +2902,9 @@ public Answer deleteSnapshot(final DeleteCommand cmd) { logger.info(String.format("Deleting snapshot (id=%s, name=%s, path=%s, storage type=%s) on primary storage", snapshotTO.getId(), snapshotTO.getName(), snapshotTO.getPath(), primaryPool.getType())); deleteSnapshotFile(snapshotTO); + if (snapshotTO.isKvmIncrementalSnapshot()) { + deleteCheckpoint(snapshotTO); + } } else { logger.warn("Operation not implemented for storage pool type of " + primaryPool.getType().toString()); throw new InternalErrorException("Operation not implemented for storage pool type of " + primaryPool.getType().toString()); @@ -2399,6 +2926,24 @@ public Answer deleteSnapshot(final DeleteCommand cmd) { } } + /** + * Deletes the checkpoint dump if it exists. And deletes the checkpoint definition on the VM if it is running. + * */ + protected void deleteCheckpoint(SnapshotObjectTO snapshotTO) throws IOException { + if (snapshotTO.getCheckpointPath() != null) { + Files.deleteIfExists(Path.of(snapshotTO.getCheckpointPath())); + } + + String vmName = snapshotTO.getVmName(); + if (vmName == null) { + return; + } + String checkpointName = snapshotTO.getPath().substring(snapshotTO.getPath().lastIndexOf(File.separator) + 1); + + logger.debug("Deleting checkpoint [{}] of VM [{}].", checkpointName, vmName); + Script.runSimpleBashScript(String.format(LibvirtComputingResource.CHECKPOINT_DELETE_COMMAND, vmName, checkpointName)); + } + /** * Deletes the snapshot's file. * @throws CloudRuntimeException If can't delete the snapshot file. diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index e93f82ac4dd8..d3710b576631 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -34,6 +34,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.qemu.QemuImageOptions; import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.cloudstack.utils.qemu.QemuImgException; @@ -1608,13 +1609,13 @@ to support snapshots(backuped) as qcow2 files. */ } else { destFile = new QemuImgFile(destPath, destFormat); try { - qemu.convert(srcFile, destFile); + qemu.convert(srcFile, destFile, null, null, new QemuImageOptions(srcFile.getFormat(), srcFile.getFileName(), null), null, false, true); Map destInfo = qemu.info(destFile); Long virtualSize = Long.parseLong(destInfo.get(QemuImg.VIRTUAL_SIZE)); newDisk.setVirtualSize(virtualSize); newDisk.setSize(virtualSize); - } catch (QemuImgException | LibvirtException e) { - logger.error("Failed to convert " + srcFile.getFileName() + " to " + destFile.getFileName() + " the error was: " + e.getMessage()); + } catch (QemuImgException e) { + logger.error("Failed to convert [{}] to [{}] due to: [{}].", srcFile.getFileName(), destFile.getFileName(), e.getMessage(), e); newDisk = null; } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java index bc123b294ce7..751a16a15541 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java @@ -51,6 +51,7 @@ public class QemuImg { public static final String ENCRYPT_KEY_SECRET = "encrypt.key-secret"; public static final String TARGET_ZERO_FLAG = "--target-is-zero"; public static final long QEMU_2_10 = 2010000; + public static final long QEMU_5_10 = 5010000; /* The qemu-img binary. We expect this to be in $PATH */ public String _qemuImgPath = "qemu-img"; @@ -109,6 +110,10 @@ public static PreallocationType getPreallocationType(final Storage.ProvisioningT } } + public enum BitmapOperation { + Add, Remove, Clear, Enable, Disable, Merge + } + /** * Create a QemuImg object that supports skipping target zeroes * We detect this support via qemu-img help since support can @@ -381,6 +386,37 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, */ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, final Map options, final List qemuObjects, final QemuImageOptions srcImageOpts, final String snapshotName, final boolean forceSourceFormat) throws QemuImgException { + convert(srcFile, destFile, options, qemuObjects, srcImageOpts, snapshotName, forceSourceFormat, false); + } + + /** + * Converts an image from source to destination. + * + * This method is a facade for 'qemu-img convert' and converts a disk image or snapshot into a disk image with the specified filename and format. + * + * @param srcFile + * The source file. + * @param destFile + * The destination file. + * @param options + * Options for the conversion. Takes a Map with key value + * pairs which are passed on to qemu-img without validation. + * @param qemuObjects + * Pass qemu Objects to create - see objects in the qemu main page. + * @param srcImageOpts + * pass qemu --image-opts to convert. + * @param snapshotName + * If it is provided, conversion uses it as parameter. + * @param forceSourceFormat + * If true, specifies the source format in the conversion command. + * @param keepBitmaps + * If true, copies the bitmaps to the destination image. + * @return void + */ + public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, + final Map options, final List qemuObjects, final QemuImageOptions srcImageOpts, final String snapshotName, final boolean forceSourceFormat, + boolean keepBitmaps) throws QemuImgException { + Script script = new Script(_qemuImgPath, timeout); if (StringUtils.isNotBlank(snapshotName)) { String qemuPath = Script.runSimpleBashScript(getQemuImgPathScript); @@ -430,6 +466,10 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, script.add(srcFile.getFileName()); } + if (this.version >= QEMU_5_10 && keepBitmaps) { + script.add("--bitmaps"); + } + script.add(destFile.getFileName()); final String result = script.execute(); @@ -859,4 +899,39 @@ public String checkAndRepair(final QemuImgFile file, final QemuImageOptions imag return result; } + + + /** + * Perform one or more modifications of the persistent bitmap in {@code srcFile} + *
+ * This method is a facade for 'qemu-img bitmap'. + *
+ * Currently only the {@link BitmapOperation#Remove} is implemented + * + * @param bitmapOperation + * The operation to be performed + * @param srcfile + * The src file where the operation will be performed + * @param bitmapName + * The name of the bitmap + */ + public void bitmap(BitmapOperation bitmapOperation, QemuImgFile srcfile, String bitmapName) throws QemuImgException { + if (bitmapOperation != BitmapOperation.Remove) { + throw new QemuImgException("Operation not implemented."); + } + removeBitmap(srcfile, bitmapName); + } + + private void removeBitmap(QemuImgFile srcFile, String bitmapName) throws QemuImgException { + final Script script = new Script(_qemuImgPath); + script.add("bitmap"); + script.add("--remove"); + script.add(srcFile.getFileName()); + script.add(bitmapName); + + String result = script.execute(); + if (result != null) { + throw new QemuImgException(String.format("Exception while removing bitmap [%s] from file [%s]. Result is [%s].", srcFile.getFileName(), bitmapName, result)); + } + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index d586f01d4eb8..35c5a3bea930 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -241,6 +242,13 @@ public class LibvirtComputingResourceTest { @Mock LibvirtDomainXMLParser parserMock; + @Mock + DiskTO diskToMock; + + @Mock + private VolumeObjectTO volumeObjectToMock; + + @Spy private LibvirtComputingResource libvirtComputingResourceSpy = Mockito.spy(new LibvirtComputingResource()); @@ -5254,8 +5262,8 @@ public void testStartCommand() throws Exception { try (MockedStatic sshHelperMockedStatic = Mockito.mockStatic(SshHelper.class)) { sshHelperMockedStatic.when(() -> SshHelper.scpTo( Mockito.anyString(), Mockito.anyInt(), - Mockito.anyString(), Mockito.any(File.class), nullable(String.class), Mockito.anyString(), - Mockito.any(String[].class), Mockito.anyString())).thenAnswer(invocation -> null); + Mockito.anyString(), any(File.class), nullable(String.class), Mockito.anyString(), + any(String[].class), Mockito.anyString())).thenAnswer(invocation -> null); final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance(); assertNotNull(wrapper); @@ -5334,8 +5342,8 @@ public void testStartCommandIsolationEc2() throws Exception { try (MockedStatic sshHelperMockedStatic = Mockito.mockStatic(SshHelper.class)) { sshHelperMockedStatic.when(() -> SshHelper.scpTo( Mockito.anyString(), Mockito.anyInt(), - Mockito.anyString(), Mockito.any(File.class), nullable(String.class), Mockito.anyString(), - Mockito.any(String[].class), Mockito.anyString())).thenAnswer(invocation -> null); + Mockito.anyString(), any(File.class), nullable(String.class), Mockito.anyString(), + any(String[].class), Mockito.anyString())).thenAnswer(invocation -> null); final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance(); assertNotNull(wrapper); @@ -5373,8 +5381,12 @@ public void testStartCommandHostMemory() { when(vmSpec.getNics()).thenReturn(nics); when(vmSpec.getType()).thenReturn(VirtualMachine.Type.User); when(vmSpec.getName()).thenReturn(vmName); + when(vmSpec.getDisks()).thenReturn(new DiskTO[]{diskToMock}); + when(diskToMock.getData()).thenReturn(new VolumeObjectTO()); when(libvirtComputingResourceMock.createVMFromSpec(vmSpec)).thenReturn(vmDef); + when(libvirtComputingResourceMock.recreateCheckpointsOnVm(any(), any(), any())).thenReturn(true); + when(libvirtComputingResourceMock.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); try { when(libvirtUtilitiesHelper.getConnectionByType(vmDef.getHvsType())).thenReturn(conn); @@ -5585,7 +5597,7 @@ public void testKnownCommand() { public void testAddExtraConfigComponentEmptyExtraConfig() { libvirtComputingResourceMock = new LibvirtComputingResource(); libvirtComputingResourceMock.addExtraConfigComponent(new HashMap<>(), vmDef); - Mockito.verify(vmDef, never()).addComp(Mockito.any()); + Mockito.verify(vmDef, never()).addComp(any()); } @Test @@ -5596,7 +5608,7 @@ public void testAddExtraConfigComponentNotEmptyExtraConfig() { extraConfig.put("extraconfig-2", "value2"); extraConfig.put("extraconfig-3", "value3"); libvirtComputingResourceMock.addExtraConfigComponent(extraConfig, vmDef); - Mockito.verify(vmDef, times(1)).addComp(Mockito.any()); + Mockito.verify(vmDef, times(1)).addComp(any()); } public void validateGetCurrentMemAccordingToMemBallooningWithoutMemBalooning(){ @@ -5815,7 +5827,7 @@ public void defineResourceNetworkInterfacesTestUseProperties() { try (MockedStatic ignored = Mockito.mockStatic(AgentPropertiesFileHandler.class); MockedStatic netUtilsMockedStatic = Mockito.mockStatic(NetUtils.class)) { - Mockito.when(AgentPropertiesFileHandler.getPropertyValue(Mockito.any())).thenReturn("cloudbr15", + Mockito.when(AgentPropertiesFileHandler.getPropertyValue(any())).thenReturn("cloudbr15", "cloudbr28"); Mockito.when(NetUtils.getNetworkInterface(Mockito.anyString())).thenReturn(networkInterfaceMock1, @@ -5890,7 +5902,7 @@ public void testGetHaproxyStatsMethod() throws Exception { (mock, context) -> { doNothing().when(mock).add(Mockito.anyString()); when(mock.execute()).thenReturn(null); - when(mock.execute(Mockito.any())).thenReturn(null); + when(mock.execute(any())).thenReturn(null); }); MockedConstruction ignored = Mockito.mockConstruction(OneLineParser.class, (mock, context) -> { when(mock.getLine()).thenReturn("result"); @@ -5946,7 +5958,7 @@ public void testNetworkUsageMethod1() throws Exception { (mock, context) -> { doNothing().when(mock).add(Mockito.anyString()); when(mock.execute()).thenReturn(null); - when(mock.execute(Mockito.any())).thenReturn(null); + when(mock.execute(any())).thenReturn(null); }); MockedConstruction ignored2 = Mockito.mockConstruction(OneLineParser.class, (mock, context) -> {when(mock.getLine()).thenReturn("result");})) { @@ -5968,7 +5980,7 @@ public void testNetworkUsageMethod2() throws Exception { (mock, context) -> { doNothing().when(mock).add(Mockito.anyString()); when(mock.execute()).thenReturn(null); - when(mock.execute(Mockito.any())).thenReturn(null); + when(mock.execute(any())).thenReturn(null); }); MockedConstruction ignored2 = Mockito.mockConstruction(OneLineParser.class, (mock, context) -> {when(mock.getLine()).thenReturn("result");})) { @@ -5988,7 +6000,7 @@ public void getVmsToSetMemoryBalloonStatsPeriodTestLibvirtError() throws Libvirt List result = libvirtComputingResourceSpy.getVmsToSetMemoryBalloonStatsPeriod(connMock); - Mockito.verify(loggerMock).error(Mockito.anyString(), (Throwable) Mockito.any()); + Mockito.verify(loggerMock).error(Mockito.anyString(), (Throwable) any()); Assert.assertTrue(result.isEmpty()); } @@ -6078,7 +6090,7 @@ public void setupMemoryBalloonStatsPeriodTestMemBalloonPropertyDisabled() throws Mockito.when(parserMock.parseDomainXML(Mockito.anyString())).thenReturn(true); Mockito.when(parserMock.getMemBalloon()).thenReturn(memBalloonDef); try (MockedStatic