Migrate volume improvements, to bypass secondary storage when copy volume between pools is allowed directly (apache#11625)

* Migrate volume improvements, to bypass secondary storage when copy volume between pools is allowed directly

* Bypass secondary storage for copy volume between zone-wide pools and
- local storage on host in the same zone
- cluser-wide pools in the same zone

* Bypass secondary storage for volumes on ceph/rdb pool when the scope permits

* Fix dest disk format while migrating volume from ceph/rbd to nfs, and some code improvements

* unit tests

* Update suitable disk offering(s) for volume(s) after migrate VM with volumes when change in pool type (shared or local)

Currently, Migrate VM with volume(s) bypasses the service and disk offerings of the volumes, as the target pools for migration are specified,
which ignores the offerings. Offering change is required when pool type (shared or local) is changed, mainly
- when volume on shared pool is migrated to local pool
- when volume on local pool is migrated to shared pool

* Update with proper message while migrate volume when target pool and offering type mismatches (both are not shared/local)

* Consider host scope first during endpoint selection while copying between primary storages

* Update disk offering count (for listDiskOfferings api) while removing offerings with tags mismatch with storage tags

Author: sureshanaparti

api/src/main/java/com/cloud/offering/DiskOffering.java

@@ -69,6 +69,8 @@ public String toString() {
 
     boolean isCustomized();
 
+    boolean isShared();
+
     void setDiskSize(long diskSize);
 
     long getDiskSize();
@@ -99,7 +101,6 @@ public String toString() {
 
     Long getBytesReadRateMaxLength();
 
-
     void setBytesWriteRate(Long bytesWriteRate);
 
     Long getBytesWriteRate();
@@ -112,7 +113,6 @@ public String toString() {
 
     Long getBytesWriteRateMaxLength();
 
-
     void setIopsReadRate(Long iopsReadRate);
 
     Long getIopsReadRate();
@@ -133,7 +133,6 @@ public String toString() {
 
     Long getIopsWriteRateMax();
 
-
     void setIopsWriteRateMaxLength(Long iopsWriteRateMaxLength);
 
     Long getIopsWriteRateMaxLength();

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -183,6 +183,8 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId,
      */
     boolean doesStoragePoolSupportDiskOfferingTags(StoragePool destPool, String diskOfferingTags);
 
+    boolean validateConditionsToReplaceDiskOfferingOfVolume(Volume volume, DiskOffering newDiskOffering, StoragePool destPool);
+
     Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge);
 
     void destroyVolume(long volumeId);

engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java

@@ -23,6 +23,7 @@
 import java.util.Set;
 
 import com.cloud.exception.ResourceAllocationException;
+import com.cloud.storage.Storage;
 import com.cloud.utils.Pair;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
@@ -187,10 +188,10 @@ List<DiskProfile> allocateTemplatedVolumes(Type type, String name, DiskOffering
      */
     DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops,
                              Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template,
-                             Account owner, Long deviceId, Long poolId, String path, String chainInfo);
+                             Account owner, Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo);
 
     DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template,
-                                     Long deviceId, Long poolId, String path, String chainInfo, DiskProfile diskProfile);
+                                     Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile);
 
     /**
      * Unmanage VM volumes

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ClusterScope.java

@@ -19,6 +19,7 @@
 package org.apache.cloudstack.engine.subsystem.api.storage;
 
 import com.cloud.storage.ScopeType;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 
 public class ClusterScope extends AbstractScope {
     private ScopeType type = ScopeType.CLUSTER;
@@ -51,4 +52,9 @@ public Long getZoneId() {
         return this.zoneId;
     }
 
+    @Override
+    public String toString() {
+        return String.format("ClusterScope %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(
+                this, "zoneId", "clusterId", "podId"));
+    }
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/HostScope.java

@@ -19,8 +19,10 @@
 package org.apache.cloudstack.engine.subsystem.api.storage;
 
 import com.cloud.storage.ScopeType;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 
 public class HostScope extends AbstractScope {
+    private ScopeType type = ScopeType.HOST;
     private Long hostId;
     private Long clusterId;
     private Long zoneId;
@@ -34,7 +36,7 @@ public HostScope(Long hostId, Long clusterId, Long zoneId) {
 
     @Override
     public ScopeType getScopeType() {
-        return ScopeType.HOST;
+        return this.type;
     }
 
     @Override
@@ -49,4 +51,10 @@ public Long getClusterId() {
     public Long getZoneId() {
         return zoneId;
     }
+
+    @Override
+    public String toString() {
+        return String.format("HostScope %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(
+                this, "zoneId", "clusterId", "hostId"));
+    }
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ZoneScope.java

@@ -19,6 +19,7 @@
 package org.apache.cloudstack.engine.subsystem.api.storage;
 
 import com.cloud.storage.ScopeType;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 
 public class ZoneScope extends AbstractScope {
     private ScopeType type = ScopeType.ZONE;
@@ -39,4 +40,9 @@ public Long getScopeId() {
         return this.zoneId;
     }
 
+    @Override
+    public String toString() {
+        return String.format("ZoneScope %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(
+                this, "zoneId"));
+    }
 }

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -2871,6 +2871,7 @@ private void markVolumesInPool(VMInstanceVO vm, Answer[] hypervisorMigrationResu
             }
             volume.setPath(result.getPath());
             volume.setPoolId(pool.getId());
+            volume.setPoolType(pool.getPoolType());
             if (result.getChainInfo() != null) {
                 volume.setChainInfo(result.getChainInfo());
             }

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java

@@ -1606,7 +1606,7 @@ public Volume migrateVolume(Volume volume, StoragePool destPool) throws StorageU
         String volumeToString = getVolumeIdentificationInfos(volume);
 
         VolumeInfo vol = volFactory.getVolume(volume.getId());
-        if (vol == null){
+        if (vol == null) {
             throw new CloudRuntimeException(String.format("Volume migration failed because volume [%s] is null.", volumeToString));
         }
         if (destPool == null) {
@@ -2487,6 +2487,7 @@ public void updateVolumeDiskChain(long volumeId, String path, String chainInfo,
                 StoragePoolVO pool = _storagePoolDao.findByUuid(updatedDataStoreUUID);
                 if (pool != null) {
                     vol.setPoolId(pool.getId());
+                    vol.setPoolType(pool.getPoolType());
                 }
             }
             _volsDao.update(volumeId, vol);
@@ -2496,7 +2497,7 @@ public void updateVolumeDiskChain(long volumeId, String path, String chainInfo,
     @Override
     public DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops,
                                     Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template, Account owner,
-                                    Long deviceId, Long poolId, String path, String chainInfo) {
+                                    Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo) {
         if (sizeInBytes == null) {
             sizeInBytes = offering.getDiskSize();
         }
@@ -2537,6 +2538,7 @@ public DiskProfile importVolume(Type type, String name, DiskOffering offering, L
 
         vol.setFormat(getSupportedImageFormatForCluster(hypervisorType));
         vol.setPoolId(poolId);
+        vol.setPoolType(poolType);
         vol.setPath(path);
         vol.setChainInfo(chainInfo);
         vol.setState(Volume.State.Ready);
@@ -2546,7 +2548,7 @@ public DiskProfile importVolume(Type type, String name, DiskOffering offering, L
 
     @Override
     public DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template,
-                                    Long deviceId, Long poolId, String path, String chainInfo, DiskProfile diskProfile) {
+                                    Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile) {
 
         VolumeVO vol = _volsDao.findById(diskProfile.getVolumeId());
         if (vm != null) {
@@ -2580,6 +2582,7 @@ public DiskProfile updateImportedVolume(Type type, DiskOffering offering, Virtua
 
         vol.setFormat(getSupportedImageFormatForCluster(vm.getHypervisorType()));
         vol.setPoolId(poolId);
+        vol.setPoolType(poolType);
         vol.setPath(path);
         vol.setChainInfo(chainInfo);
         vol.setSize(diskProfile.getSize());

engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java

@@ -241,7 +241,7 @@ public void testImportVolume() {
 
         volumeOrchestrator.importVolume(volumeType, name, diskOffering, sizeInBytes, null, null,
                 zoneId, hypervisorType, null, null, owner,
-                deviceId, poolId, path, chainInfo);
+                deviceId, poolId, Storage.StoragePoolType.NetworkFilesystem, path, chainInfo);
 
         VolumeVO volume = volumeVOMockedConstructionConstruction.constructed().get(0);
         Mockito.verify(volume, Mockito.never()).setInstanceId(Mockito.anyLong());

engine/schema/src/main/java/com/cloud/storage/DiskOfferingVO.java

@@ -604,6 +604,7 @@ public Integer getHypervisorSnapshotReserve() {
     @Override
     public void setEncrypt(boolean encrypt) { this.encrypt = encrypt; }
 
+    @Override
     public boolean isShared() {
         return !useLocalStorage;
     }