Skip to content

Commit 6d16ac2

Browse files
sureshanapartisureshanaparti
authored
ScaleIO/PowerFlex smoke tests improvements, and some fixes (#11554)
* ScaleIO/PowerFlex smoke tests improvements, and some fixes * Fix test_volumes.py, encrypted volume size check (for powerflex volumes) * Fix test_over_provisioning.py (over provisioning supported for powerflex) * Update vm snapshot tests * Update volume size delta in primary storage resource count for user vm volumes only The VR volumes resource count for PowerFlex volumes is updated here, resulting in resource count discrepancy (which is re-calculated through ResourceCountCheckTask later, and skips the VR volumes) * Fix test_import_unmanage_volumes.py (unsupported for powerflex) * Fix test_sharedfs_lifecycle.py (volume size check for powerflex) * Update powerflex.connect.on.demand config default to true
1 parent 253ac03 commit 6d16ac2

File tree

22 files changed

+375
-93
lines changed

22 files changed

+375
-93
lines changed

‎engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,7 @@
177177
import com.cloud.vm.dao.UserVmCloneSettingDao;
178178
import com.cloud.vm.dao.UserVmDao;
179179
import com.cloud.vm.dao.UserVmDetailsDao;
180+
import com.cloud.vm.dao.VMInstanceDao;
180181

181182
public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrationService, Configurable {
182183

@@ -257,6 +258,8 @@ public enum UserVmCloneType {
257258
StoragePoolHostDao storagePoolHostDao;
258259
@Inject
259260
DiskOfferingDao diskOfferingDao;
261+
@Inject
262+
VMInstanceDao vmInstanceDao;
260263

261264
@Inject
262265
protected SnapshotHelper snapshotHelper;
@@ -933,9 +936,7 @@ private DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering
933936

934937
// Create event and update resource count for volumes if vm is a user vm
935938
if (vm.getType() == VirtualMachine.Type.User) {
936-
937939
Long offeringId = null;
938-
939940
if (!offering.isComputeOnly()) {
940941
offeringId = offering.getId();
941942
}
@@ -1868,14 +1869,18 @@ protected void updateVolumeSize(DataStore store, VolumeVO vol) throws ResourceAl
18681869

18691870
if (newSize != vol.getSize()) {
18701871
DiskOfferingVO diskOffering = diskOfferingDao.findByIdIncludingRemoved(vol.getDiskOfferingId());
1871-
if (newSize > vol.getSize()) {
1872-
_resourceLimitMgr.checkPrimaryStorageResourceLimit(_accountMgr.getActiveAccountById(vol.getAccountId()),
1873-
vol.isDisplay(), newSize - vol.getSize(), diskOffering);
1874-
_resourceLimitMgr.incrementVolumePrimaryStorageResourceCount(vol.getAccountId(), vol.isDisplay(),
1875-
newSize - vol.getSize(), diskOffering);
1876-
} else {
1877-
_resourceLimitMgr.decrementVolumePrimaryStorageResourceCount(vol.getAccountId(), vol.isDisplay(),
1878-
vol.getSize() - newSize, diskOffering);
1872+
VMInstanceVO vm = vol.getInstanceId() != null ? vmInstanceDao.findById(vol.getInstanceId()) : null;
1873+
if (vm == null || vm.getType() == VirtualMachine.Type.User) {
1874+
// Update resource count for user vm volumes when volume is attached
1875+
if (newSize > vol.getSize()) {
1876+
_resourceLimitMgr.checkPrimaryStorageResourceLimit(_accountMgr.getActiveAccountById(vol.getAccountId()),
1877+
vol.isDisplay(), newSize - vol.getSize(), diskOffering);
1878+
_resourceLimitMgr.incrementVolumePrimaryStorageResourceCount(vol.getAccountId(), vol.isDisplay(),
1879+
newSize - vol.getSize(), diskOffering);
1880+
} else {
1881+
_resourceLimitMgr.decrementVolumePrimaryStorageResourceCount(vol.getAccountId(), vol.isDisplay(),
1882+
vol.getSize() - newSize, diskOffering);
1883+
}
18791884
}
18801885
vol.setSize(newSize);
18811886
_volsDao.persist(vol);

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3101,7 +3101,7 @@ public int compare(final DiskTO arg0, final DiskTO arg1) {
31013101
}
31023102

31033103
if (vmSpec.getOs().toLowerCase().contains("window")) {
3104-
isWindowsTemplate =true;
3104+
isWindowsTemplate = true;
31053105
}
31063106
for (final DiskTO volume : disks) {
31073107
KVMPhysicalDisk physicalDisk = null;
@@ -3220,6 +3220,9 @@ public int compare(final DiskTO arg0, final DiskTO arg1) {
32203220
disk.defNetworkBasedDisk(physicalDisk.getPath().replace("rbd:", ""), pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(),
32213221
pool.getUuid(), devId, diskBusType, DiskProtocol.RBD, DiskDef.DiskFmtType.RAW);
32223222
} else if (pool.getType() == StoragePoolType.PowerFlex) {
3223+
if (isWindowsTemplate && isUefiEnabled) {
3224+
diskBusTypeData = DiskDef.DiskBus.SATA;
3225+
}
32233226
disk.defBlockBasedDisk(physicalDisk.getPath(), devId, diskBusTypeData);
32243227
if (physicalDisk.getFormat().equals(PhysicalDiskFormat.QCOW2)) {
32253228
disk.setDiskFormatType(DiskDef.DiskFmtType.QCOW2);
@@ -3250,7 +3253,6 @@ public int compare(final DiskTO arg0, final DiskTO arg1) {
32503253
disk.defFileBasedDisk(physicalDisk.getPath(), devId, diskBusType, DiskDef.DiskFmtType.QCOW2);
32513254
}
32523255
}
3253-
32543256
}
32553257
pool.customizeLibvirtDiskDef(disk);
32563258
}
@@ -4527,6 +4529,14 @@ protected String getDiskPathFromDiskDef(DiskDef disk) {
45274529
return token[1];
45284530
}
45294531
} else if (token.length > 3) {
4532+
// for powerflex/scaleio, path = /dev/disk/by-id/emc-vol-2202eefc4692120f-540fd8fa00000003
4533+
if (token.length > 4 && StringUtils.isNotBlank(token[4]) && token[4].startsWith("emc-vol-")) {
4534+
final String[] emcVolToken = token[4].split("-");
4535+
if (emcVolToken.length == 4) {
4536+
return emcVolToken[3];
4537+
}
4538+
}
4539+
45304540
// for example, path = /mnt/pool_uuid/disk_path/
45314541
return token[3];
45324542
}

‎plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ public ListResponse<VmMetricsStatsResponse> searchForSystemVmMetricsStats(ListSy
234234
@Override
235235
public ListResponse<VolumeMetricsStatsResponse> searchForVolumeMetricsStats(ListVolumesUsageHistoryCmd cmd) {
236236
Pair<List<VolumeVO>, Integer> volumeList = searchForVolumesInternal(cmd);
237-
Map<Long,List<VolumeStatsVO>> volumeStatsList = searchForVolumeMetricsStatsInternal(cmd, volumeList.first());
237+
Map<Long, List<VolumeStatsVO>> volumeStatsList = searchForVolumeMetricsStatsInternal(cmd, volumeList.first());
238238
return createVolumeMetricsStatsResponse(volumeList, volumeStatsList);
239239
}
240240

‎plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -571,8 +571,8 @@ public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId
571571
}
572572
}
573573
} else {
574-
logger.debug("No encryption configured for data volume [id: {}, uuid: {}, name: {}]",
575-
volumeInfo.getId(), volumeInfo.getUuid(), volumeInfo.getName());
574+
logger.debug("No encryption configured for volume [id: {}, uuid: {}, name: {}]",
575+
volumeInfo.getId(), volumeInfo.getUuid(), volumeInfo.getName());
576576
}
577577

578578
return answer;
@@ -1512,7 +1512,7 @@ public void provideVmTags(long vmId, long volumeId, String tagValue) {
15121512
* @return true if resize is required
15131513
*/
15141514
private boolean needsExpansionForEncryptionHeader(long srcSize, long dstSize) {
1515-
int headerSize = 32<<20; // ensure we have 32MiB for encryption header
1515+
int headerSize = 32 << 20; // ensure we have 32MiB for encryption header
15161516
return srcSize + headerSize > dstSize;
15171517
}
15181518

‎plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public class ScaleIOSDCManagerImpl implements ScaleIOSDCManager, Configurable {
6161
static ConfigKey<Boolean> ConnectOnDemand = new ConfigKey<>("Storage",
6262
Boolean.class,
6363
"powerflex.connect.on.demand",
64-
Boolean.FALSE.toString(),
64+
Boolean.TRUE.toString(),
6565
"Connect PowerFlex client on Host when first Volume is mapped to SDC and disconnect when last Volume is unmapped from SDC," +
6666
" otherwise no action (that is connection remains in the same state whichever it is, connected or disconnected).",
6767
Boolean.TRUE,

‎server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -511,7 +511,7 @@ protected void checkDomainResourceLimit(final Account account, final Project pro
511511
String convCurrentResourceReservation = String.valueOf(currentResourceReservation);
512512
String convNumResources = String.valueOf(numResources);
513513

514-
if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){
514+
if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage) {
515515
convDomainResourceLimit = toHumanReadableSize(domainResourceLimit);
516516
convCurrentDomainResourceCount = toHumanReadableSize(currentDomainResourceCount);
517517
convCurrentResourceReservation = toHumanReadableSize(currentResourceReservation);
@@ -554,7 +554,7 @@ protected void checkAccountResourceLimit(final Account account, final Project pr
554554
String convertedCurrentResourceReservation = String.valueOf(currentResourceReservation);
555555
String convertedNumResources = String.valueOf(numResources);
556556

557-
if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){
557+
if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage) {
558558
convertedAccountResourceLimit = toHumanReadableSize(accountResourceLimit);
559559
convertedCurrentResourceCount = toHumanReadableSize(currentResourceCount);
560560
convertedCurrentResourceReservation = toHumanReadableSize(currentResourceReservation);
@@ -1137,7 +1137,7 @@ protected boolean updateResourceCountForAccount(final long accountId, final Reso
11371137
}
11381138
if (logger.isDebugEnabled()) {
11391139
String convertedDelta = String.valueOf(delta);
1140-
if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){
1140+
if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage) {
11411141
convertedDelta = toHumanReadableSize(delta);
11421142
}
11431143
String typeStr = StringUtils.isNotEmpty(tag) ? String.format("%s (tag: %s)", type, tag) : type.getName();

‎server/src/main/java/com/cloud/server/StatsCollector.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1459,7 +1459,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
14591459
for (VmDiskStats vmDiskStat : vmDiskStats) {
14601460
VmDiskStatsEntry vmDiskStatEntry = (VmDiskStatsEntry)vmDiskStat;
14611461
SearchCriteria<VolumeVO> sc_volume = _volsDao.createSearchCriteria();
1462-
sc_volume.addAnd("path", SearchCriteria.Op.EQ, vmDiskStatEntry.getPath());
1462+
sc_volume.addAnd("path", SearchCriteria.Op.LIKE, vmDiskStatEntry.getPath() + "%");
14631463
List<VolumeVO> volumes = _volsDao.search(sc_volume, null);
14641464

14651465
if (CollectionUtils.isEmpty(volumes))

‎server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public interface SnapshotManager extends Configurable {
5757
public static final ConfigKey<Integer> BackupRetryInterval = new ConfigKey<Integer>(Integer.class, "backup.retry.interval", "Advanced", "300",
5858
"Time in seconds between retries in backing up snapshot to secondary", false, ConfigKey.Scope.Global, null);
5959

60-
public static final ConfigKey<Boolean> VmStorageSnapshotKvm = new ConfigKey<>(Boolean.class, "kvm.vmstoragesnapshot.enabled", "Snapshots", "false", "For live snapshot of virtual machine instance on KVM hypervisor without memory. Requieres qemu version 1.6+ (on NFS or Local file system) and qemu-guest-agent installed on guest VM", true, ConfigKey.Scope.Global, null);
60+
public static final ConfigKey<Boolean> VmStorageSnapshotKvm = new ConfigKey<>(Boolean.class, "kvm.vmstoragesnapshot.enabled", "Snapshots", "false", "For live snapshot of virtual machine instance on KVM hypervisor without memory. Requires qemu version 1.6+ (on NFS or Local file system) and qemu-guest-agent installed on guest VM", true, ConfigKey.Scope.Global, null);
6161

6262
void deletePoliciesForVolume(Long volumeId);
6363

‎server/src/main/java/com/cloud/vm/UserVmManagerImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5860,7 +5860,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
58605860

58615861
for (VmDiskStatsEntry vmDiskStat : vmDiskStats) {
58625862
SearchCriteria<VolumeVO> sc_volume = _volsDao.createSearchCriteria();
5863-
sc_volume.addAnd("path", SearchCriteria.Op.EQ, vmDiskStat.getPath());
5863+
sc_volume.addAnd("path", SearchCriteria.Op.LIKE, vmDiskStat.getPath() + "%");
58645864
List<VolumeVO> volumes = _volsDao.search(sc_volume, null);
58655865
if ((volumes == null) || (volumes.size() == 0)) {
58665866
break;

‎test/integration/smoke/test_deploy_vm_root_resize.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
RESOURCE_PRIMARY_STORAGE
3333
from nose.plugins.attrib import attr
3434
from marvin.sshClient import SshClient
35+
import math
3536
import time
3637
import re
3738
from marvin.cloudstackAPI import updateTemplate,registerTemplate
@@ -276,6 +277,14 @@ def test_00_deploy_vm_root_resize(self):
276277
self.assertNotEqual(res[2], INVALID_INPUT, "Invalid list VM "
277278
"response")
278279
rootvolume = list_volume_response[0]
280+
list_volume_pool_response = list_storage_pools(
281+
self.apiclient,
282+
id=rootvolume.storageid
283+
)
284+
rootvolume_pool = list_volume_pool_response[0]
285+
if rootvolume_pool.type.lower() == "powerflex":
286+
newrootsize = (int(math.ceil(newrootsize / 8) * 8))
287+
279288
success = False
280289
if rootvolume is not None and rootvolume.size == (newrootsize << 30):
281290
success = True

0 commit comments

Comments
 (0)