Skip to content

Commit da47703

Browse files
sureshanapartisureshanaparti
committed
Added wait time after SDC service start/restart/stop, retries to fetch SDC id/guid and some code improvements
1 parent 0ae2156 commit da47703

File tree

4 files changed

+87
-36
lines changed

4 files changed

+87
-36
lines changed

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.apache.cloudstack.utils.qemu.QemuImgException;
4141
import org.apache.cloudstack.utils.qemu.QemuImgFile;
4242
import org.apache.cloudstack.utils.qemu.QemuObject;
43+
import org.apache.commons.collections.MapUtils;
4344
import org.apache.commons.io.filefilter.WildcardFileFilter;
4445
import org.apache.logging.log4j.Logger;
4546
import org.apache.logging.log4j.LogManager;
@@ -643,12 +644,13 @@ public Ternary<Boolean, Map<String, String>, String> prepareStorageClient(String
643644
}
644645

645646
if (!ScaleIOUtil.isSDCServiceActive()) {
647+
logger.debug("SDC service is not active on host, starting it");
646648
if (!ScaleIOUtil.startSDCService()) {
647649
return new Ternary<>(false, null, "Couldn't start SDC service on host");
648650
}
649651
}
650652

651-
if (details != null && details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_MDMS)) {
653+
if (MapUtils.isNotEmpty(details) && details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_MDMS)) {
652654
// Assuming SDC service is started, add mdms
653655
String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS);
654656
String[] mdmAddresses = mdms.split(",");
@@ -667,7 +669,12 @@ public Ternary<Boolean, Map<String, String>, String> prepareStorageClient(String
667669
}
668670
}
669671

670-
return new Ternary<>( true, getSDCDetails(details), "Prepared client successfully");
672+
Map<String, String> sdcDetails = getSDCDetails(details);
673+
if (MapUtils.isEmpty(sdcDetails)) {
674+
return new Ternary<>(false, null, "Couldn't get the SDC details on the host");
675+
}
676+
677+
return new Ternary<>(true, sdcDetails, "Prepared client successfully");
671678
}
672679

673680
public Pair<Boolean, String> unprepareStorageClient(String uuid, Map<String, String> details) {
@@ -758,20 +765,37 @@ private void applyTimeout(Map<String, String> details) {
758765

759766
private Map<String, String> getSDCDetails(Map<String, String> details) {
760767
Map<String, String> sdcDetails = new HashMap<String, String>();
761-
if (details == null || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)) {
768+
if (MapUtils.isEmpty(details) || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)) {
762769
return sdcDetails;
763770
}
764771

765772
String storageSystemId = details.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
766-
String sdcId = ScaleIOUtil.getSdcId(storageSystemId);
767-
if (sdcId != null) {
768-
sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId);
769-
} else {
773+
if (StringUtils.isEmpty(storageSystemId)) {
774+
return sdcDetails;
775+
}
776+
777+
int waitTimeInSecs = 5;
778+
int timeBetweenTries = 1000; // Try more frequently (every sec) and return early when SDC Id or Guid found
779+
do {
780+
String sdcId = ScaleIOUtil.getSdcId(storageSystemId);
781+
if (sdcId != null) {
782+
sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId);
783+
return sdcDetails;
784+
}
785+
770786
String sdcGuId = ScaleIOUtil.getSdcGuid();
771787
if (sdcGuId != null) {
772788
sdcDetails.put(ScaleIOGatewayClient.SDC_GUID, sdcGuId);
789+
return sdcDetails;
773790
}
774-
}
791+
792+
try {
793+
Thread.sleep(timeBetweenTries);
794+
} catch (Exception ignore) {
795+
}
796+
waitTimeInSecs--;
797+
} while (waitTimeInSecs > 0);
798+
775799
return sdcDetails;
776800
}
777801

‎plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import javax.inject.Inject;
3232

3333
import com.cloud.host.HostVO;
34-
import com.cloud.storage.dao.StoragePoolAndAccessGroupMapDao;
3534
import org.apache.cloudstack.api.ApiConstants;
3635
import com.cloud.utils.StringUtils;
3736
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
@@ -105,8 +104,6 @@ public class ScaleIOPrimaryDataStoreLifeCycle extends BasePrimaryDataStoreLifeCy
105104
@Inject
106105
private AgentManager agentMgr;
107106
private ScaleIOSDCManager sdcManager;
108-
@Inject
109-
private StoragePoolAndAccessGroupMapDao storagePoolAndAccessGroupMapDao;
110107

111108
public ScaleIOPrimaryDataStoreLifeCycle() {
112109
sdcManager = new ScaleIOSDCManagerImpl();
@@ -306,13 +303,10 @@ public boolean attachZone(DataStore dataStore, ZoneScope scope, Hypervisor.Hyper
306303

307304
@Override
308305
public boolean maintain(DataStore store) {
309-
Map<String,String> details = new HashMap<>();
306+
Map<String, String> details = new HashMap<>();
310307
StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(store.getId());
311308
if (storagePoolVO != null) {
312-
details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(storagePoolVO.getDataCenterId())));
313-
details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(storagePoolVO.getDataCenterId())));
314-
details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(storagePoolVO.getDataCenterId())));
315-
309+
populateScaleIOConfiguration(details, storagePoolVO.getDataCenterId());
316310
StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
317311
if (systemIdDetail != null) {
318312
details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue());
@@ -331,13 +325,10 @@ public boolean maintain(DataStore store) {
331325

332326
@Override
333327
public boolean cancelMaintain(DataStore store) {
334-
Map<String,String> details = new HashMap<>();
328+
Map<String, String> details = new HashMap<>();
335329
StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(store.getId());
336330
if (storagePoolVO != null) {
337-
details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(storagePoolVO.getDataCenterId())));
338-
details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(storagePoolVO.getDataCenterId())));
339-
details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(storagePoolVO.getDataCenterId())));
340-
331+
populateScaleIOConfiguration(details, storagePoolVO.getDataCenterId());
341332
StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(store.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
342333
if (systemIdDetail != null) {
343334
details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemIdDetail.getValue());
@@ -420,4 +411,14 @@ private Hypervisor.HypervisorType getHypervisorTypeForCluster(long clusterId) {
420411
private static boolean isSupportedHypervisorType(Hypervisor.HypervisorType hypervisorType) {
421412
return Hypervisor.HypervisorType.KVM.equals(hypervisorType);
422413
}
414+
415+
private void populateScaleIOConfiguration(Map<String, String> details, long dataCenterId) {
416+
if (details == null) {
417+
details = new HashMap<>();
418+
}
419+
420+
details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(dataCenterId)));
421+
details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(dataCenterId)));
422+
details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(dataCenterId)));
423+
}
423424
}

‎plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
2828
import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
2929
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
30-
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
3130
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
3231
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
3332
import org.apache.cloudstack.storage.datastore.manager.ScaleIOSDCManager;
@@ -59,7 +58,6 @@ public class ScaleIOHostListener implements HypervisorHostListener {
5958
@Inject private DataStoreManager _dataStoreMgr;
6059
@Inject private HostDao _hostDao;
6160
@Inject private StoragePoolHostDao _storagePoolHostDao;
62-
@Inject private PrimaryDataStoreDao _primaryDataStoreDao;
6361
@Inject private StoragePoolDetailsDao _storagePoolDetailsDao;
6462
private ScaleIOSDCManager _sdcManager = new ScaleIOSDCManagerImpl();
6563

@@ -109,11 +107,9 @@ private String getSdcIdOfHost(HostVO host, DataStore dataStore) {
109107
if (systemId == null) {
110108
throw new CloudRuntimeException("Failed to get the system id for PowerFlex storage pool " + storagePool.getName());
111109
}
112-
Map<String,String> details = new HashMap<>();
110+
Map<String, String> details = new HashMap<>();
113111
details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
114-
details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(host.getDataCenterId())));
115-
details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(host.getDataCenterId())));
116-
details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(host.getDataCenterId())));
112+
populateScaleIOConfiguration(details,host.getDataCenterId());
117113
_sdcManager = ComponentContext.inject(_sdcManager);
118114
if (_sdcManager.areSDCConnectionsWithinLimit(poolId)) {
119115
details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));
@@ -123,7 +119,7 @@ private String getSdcIdOfHost(HostVO host, DataStore dataStore) {
123119

124120
ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, storagePool, storagePool.getPath(), details);
125121
ModifyStoragePoolAnswer answer = sendModifyStoragePoolCommand(cmd, storagePool, host);
126-
Map<String,String> poolDetails = answer.getPoolInfo().getDetails();
122+
Map<String, String> poolDetails = answer.getPoolInfo().getDetails();
127123
if (MapUtils.isEmpty(poolDetails)) {
128124
String msg = String.format("PowerFlex storage SDC details not found on the host: %s, (re)install SDC and restart agent", host);
129125
logger.warn(msg);
@@ -204,11 +200,9 @@ public boolean hostDisconnected(long hostId, long poolId) {
204200
if (systemId == null) {
205201
throw new CloudRuntimeException("Failed to get the system id for PowerFlex storage pool " + storagePool.getName());
206202
}
207-
Map<String,String> details = new HashMap<>();
203+
Map<String, String> details = new HashMap<>();
208204
details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
209-
details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(host.getDataCenterId())));
210-
details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(host.getDataCenterId())));
211-
details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(host.getDataCenterId())));
205+
populateScaleIOConfiguration(details,host.getDataCenterId());
212206
_sdcManager = ComponentContext.inject(_sdcManager);
213207
if (_sdcManager.canUnprepareSDC(host, dataStore)) {
214208
details.put(ScaleIOSDCManager.ConnectOnDemand.key(), String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));
@@ -253,4 +247,14 @@ private String getStoragePoolDetails(StoragePool storagePool) {
253247
}
254248
return poolDetails;
255249
}
250+
251+
private void populateScaleIOConfiguration(Map<String, String> details, long dataCenterId) {
252+
if (details == null) {
253+
details = new HashMap<>();
254+
}
255+
256+
details.put(ScaleIOSDCManager.MdmsChangeApplyTimeout.key(), String.valueOf(ScaleIOSDCManager.MdmsChangeApplyTimeout.valueIn(dataCenterId)));
257+
details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(), String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(dataCenterId)));
258+
details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(), String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(dataCenterId)));
259+
}
256260
}

‎plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ private static boolean removeMdm(String mdmAddress) {
176176

177177
/**
178178
* Run command, log command result and return {@link Boolean#TRUE} if command succeeded.
179-
* FIXME: may need to do refactoring and replace static method callss with dynamic.
179+
* FIXME: may need to do refactoring and replace static method calls with dynamic.
180180
*/
181181
private static boolean runCmd(String command) {
182182
Pair<String, String> result = Script.executeCommand(command);
@@ -467,17 +467,39 @@ public static boolean enableSDCService() {
467467

468468
public static boolean startSDCService() {
469469
int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_START_CMD);
470-
return exitValue == 0;
470+
if (exitValue != 0) {
471+
return false;
472+
}
473+
waitForSecs(3);
474+
return true;
471475
}
472476

473477
public static boolean stopSDCService() {
474478
int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_STOP_CMD);
475-
return exitValue == 0;
479+
if (exitValue != 0) {
480+
return false;
481+
}
482+
waitForSecs(1);
483+
return true;
476484
}
477485

478486
public static boolean restartSDCService() {
479487
int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_RESTART_CMD);
480-
return exitValue == 0;
488+
if (exitValue != 0) {
489+
return false;
490+
}
491+
waitForSecs(3);
492+
return true;
493+
}
494+
495+
private static void waitForSecs(long waitTimeInSecs) {
496+
if (waitTimeInSecs < 0) {
497+
waitTimeInSecs = 1;
498+
}
499+
try {
500+
Thread.sleep(waitTimeInSecs * 1000);
501+
} catch (InterruptedException ignore) {
502+
}
481503
}
482504

483505
/**

0 commit comments

Comments
 (0)