PowerFlex/ScaleIO client initialization, authentication and command execution improvements

sureshanaparti · sureshanaparti · commit cce01f1f7e52 · 2026-01-08T18:07:34.000+05:30
diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java
@@ -400,6 +400,13 @@ protected T valueInGlobalOrAvailableParentScope(Scope scope, Long id) {
         return value();
     }
 
+    /**
+     * Use {@link ConfigKey#valueInScope(Scope, Long)} instead.
+     */
+    public T valueInDomain(Long domainId) {
+        return valueInScope(Scope.Domain, domainId);
+    }
+
     public T valueInScope(Scope scope, Long id) {
         if (id == null) {
             return value();
diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
@@ -180,7 +180,8 @@ public void testUnprepareStorageClient_MDMNotAdded() {
         details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, "1.1.1.1,2.2.2.2");
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3);
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0);
-        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-3.3.3.3 [1]-4.4.4.4");
+        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 1.1.1.1"), Mockito.eq(ScaleIOUtil.DEFAULT_TIMEOUT_MS)))
+                .thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-3.3.3.3 [1]-4.4.4.4");
 
         Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(poolUuid, details);
 
@@ -196,11 +197,11 @@ public void testUnprepareStorageClient_RemoveMDMFailed() {
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0);
         when(Script.executeCommand(Mockito.eq("sed -i '/1.1.1.1\\,/d' /etc/emc/scaleio/drv_cfg.txt"))).thenReturn(new Pair<>(null, null));
         when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl restart scini"))).thenReturn(0);
-        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-1.1.1.1 [1]-2.2.2.2");
+        when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 1.1.1.1"), Mockito.eq(ScaleIOUtil.DEFAULT_TIMEOUT_MS)))
+                .thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-1.1.1.1 [1]-2.2.2.2");
         when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg"))).thenReturn(new Pair<>(null, null));
         when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_vols"))).thenReturn(new Pair<>("", null));
 
-
         Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(poolUuid, details);
 
         Assert.assertFalse(result.first());
diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePoolTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePoolTest.java
@@ -95,9 +95,8 @@ public void testSdcIdAttribute() {
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
 
         try (MockedStatic<Script> ignored = Mockito.mockStatic(Script.class)) {
-            when(Script.runSimpleBashScript(
-                    "/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 218ce1797566a00f|awk '{print $5}'")).thenReturn(
-                    sdcId);
+            when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 218ce1797566a00f|awk '{print $5}'"), Mockito.eq(ScaleIOUtil.DEFAULT_TIMEOUT_MS)))
+                    .thenReturn(sdcId);
 
             ScaleIOStoragePool pool1 = new ScaleIOStoragePool(uuid, "192.168.1.19", 443, "a519be2f00000000", type,
                     details, adapter);
@@ -116,10 +115,10 @@ public void testSdcGuidAttribute() {
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
 
         try (MockedStatic<Script> ignored = Mockito.mockStatic(Script.class)) {
-            when(Script.runSimpleBashScript(
-                    "/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 218ce1797566a00f|awk '{print $5}'")).thenReturn(
-                    null);
-            when(Script.runSimpleBashScript("/opt/emc/scaleio/sdc/bin/drv_cfg --query_guid")).thenReturn(sdcGuid);
+            when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 218ce1797566a00f|awk '{print $5}'"), Mockito.eq(ScaleIOUtil.DEFAULT_TIMEOUT_MS)))
+                    .thenReturn(null);
+            when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_guid --file /etc/emc/scaleio/drv_cfg.txt"), Mockito.eq(ScaleIOUtil.DEFAULT_TIMEOUT_MS)))
+                    .thenReturn(sdcGuid);
 
             ScaleIOStoragePool pool1 = new ScaleIOStoragePool(uuid, "192.168.1.19", 443, "a519be2f00000000", type,
                     details, adapter);
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientConnectionPool.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientConnectionPool.java
@@ -17,12 +17,12 @@
 
 package org.apache.cloudstack.storage.datastore.client;
 
-import java.net.URISyntaxException;
-import java.security.KeyManagementException;
-import java.security.NoSuchAlgorithmException;
+import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 
 import com.cloud.storage.StoragePool;
+import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
@@ -36,9 +36,9 @@
 public class ScaleIOGatewayClientConnectionPool {
     protected Logger logger = LogManager.getLogger(getClass());
 
-    private ConcurrentHashMap<Long, ScaleIOGatewayClient> gatewayClients;
-
+    private Map<Long, ScaleIOGatewayClient> gatewayClients;
     private static final ScaleIOGatewayClientConnectionPool instance;
+    private final Object lock = new Object();
 
     static {
         instance = new ScaleIOGatewayClientConnectionPool();
@@ -49,69 +49,76 @@ public static ScaleIOGatewayClientConnectionPool getInstance() {
     }
 
     private ScaleIOGatewayClientConnectionPool() {
-        gatewayClients = new ConcurrentHashMap<Long, ScaleIOGatewayClient>();
+        gatewayClients = new ConcurrentHashMap<>();
     }
 
     public ScaleIOGatewayClient getClient(StoragePool storagePool,
-                                          StoragePoolDetailsDao storagePoolDetailsDao)
-            throws NoSuchAlgorithmException, KeyManagementException, URISyntaxException {
+                                          StoragePoolDetailsDao storagePoolDetailsDao) {
         return getClient(storagePool.getId(), storagePool.getUuid(), storagePoolDetailsDao);
     }
 
 
     public ScaleIOGatewayClient getClient(DataStore dataStore,
-                                          StoragePoolDetailsDao storagePoolDetailsDao)
-            throws NoSuchAlgorithmException, KeyManagementException, URISyntaxException {
+                                          StoragePoolDetailsDao storagePoolDetailsDao) {
         return getClient(dataStore.getId(), dataStore.getUuid(), storagePoolDetailsDao);
     }
 
-
     private ScaleIOGatewayClient getClient(Long storagePoolId, String storagePoolUuid,
-                                           StoragePoolDetailsDao storagePoolDetailsDao)
-            throws NoSuchAlgorithmException, KeyManagementException, URISyntaxException {
+                                           StoragePoolDetailsDao storagePoolDetailsDao) {
 
         Preconditions.checkArgument(storagePoolId != null && storagePoolId > 0,
                 "Invalid storage pool id");
 
-        ScaleIOGatewayClient client = null;
-        synchronized (gatewayClients) {
-            client = gatewayClients.get(storagePoolId);
-            if (client == null) {
-                String url = null;
-                StoragePoolDetailVO urlDetail  = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_ENDPOINT);
-                if (urlDetail != null) {
-                    url = urlDetail.getValue();
-                }
-                String username = null;
-                StoragePoolDetailVO encryptedUsernameDetail = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_USERNAME);
-                if (encryptedUsernameDetail != null) {
-                    final String encryptedUsername = encryptedUsernameDetail.getValue();
-                    username = DBEncryptionUtil.decrypt(encryptedUsername);
+        logger.debug("Getting ScaleIO client for {} ({})", storagePoolId, storagePoolUuid);
+
+        ScaleIOGatewayClient client = gatewayClients.get(storagePoolId);
+        if (client == null) {
+            logger.debug("Before acquiring lock to create ScaleIO client for {} ({})", storagePoolId, storagePoolUuid);
+            synchronized (lock) {
+                logger.debug("Acquired lock to create ScaleIO client for {} ({})", storagePoolId, storagePoolUuid);
+                client = gatewayClients.get(storagePoolId);
+                if (client == null) {
+                    logger.debug("Initializing ScaleIO client for {} ({})", storagePoolId, storagePoolUuid);
+
+                    String url = Optional.ofNullable(storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_ENDPOINT))
+                            .map(StoragePoolDetailVO::getValue)
+                            .orElse(null);
+
+                    String username = Optional.ofNullable(storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_USERNAME))
+                            .map(StoragePoolDetailVO::getValue)
+                            .map(DBEncryptionUtil::decrypt)
+                            .orElse(null);
+
+                    String password = Optional.ofNullable(storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_PASSWORD))
+                            .map(StoragePoolDetailVO::getValue)
+                            .map(DBEncryptionUtil::decrypt)
+                            .orElse(null);
+
+                    int clientTimeout = StorageManager.STORAGE_POOL_CLIENT_TIMEOUT.valueIn(storagePoolId);
+                    int clientMaxConnections = StorageManager.STORAGE_POOL_CLIENT_MAX_CONNECTIONS.valueIn(storagePoolId);
+
+                    try {
+                        client = new ScaleIOGatewayClientImpl(url, username, password, false, clientTimeout, clientMaxConnections);
+                        logger.debug("Created ScaleIO client for the storage pool [id: {}, uuid: {}]", storagePoolId, storagePoolUuid);
+                        gatewayClients.put(storagePoolId, client);
+                    } catch (Exception e) {
+                        String msg = String.format("Failed to create ScaleIO client for the storage pool [id: %d, uuid: %s]", storagePoolId, storagePoolUuid);
+                        throw new CloudRuntimeException(msg, e);
+                    }
                 }
-                String password = null;
-                StoragePoolDetailVO encryptedPasswordDetail = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_PASSWORD);
-                if (encryptedPasswordDetail != null) {
-                    final String encryptedPassword = encryptedPasswordDetail.getValue();
-                    password = DBEncryptionUtil.decrypt(encryptedPassword);
-                }
-                final int clientTimeout = StorageManager.STORAGE_POOL_CLIENT_TIMEOUT.valueIn(storagePoolId);
-                final int clientMaxConnections = StorageManager.STORAGE_POOL_CLIENT_MAX_CONNECTIONS.valueIn(storagePoolId);
-
-                client = new ScaleIOGatewayClientImpl(url, username, password, false, clientTimeout, clientMaxConnections);
-                gatewayClients.put(storagePoolId, client);
-                logger.debug("Added gateway client for the storage pool [id: {}, uuid: {}]", storagePoolId, storagePoolUuid);
             }
         }
 
+        logger.debug("Returning ScaleIO client for {} ({})", storagePoolId, storagePoolUuid);
         return client;
     }
 
     public boolean removeClient(DataStore dataStore) {
         Preconditions.checkArgument(dataStore != null && dataStore.getId() > 0,
                 "Invalid storage pool id");
 
-        ScaleIOGatewayClient client = null;
-        synchronized (gatewayClients) {
+        ScaleIOGatewayClient client;
+        synchronized (lock) {
             client = gatewayClients.remove(dataStore.getId());
         }
 
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java
@@ -92,7 +92,7 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient {
 
     private String username;
     private String password;
-    private String sessionKey = null;
+    private String sessionKey;
 
     // The session token is valid for 8 hours from the time it was created, unless there has been no activity for 10 minutes
     // Reference: https://cpsdocs.dellemc.com/bundle/PF_REST_API_RG/page/GUID-92430F19-9F44-42B6-B898-87D5307AE59B.html
@@ -102,7 +102,7 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient {
     private static final long MAX_IDLE_TIME_IN_MILLISECS = MAX_IDLE_TIME_IN_MINS * 60 * 1000;
     private static final long BUFFER_TIME_IN_MILLISECS = 30 * 1000; // keep 30 secs buffer before the expiration (to avoid any last-minute operations)
 
-    private boolean authenticating = false;
+    private volatile boolean authenticating = false;
     private long createTime = 0;
     private long lastUsedTime = 0;
 
@@ -142,7 +142,6 @@ public ScaleIOGatewayClientImpl(final String url, final String username, final S
         this.username = username;
         this.password = password;
 
-        authenticate();
         logger.debug("API client for the PowerFlex gateway " + apiURI.getHost() + " is created successfully, with max connections: "
                 + maxConnections + " and timeout: " + timeout + " secs");
     }
@@ -181,7 +180,7 @@ private synchronized void authenticate() {
             long now = System.currentTimeMillis();
             createTime = lastUsedTime = now;
         } catch (final IOException e) {
-            logger.error("Failed to authenticate PowerFlex API Gateway " + apiURI.getHost() + " due to: " + e.getMessage() + getConnectionManagerStats());
+            logger.error("Failed to authenticate PowerFlex API Gateway " + apiURI.getHost() + " due to: " + e.getMessage() + getConnectionManagerStats(), e);
             throw new CloudRuntimeException("Failed to authenticate PowerFlex API Gateway " + apiURI.getHost() + " due to: " + e.getMessage());
         } finally {
             authenticating = false;
@@ -199,6 +198,10 @@ private synchronized void renewClientSessionOnExpiry() {
     }
 
     private boolean isSessionExpired() {
+        if (sessionKey == null) {
+            logger.debug("Session never created for the Gateway " + apiURI.getHost());
+            return true;
+        }
         long now = System.currentTimeMillis() + BUFFER_TIME_IN_MILLISECS;
         if ((now - createTime) > MAX_VALID_SESSION_TIME_IN_MILLISECS) {
             logger.debug("Session expired for the Gateway " + apiURI.getHost() + ", token is invalid after " + MAX_VALID_SESSION_TIME_IN_HRS
@@ -281,7 +284,11 @@ private <T> T get(final String path, final Class<T> type, final boolean renewAnd
         HttpResponse response = null;
         boolean responseConsumed = false;
         try {
-            while (authenticating); // wait for authentication request (if any) to complete (and to pick the new session key)
+            while (authenticating) { // wait for authentication request (if any)
+                // to complete (and to pick the new session key)
+                Thread.yield();
+            }
+
             final HttpGet request = new HttpGet(apiURI.toString() + path);
             request.setHeader(HttpHeaders.AUTHORIZATION, "Basic " + Base64.getEncoder().encodeToString((this.username + ":" + this.sessionKey).getBytes()));
             logger.debug("Sending GET request: " + request.toString());
@@ -316,7 +323,10 @@ private <T> T post(final String path, final Object obj, final Class<T> type, fin
         HttpResponse response = null;
         boolean responseConsumed = false;
         try {
-            while (authenticating); // wait for authentication request (if any) to complete (and to pick the new session key)
+            while (authenticating) { // wait for authentication request (if any)
+                // to complete (and to pick the new session key)
+                Thread.yield();
+            }
             final HttpPost request = new HttpPost(apiURI.toString() + path);
             request.setHeader(HttpHeaders.AUTHORIZATION, "Basic " + Base64.getEncoder().encodeToString((this.username + ":" + this.sessionKey).getBytes()));
             request.setHeader("content-type", "application/json");
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java
@@ -153,15 +153,15 @@ public ScaleIOPrimaryDataStoreDriver() {
         sdcManager = new ScaleIOSDCManagerImpl();
     }
 
-    ScaleIOGatewayClient getScaleIOClient(final StoragePool storagePool) throws Exception {
+    ScaleIOGatewayClient getScaleIOClient(final StoragePool storagePool) {
         return ScaleIOGatewayClientConnectionPool.getInstance().getClient(storagePool, storagePoolDetailsDao);
     }
 
-    ScaleIOGatewayClient getScaleIOClient(final DataStore dataStore) throws Exception {
+    ScaleIOGatewayClient getScaleIOClient(final DataStore dataStore) {
         return ScaleIOGatewayClientConnectionPool.getInstance().getClient(dataStore, storagePoolDetailsDao);
     }
 
-    private boolean setVolumeLimitsOnSDC(VolumeVO volume, Host host, DataStore dataStore, Long iopsLimit, Long bandwidthLimitInKbps) throws Exception {
+    private boolean setVolumeLimitsOnSDC(VolumeVO volume, Host host, DataStore dataStore, Long iopsLimit, Long bandwidthLimitInKbps) {
         sdcManager = ComponentContext.inject(sdcManager);
         final String sdcId = sdcManager.prepareSDC(host, dataStore);
         if (StringUtils.isBlank(sdcId)) {
@@ -173,7 +173,7 @@ private boolean setVolumeLimitsOnSDC(VolumeVO volume, Host host, DataStore dataS
         return client.mapVolumeToSdcWithLimits(ScaleIOUtil.getVolumePath(volume.getPath()), sdcId, iopsLimit, bandwidthLimitInKbps);
     }
 
-    private boolean setVolumeLimitsFromDetails(VolumeVO volume, Host host, DataStore dataStore) throws Exception {
+    private boolean setVolumeLimitsFromDetails(VolumeVO volume, Host host, DataStore dataStore) {
         Long bandwidthLimitInKbps = 0L; // Unlimited
         // Check Bandwidth Limit parameter in volume details
         final VolumeDetailVO bandwidthVolumeDetail = volumeDetailsDao.findDetail(volume.getId(), Volume.BANDWIDTH_LIMIT_IN_MBPS);
@@ -997,7 +997,7 @@ private Host findEndpointForVolumeOperation(DataObject srcData) {
         return host;
     }
 
-    public void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destData) throws Exception {
+    public void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destData) {
         final long srcVolumeId = srcData.getId();
         DataStore srcStore = srcData.getDataStore();
         final ScaleIOGatewayClient client = getScaleIOClient(srcStore);
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
diff --git a/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImplTest.java b/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImplTest.java
