Updated Endpoint Selector to pick the Cluster in Enabled state (in addition to Host state) (#10757)

* Consider the clusters with allocation state 'Enabled' for EndPoint selection (in addition to Host state)

* Reset the pool id when create volume fails on the allocated pool

- the pool id is persisted while creating the volume, when it fails the pool id is not reverted. On next create volume attempt, CloudStack couldn't find any suitable primary storage even there are pools available with enough capacity as the pool is already assigned to volume which is in Allocated state (and storage pool compatibility check fails). Ensure volume is not assigned to any pool if create volume fails (so the next creation job would pick the suitable pool).

* endpoint check for resize

* update the resize error through callback result instead of exception

* logger fix

Author: sureshanaparti

engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java

@@ -80,7 +80,7 @@ public class DefaultEndPointSelector implements EndPointSelector {
     private final String findOneHostOnPrimaryStorage = "select t.id from "
                             + "(select h.id, cd.value, hd.value as " + VOL_ENCRYPT_COLUMN_NAME + " "
                             + "from host h join storage_pool_host_ref s on h.id = s.host_id  "
-                            + "join cluster c on c.id=h.cluster_id "
+                            + "join cluster c on c.id=h.cluster_id and c.allocation_state = 'Enabled'"
                             + "left join cluster_details cd on c.id=cd.cluster_id and cd.name='" + CapacityManager.StorageOperationsExcludeCluster.key() + "' "
                             + "left join host_details hd on h.id=hd.host_id and hd.name='" + HOST_VOLUME_ENCRYPTION + "' "
                             + "where h.status = 'Up' and h.type = 'Routing' and h.resource_state = 'Enabled' and s.pool_id = ? ";

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java

@@ -328,6 +328,11 @@ protected Void createVolumeCallback(AsyncCallbackDispatcher<VolumeServiceImpl, C
         } else {
             vo.processEvent(Event.OperationFailed);
             errMsg = result.getResult();
+            VolumeVO volume = volDao.findById(vo.getId());
+            if (volume != null && volume.getState() == State.Allocated && volume.getPodId() != null) {
+                volume.setPoolId(null);
+                volDao.update(volume.getId(), volume);
+            }
         }
         VolumeApiResult volResult = new VolumeApiResult((VolumeObject)vo);
         if (errMsg != null) {
@@ -1237,6 +1242,10 @@ private void destroyAndReallocateManagedVolume(VolumeInfo volumeInfo) {
         }
 
         if (volume.getState() == State.Allocated) { // Possible states here: Allocated, Ready & Creating
+            if (volume.getPodId() != null) {
+                volume.setPoolId(null);
+                volDao.update(volume.getId(), volume);
+            }
             return;
         }
 
@@ -2476,7 +2485,7 @@ public AsyncCallFuture<VolumeApiResult> resize(VolumeInfo volume) {
         try {
             volume.processEvent(Event.ResizeRequested);
         } catch (Exception e) {
-            logger.debug("Failed to change state to resize", e);
+            logger.debug("Failed to change volume state to resize", e);
             result.setResult(e.toString());
             future.complete(result);
             return future;
@@ -2488,10 +2497,8 @@ public AsyncCallFuture<VolumeApiResult> resize(VolumeInfo volume) {
         try {
             volume.getDataStore().getDriver().resize(volume, caller);
         } catch (Exception e) {
-            logger.debug("Failed to change state to resize", e);
-
+            logger.debug("Failed to resize volume", e);
             result.setResult(e.toString());
-
             future.complete(result);
         }
 
@@ -2535,7 +2542,7 @@ protected Void resizeVolumeCallback(AsyncCallbackDispatcher<VolumeServiceImpl, C
             try {
                 volume.processEvent(Event.OperationFailed);
             } catch (Exception e) {
-                logger.debug("Failed to change state", e);
+                logger.debug("Failed to change volume state (after resize failure)", e);
             }
             VolumeApiResult res = new VolumeApiResult(volume);
             res.setResult(result.getResult());
@@ -2546,13 +2553,8 @@ protected Void resizeVolumeCallback(AsyncCallbackDispatcher<VolumeServiceImpl, C
         try {
             volume.processEvent(Event.OperationSuccessed);
         } catch (Exception e) {
-            logger.debug("Failed to change state", e);
-            VolumeApiResult res = new VolumeApiResult(volume);
-            res.setResult(result.getResult());
-            future.complete(res);
-            return null;
+            logger.debug("Failed to change volume state (after resize success)", e);
         }
-
         VolumeApiResult res = new VolumeApiResult(volume);
         future.complete(res);
 

plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java

@@ -432,17 +432,25 @@ public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> cal
         boolean encryptionRequired = anyVolumeRequiresEncryption(vol);
         long [] endpointsToRunResize = resizeParameter.hosts;
 
+        CreateCmdResult result = new CreateCmdResult(null, null);
+
         // if hosts are provided, they are where the VM last ran. We can use that.
         if (endpointsToRunResize == null || endpointsToRunResize.length == 0) {
             EndPoint ep = epSelector.select(data, encryptionRequired);
+            if (ep == null) {
+                String errMsg = String.format(NO_REMOTE_ENDPOINT_WITH_ENCRYPTION, encryptionRequired);
+                logger.error(errMsg);
+                result.setResult(errMsg);
+                callback.complete(result);
+                return;
+            }
             endpointsToRunResize = new long[] {ep.getId()};
         }
         ResizeVolumeCommand resizeCmd = new ResizeVolumeCommand(vol.getPath(), new StorageFilerTO(pool), vol.getSize(),
                 resizeParameter.newSize, resizeParameter.shrinkOk, resizeParameter.instanceName, vol.getChainInfo(), vol.getPassphrase(), vol.getEncryptFormat());
         if (pool.getParent() != 0) {
             resizeCmd.setContextParam(DiskTO.PROTOCOL_TYPE, Storage.StoragePoolType.DatastoreCluster.toString());
         }
-        CreateCmdResult result = new CreateCmdResult(null, null);
         try {
             ResizeVolumeAnswer answer = (ResizeVolumeAnswer) storageMgr.sendToPool(pool, endpointsToRunResize, resizeCmd);
             if (answer != null && answer.getResult()) {
@@ -459,7 +467,6 @@ public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> cal
                 logger.debug("return a null answer, mark it as failed for unknown reason");
                 result.setResult("return a null answer, mark it as failed for unknown reason");
             }
-
         } catch (Exception e) {
             logger.debug("sending resize command failed", e);
             result.setResult(e.toString());