Linstor fix host picking (#12047)

Author: rp-

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java

@@ -63,6 +63,8 @@
 import com.cloud.api.storage.LinstorRevertBackupSnapshotCommand;
 import com.cloud.configuration.Config;
 import com.cloud.host.Host;
+import com.cloud.host.HostVO;
+import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
 import com.cloud.resource.ResourceState;
 import com.cloud.storage.DataStoreRole;
@@ -921,9 +923,10 @@ private String revertSnapshotFromImageStore(
             _backupsnapshotwait,
             VirtualMachineManager.ExecuteInSequence.value());
 
-        Optional<RemoteHostEndPoint> optEP = getDiskfullEP(linstorApi, rscName);
+        final StoragePool pool = (StoragePool) volumeInfo.getDataStore();
+        Optional<RemoteHostEndPoint> optEP = getDiskfullEP(linstorApi, pool, rscName);
         if (optEP.isEmpty()) {
-            optEP = getLinstorEP(linstorApi, rscName);
+            optEP = getLinstorEP(linstorApi, pool, rscName);
         }
 
         if (optEP.isPresent()) {
@@ -1063,13 +1066,29 @@ public void copyAsync(DataObject srcData, DataObject dstData, AsyncCompletionCal
             Answer answer = copyVolume(srcData, dstData);
             res = new CopyCommandResult(null, answer);
         } else {
-            Answer answer = new Answer(null, false, "noimpl");
-            res = new CopyCommandResult(null, answer);
-            res.setResult("Not implemented yet");
+            throw new CloudRuntimeException("Not implemented for Linstor primary storage.");
         }
         callback.complete(res);
     }
 
+    private Host getEnabledClusterHost(StoragePool storagePool, List<String> linstorNodeNames) {
+        List<HostVO> csHosts;
+        if (storagePool.getClusterId() != null) {
+            csHosts = _hostDao.findByClusterId(storagePool.getClusterId());
+        } else {
+            csHosts = _hostDao.findByDataCenterId(storagePool.getDataCenterId());
+        }
+        Collections.shuffle(csHosts); // so we do not always pick the same host for operations
+        for (HostVO host : csHosts) {
+            if (host.getResourceState() == ResourceState.Enabled &&
+                    host.getStatus() == Status.Up &&
+                    linstorNodeNames.contains(host.getName())) {
+                return host;
+            }
+        }
+        return null;
+    }
+
     /**
      * Tries to get a Linstor cloudstack end point, that is at least diskless.
      *
@@ -1078,47 +1097,37 @@ public void copyAsync(DataObject srcData, DataObject dstData, AsyncCompletionCal
      * @return Optional RemoteHostEndPoint if one could get found.
      * @throws ApiException
      */
-    private Optional<RemoteHostEndPoint> getLinstorEP(DevelopersApi api, String rscName) throws ApiException {
+    private Optional<RemoteHostEndPoint> getLinstorEP(DevelopersApi api, StoragePool storagePool, String rscName)
+            throws ApiException {
         List<String> linstorNodeNames = LinstorUtil.getLinstorNodeNames(api);
-        Collections.shuffle(linstorNodeNames);  // do not always pick the first linstor node
-
-        Host host = null;
-        for (String nodeName : linstorNodeNames) {
-            host = _hostDao.findByName(nodeName);
-            if (host != null && host.getResourceState() == ResourceState.Enabled) {
-                logger.info(String.format("Linstor: Make resource %s available on node %s ...", rscName, nodeName));
-                ApiCallRcList answers = api.resourceMakeAvailableOnNode(rscName, nodeName, new ResourceMakeAvailable());
-                if (!answers.hasError()) {
-                    break; // found working host
-                } else {
-                    logger.error(
-                        String.format("Linstor: Unable to make resource %s on node %s available: %s",
-                            rscName,
-                            nodeName,
-                            LinstorUtil.getBestErrorMessage(answers)));
-                }
+        Host host = getEnabledClusterHost(storagePool, linstorNodeNames);
+        if (host != null) {
+            logger.info("Linstor: Make resource {} available on node {} ...", rscName, host.getName());
+            ApiCallRcList answers = api.resourceMakeAvailableOnNode(
+                    rscName, host.getName(), new ResourceMakeAvailable());
+            if (answers.hasError()) {
+                logger.error("Linstor: Unable to make resource {} on node {} available: {}",
+                        rscName, host.getName(), LinstorUtil.getBestErrorMessage(answers));
+                return Optional.empty();
+            } else {
+                return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
             }
         }
 
-        if (host == null)
-        {
-            logger.error("Linstor: Couldn't create a resource on any cloudstack host.");
-            return Optional.empty();
-        }
-        else
-        {
-            return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
-        }
+        logger.error("Linstor: Couldn't create a resource on any cloudstack host.");
+        return Optional.empty();
     }
 
-    private Optional<RemoteHostEndPoint> getDiskfullEP(DevelopersApi api, String rscName) throws ApiException {
+    private Optional<RemoteHostEndPoint> getDiskfullEP(DevelopersApi api, StoragePool storagePool, String rscName)
+            throws ApiException {
         List<com.linbit.linstor.api.model.StoragePool> linSPs = LinstorUtil.getDiskfulStoragePools(api, rscName);
         if (linSPs != null) {
-            for (com.linbit.linstor.api.model.StoragePool sp : linSPs) {
-                Host host = _hostDao.findByName(sp.getNodeName());
-                if (host != null && host.getResourceState() == ResourceState.Enabled) {
-                    return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
-                }
+            List<String> linstorNodeNames = linSPs.stream()
+                    .map(com.linbit.linstor.api.model.StoragePool::getNodeName)
+                    .collect(Collectors.toList());
+            Host host = getEnabledClusterHost(storagePool, linstorNodeNames);
+            if (host != null) {
+                return Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
             }
         }
         logger.error("Linstor: No diskfull host found.");
@@ -1199,12 +1208,12 @@ private Answer copyTemplate(DataObject srcData, DataObject dstData) {
                     VirtualMachineManager.ExecuteInSequence.value());
 
             try {
-                Optional<RemoteHostEndPoint> optEP = getLinstorEP(api, rscName);
+                Optional<RemoteHostEndPoint> optEP = getLinstorEP(api, pool, rscName);
                 if (optEP.isPresent()) {
                     answer = optEP.get().sendMessage(cmd);
                 } else {
-                    answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint.");
                     deleteResourceDefinition(pool, rscName);
+                    throw new CloudRuntimeException("Unable to get matching Linstor endpoint.");
                 }
             } catch (ApiException exc) {
                 logger.error("copy template failed: ", exc);
@@ -1241,12 +1250,12 @@ private Answer copyVolume(DataObject srcData, DataObject dstData) {
         Answer answer;
 
         try {
-            Optional<RemoteHostEndPoint> optEP = getLinstorEP(api, rscName);
+            Optional<RemoteHostEndPoint> optEP = getLinstorEP(api, pool, rscName);
             if (optEP.isPresent()) {
                 answer = optEP.get().sendMessage(cmd);
             }
             else {
-                answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint.");
+                throw new CloudRuntimeException("Unable to get matching Linstor endpoint.");
             }
         } catch (ApiException exc) {
             logger.error("copy volume failed: ", exc);
@@ -1279,14 +1288,14 @@ private Answer copyFromTemporaryResource(
         try {
             String devName = restoreResourceFromSnapshot(api, pool, rscName, snapshotName, restoreName);
 
-            Optional<RemoteHostEndPoint> optEPAny = getLinstorEP(api, restoreName);
+            Optional<RemoteHostEndPoint> optEPAny = getLinstorEP(api, pool, restoreName);
             if (optEPAny.isPresent()) {
                 // patch the src device path to the temporary linstor resource
                 snapshotObject.setPath(devName);
                 origCmd.setSrcTO(snapshotObject.getTO());
                 answer = optEPAny.get().sendMessage(origCmd);
-            } else{
-                answer = new Answer(origCmd, false, "Unable to get matching Linstor endpoint.");
+            } else {
+                throw new CloudRuntimeException("Unable to get matching Linstor endpoint.");
             }
         } finally {
             // delete the temporary resource, noop if already gone
@@ -1348,7 +1357,7 @@ protected Answer copySnapshot(DataObject srcData, DataObject destData) {
                 VirtualMachineManager.ExecuteInSequence.value());
             cmd.setOptions(options);
 
-            Optional<RemoteHostEndPoint> optEP = getDiskfullEP(api, rscName);
+            Optional<RemoteHostEndPoint> optEP = getDiskfullEP(api, pool, rscName);
             Answer answer;
             if (optEP.isPresent()) {
                 answer = optEP.get().sendMessage(cmd);