Linstor: fix live migrate on non-hyperconverged setups

In non-hyperconverged setups, diskless nodes don't have a connection
to each other, so setting properties there had no effect.
Now it is checked if a connection exists,
between the live migration nodes and if not,
it will set the allow-two-primaries on resource-definition level.

Author: rp-

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java

@@ -21,6 +21,7 @@
 import java.io.InputStreamReader;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -53,6 +54,7 @@
 import com.linbit.linstor.api.model.ResourceConnectionModify;
 import com.linbit.linstor.api.model.ResourceDefinition;
 import com.linbit.linstor.api.model.ResourceGroup;
+import com.linbit.linstor.api.model.ResourceDefinitionModify;
 import com.linbit.linstor.api.model.ResourceGroupSpawn;
 import com.linbit.linstor.api.model.ResourceMakeAvailable;
 import com.linbit.linstor.api.model.ResourceWithVolumes;
@@ -268,17 +270,33 @@ private void allow2PrimariesIfInUse(DevelopersApi api, String rscName) throws Ap
         String inUseNode = LinstorUtil.isResourceInUse(api, rscName);
         if (inUseNode != null && !inUseNode.equalsIgnoreCase(localNodeName)) {
             // allow 2 primaries for live migration, should be removed by disconnect on the other end
-            ResourceConnectionModify rcm = new ResourceConnectionModify();
-            Properties props = new Properties();
-            props.put("DrbdOptions/Net/allow-two-primaries", "yes");
-            props.put("DrbdOptions/Net/protocol", "C");
-            rcm.setOverrideProps(props);
-            ApiCallRcList answers = api.resourceConnectionModify(rscName, inUseNode, localNodeName, rcm);
-            if (answers.hasError()) {
-                s_logger.error(String.format(
-                        "Unable to set protocol C and 'allow-two-primaries' on %s/%s/%s",
-                        inUseNode, localNodeName, rscName));
-                // do not fail here as adding allow-two-primaries property is only a problem while live migrating
+
+            // if non hyperconverged setup, we have to set allow-two-primaries on the resource-definition
+            // as there is no resource connection between diskless nodes.
+            if (LinstorUtil.areResourcesDiskless(api, rscName, Arrays.asList(inUseNode, localNodeName))) {
+                ResourceDefinitionModify rdm = new ResourceDefinitionModify();
+                Properties props = new Properties();
+                props.put("DrbdOptions/Net/allow-two-primaries", "yes");
+                props.put("DrbdOptions/Net/protocol", "C");
+                rdm.setOverrideProps(props);
+                ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);
+                if (answers.hasError()) {
+                    s_logger.error(String.format("Unable to set protocol C and 'allow-two-primaries' on %s", rscName));
+                    // do not fail here as adding allow-two-primaries property is only a problem while live migrating
+                }
+            } else {
+                ResourceConnectionModify rcm = new ResourceConnectionModify();
+                Properties props = new Properties();
+                props.put("DrbdOptions/Net/allow-two-primaries", "yes");
+                props.put("DrbdOptions/Net/protocol", "C");
+                rcm.setOverrideProps(props);
+                ApiCallRcList answers = api.resourceConnectionModify(rscName, inUseNode, localNodeName, rcm);
+                if (answers.hasError()) {
+                    s_logger.error(String.format(
+                            "Unable to set protocol C and 'allow-two-primaries' on %s/%s/%s",
+                            inUseNode, localNodeName, rscName));
+                    // do not fail here as adding allow-two-primaries property is only a problem while live migrating
+                }
             }
         }
     }
@@ -318,19 +336,34 @@ public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool, Map<S
     }
 
     private void removeTwoPrimariesRcProps(DevelopersApi api, String inUseNode, String rscName) throws ApiException {
-        ResourceConnectionModify rcm = new ResourceConnectionModify();
         List<String> deleteProps = new ArrayList<>();
         deleteProps.add("DrbdOptions/Net/allow-two-primaries");
         deleteProps.add("DrbdOptions/Net/protocol");
-        rcm.deleteProps(deleteProps);
-        ApiCallRcList answers = api.resourceConnectionModify(rscName, localNodeName, inUseNode, rcm);
-        if (answers.hasError()) {
-            s_logger.error(
-                    String.format("Failed to remove 'protocol' and 'allow-two-primaries' on %s/%s/%s: %s",
-                            localNodeName,
-                            inUseNode,
-                            rscName, LinstorUtil.getBestErrorMessage(answers)));
-            // do not fail here as removing allow-two-primaries property isn't fatal
+
+        {
+            ResourceDefinitionModify rdm = new ResourceDefinitionModify();
+            rdm.deleteProps(deleteProps);
+            ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);
+            if (answers.hasError()) {
+                s_logger.error(
+                        String.format("Failed to remove 'protocol' and 'allow-two-primaries' on %s: %s",
+                                rscName, LinstorUtil.getBestErrorMessage(answers)));
+                // do not fail here as removing allow-two-primaries property isn't fatal
+            }
+        }
+
+        {
+            ResourceConnectionModify rcm = new ResourceConnectionModify();
+            rcm.deleteProps(deleteProps);
+            ApiCallRcList answers = api.resourceConnectionModify(rscName, localNodeName, inUseNode, rcm);
+            if (answers.hasError()) {
+                s_logger.error(
+                        String.format("Failed to remove 'protocol' and 'allow-two-primaries' on %s/%s/%s: %s",
+                                localNodeName,
+                                inUseNode,
+                                rscName, LinstorUtil.getBestErrorMessage(answers)));
+                // do not fail here as removing allow-two-primaries property isn't fatal
+            }
         }
     }
 

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java

@@ -17,6 +17,7 @@
 package org.apache.cloudstack.storage.datastore.util;
 
 import com.linbit.linstor.api.ApiClient;
+import com.linbit.linstor.api.ApiConsts;
 import com.linbit.linstor.api.ApiException;
 import com.linbit.linstor.api.DevelopersApi;
 import com.linbit.linstor.api.model.ApiCallRc;
@@ -27,8 +28,10 @@
 import com.linbit.linstor.api.model.ResourceWithVolumes;
 import com.linbit.linstor.api.model.StoragePool;
 
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.log4j.Logger;
@@ -113,6 +116,28 @@ public static String isResourceInUse(DevelopersApi api, String rscName) throws A
         return null;
     }
 
+    /**
+     * Check if the given resources are diskless.
+     *
+     * @param api developer api object to use
+     * @param rscName resource name to check in use state.
+     * @return NodeName where the resource is inUse, if not in use `null`
+     * @throws ApiException forwards api errors
+     */
+    public static boolean areResourcesDiskless(DevelopersApi api, String rscName, Collection<String> nodeNames)
+            throws ApiException {
+        List<Resource> rscs = api.resourceList(rscName, null, null);
+        if (rscs != null) {
+            Collection<String> disklessNodes = rscs.stream()
+                .filter(rsc -> rsc.getFlags() != null && (rsc.getFlags().contains(ApiConsts.FLAG_DISKLESS) ||
+                        rsc.getFlags().contains(ApiConsts.FLAG_DRBD_DISKLESS)))
+                    .map(rsc -> rsc.getNodeName().toLowerCase())
+                    .collect(Collectors.toList());
+            return disklessNodes.containsAll(nodeNames.stream().map(String::toLowerCase).collect(Collectors.toList()));
+        }
+        return false;
+    }
+
     /**
      * Try to get the device path for the given resource name.
      * This could be made a bit more direct after java-linstor api is fixed for layer data subtypes.