linstor: set/unset allow-two-primaries and protocol on rc level (#9560)

Author: rp-

plugins/storage/volume/linstor/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+
+All notable changes to Linstor CloudStack plugin will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [2024-08-27]
+
+### Changed
+
+- Allow two primaries(+protocol c) is now set on resource-connection level instead of rd

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java

@@ -19,6 +19,8 @@
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
+
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -37,6 +39,7 @@
 
 import com.cloud.storage.Storage;
 import com.cloud.utils.exception.CloudRuntimeException;
+
 import com.linbit.linstor.api.ApiClient;
 import com.linbit.linstor.api.ApiConsts;
 import com.linbit.linstor.api.ApiException;
@@ -47,8 +50,8 @@
 import com.linbit.linstor.api.model.Properties;
 import com.linbit.linstor.api.model.ProviderKind;
 import com.linbit.linstor.api.model.Resource;
+import com.linbit.linstor.api.model.ResourceConnectionModify;
 import com.linbit.linstor.api.model.ResourceDefinition;
-import com.linbit.linstor.api.model.ResourceDefinitionModify;
 import com.linbit.linstor.api.model.ResourceGroup;
 import com.linbit.linstor.api.model.ResourceGroupSpawn;
 import com.linbit.linstor.api.model.ResourceMakeAvailable;
@@ -262,15 +265,19 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, Qemu
      * @throws ApiException if any problem connecting to the Linstor controller
      */
     private void allow2PrimariesIfInUse(DevelopersApi api, String rscName) throws ApiException {
-        if (LinstorUtil.isResourceInUse(api, rscName)) {
+        String inUseNode = LinstorUtil.isResourceInUse(api, rscName);
+        if (inUseNode != null && !inUseNode.equalsIgnoreCase(localNodeName)) {
             // allow 2 primaries for live migration, should be removed by disconnect on the other end
-            ResourceDefinitionModify rdm = new ResourceDefinitionModify();
+            ResourceConnectionModify rcm = new ResourceConnectionModify();
             Properties props = new Properties();
             props.put("DrbdOptions/Net/allow-two-primaries", "yes");
-            rdm.setOverrideProps(props);
-            ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm);
+            props.put("DrbdOptions/Net/protocol", "C");
+            rcm.setOverrideProps(props);
+            ApiCallRcList answers = api.resourceConnectionModify(rscName, inUseNode, localNodeName, rcm);
             if (answers.hasError()) {
-                s_logger.error("Unable to set 'allow-two-primaries' on " + rscName);
+                s_logger.error(String.format(
+                        "Unable to set protocol C and 'allow-two-primaries' on %s/%s/%s",
+                        inUseNode, localNodeName, rscName));
                 // do not fail here as adding allow-two-primaries property is only a problem while live migrating
             }
         }
@@ -310,6 +317,23 @@ public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool, Map<S
         return true;
     }
 
+    private void removeTwoPrimariesRcProps(DevelopersApi api, String inUseNode, String rscName) throws ApiException {
+        ResourceConnectionModify rcm = new ResourceConnectionModify();
+        List<String> deleteProps = new ArrayList<>();
+        deleteProps.add("DrbdOptions/Net/allow-two-primaries");
+        deleteProps.add("DrbdOptions/Net/protocol");
+        rcm.deleteProps(deleteProps);
+        ApiCallRcList answers = api.resourceConnectionModify(rscName, localNodeName, inUseNode, rcm);
+        if (answers.hasError()) {
+            s_logger.error(
+                    String.format("Failed to remove 'protocol' and 'allow-two-primaries' on %s/%s/%s: %s",
+                            localNodeName,
+                            inUseNode,
+                            rscName, LinstorUtil.getBestErrorMessage(answers)));
+            // do not fail here as removing allow-two-primaries property isn't fatal
+        }
+    }
+
     private boolean tryDisconnectLinstor(String volumePath, KVMStoragePool pool)
     {
         if (volumePath == null) {
@@ -339,27 +363,25 @@ private boolean tryDisconnectLinstor(String volumePath, KVMStoragePool pool)
 
 
         if (optRsc.isPresent()) {
+            Resource rsc = optRsc.get();
             try {
-                Resource rsc = optRsc.get();
+                String inUseNode = LinstorUtil.isResourceInUse(api, rsc.getName());
+                if (inUseNode != null && !inUseNode.equalsIgnoreCase(localNodeName)) {
+                    removeTwoPrimariesRcProps(api, inUseNode, rsc.getName());
+                }
+            } catch (ApiException apiEx) {
+                s_logger.error(apiEx.getBestMessage());
+                // do not fail here as removing allow-two-primaries property or deleting diskless isn't fatal
+            }
 
+            try {
                 // if diskless resource remove it, in the worst case it will be transformed to a tiebreaker
                 if (rsc.getFlags() != null &&
                         rsc.getFlags().contains(ApiConsts.FLAG_DRBD_DISKLESS) &&
                         !rsc.getFlags().contains(ApiConsts.FLAG_TIE_BREAKER)) {
                     ApiCallRcList delAnswers = api.resourceDelete(rsc.getName(), localNodeName);
                     logLinstorAnswers(delAnswers);
                 }
-
-                // remove allow-two-primaries
-                ResourceDefinitionModify rdm = new ResourceDefinitionModify();
-                rdm.deleteProps(Collections.singletonList("DrbdOptions/Net/allow-two-primaries"));
-                ApiCallRcList answers = api.resourceDefinitionModify(rsc.getName(), rdm);
-                if (answers.hasError()) {
-                    s_logger.error(
-                            String.format("Failed to remove 'allow-two-primaries' on %s: %s",
-                                    rsc.getName(), LinstorUtil.getBestErrorMessage(answers)));
-                    // do not fail here as removing allow-two-primaries property isn't fatal
-                }
             } catch (ApiException apiEx) {
                 s_logger.error(apiEx.getBestMessage());
                 // do not fail here as removing allow-two-primaries property or deleting diskless isn't fatal

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java

@@ -97,17 +97,20 @@ public static long getCapacityBytes(String linstorUrl, String rscGroupName) {
      *
      * @param api developer api object to use
      * @param rscName resource name to check in use state.
-     * @return True if a resource found that is in use(primary) state, else false.
+     * @return NodeName where the resource is inUse, if not in use `null`
      * @throws ApiException forwards api errors
      */
-    public static boolean isResourceInUse(DevelopersApi api, String rscName) throws ApiException {
+    public static String isResourceInUse(DevelopersApi api, String rscName) throws ApiException {
         List<Resource> rscs = api.resourceList(rscName, null, null);
         if (rscs != null) {
             return rscs.stream()
-                    .anyMatch(rsc -> rsc.getState() != null && Boolean.TRUE.equals(rsc.getState().isInUse()));
-        }
+                    .filter(rsc -> rsc.getState() != null && Boolean.TRUE.equals(rsc.getState().isInUse()))
+                    .map(Resource::getNodeName)
+                    .findFirst()
+                    .orElse(null);
+       }
         s_logger.error("isResourceInUse: null returned from resourceList");
-        return false;
+        return null;
     }
 
     /**