[PowerFlex] Add & Remove PowerFlex/ScaleIO MDMs for the storage SDC connections (#9903)

* Add & Remove PowerFlex/ScaleIO MDMs while preparing & unpreparing the storage SDC connections (instead of start & stop scini)

* Add/Remove MDM IP addresses during Host connection/disconnection to/from storage pool when powerflex.connect.on.demand is false

* unit test fixes

* Don't remove MDM IPs from SDC when any volumes mapped to SDC

* Don't remove MDM IPs when other pools of same ScaleIO/PowerFlex cluster are connected

* rebase fixes

* update changes, to not remove/disconnect MDMs on maintenance

* import fixes after rebase

Author: sureshanaparti

core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java

@@ -46,6 +46,10 @@ public ModifyStoragePoolAnswer(ModifyStoragePoolCommand cmd, long capacityBytes,
         templateInfo = tInfo;
     }
 
+    public ModifyStoragePoolAnswer(ModifyStoragePoolCommand cmd, boolean success, String details) {
+        super(cmd, success, details);
+    }
+
     public void setPoolInfo(StoragePoolInfo poolInfo) {
         this.poolInfo = poolInfo;
     }

core/src/main/java/com/cloud/agent/api/UnprepareStorageClientCommand.java

@@ -19,18 +19,22 @@
 
 package com.cloud.agent.api;
 
+import java.util.Map;
+
 import com.cloud.storage.Storage.StoragePoolType;
 
 public class UnprepareStorageClientCommand extends Command {
     private StoragePoolType poolType;
     private String poolUuid;
+    private Map<String, String> details;
 
     public UnprepareStorageClientCommand() {
     }
 
-    public UnprepareStorageClientCommand(StoragePoolType poolType, String poolUuid) {
+    public UnprepareStorageClientCommand(StoragePoolType poolType, String poolUuid, Map<String, String> details) {
         this.poolType = poolType;
         this.poolUuid = poolUuid;
+        this.details = details;
     }
 
     @Override
@@ -45,4 +49,8 @@ public StoragePoolType getPoolType() {
     public String getPoolUuid() {
         return poolUuid;
     }
+
+    public Map<String, String> getDetails() {
+        return details;
+    }
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreDriver.java

@@ -45,4 +45,8 @@ public interface DataStoreDriver {
     boolean canCopy(DataObject srcData, DataObject destData);
 
     void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> callback);
+
+    default boolean canDisplayDetails() {
+        return true;
+    }
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java

@@ -145,6 +145,14 @@ default boolean canHostPrepareStoragePoolAccess(Host host, StoragePool pool) {
         return false;
     }
 
+    /**
+     * intended for managed storage
+     * returns true if the host can be disconnected from storage pool
+     */
+    default boolean canDisconnectHostFromStoragePool(Host host, StoragePool pool) {
+        return true;
+    }
+
     /**
      * Used by storage pools which want to keep VMs' information
      * @return true if additional VM info is needed (intended for storage pools).

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -314,6 +314,8 @@ static Boolean getFullCloneConfiguration(Long storeId) {
 
     boolean canHostPrepareStoragePoolAccess(Host host, StoragePool pool);
 
+    boolean canDisconnectHostFromStoragePool(Host host, StoragePool pool);
+
     Host getHost(long hostId);
 
     Host updateSecondaryStorage(long secStorageId, String newUrl);

engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDao.java

@@ -30,6 +30,8 @@ public interface StoragePoolHostDao extends GenericDao<StoragePoolHostVO, Long>
 
     public StoragePoolHostVO findByPoolHost(long poolId, long hostId);
 
+    List<StoragePoolHostVO> findByLocalPath(String path);
+
     List<StoragePoolHostVO> listByHostStatus(long poolId, Status hostStatus);
 
     List<Long> findHostsConnectedToPools(List<Long> poolIds);

engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDaoImpl.java

@@ -45,6 +45,7 @@ public class StoragePoolHostDaoImpl extends GenericDaoBase<StoragePoolHostVO, Lo
     protected final SearchBuilder<StoragePoolHostVO> PoolSearch;
     protected final SearchBuilder<StoragePoolHostVO> HostSearch;
     protected final SearchBuilder<StoragePoolHostVO> PoolHostSearch;
+    protected final SearchBuilder<StoragePoolHostVO> LocalPathSearch;
 
     protected SearchBuilder<StoragePoolHostVO> poolNotInClusterSearch;
 
@@ -77,6 +78,9 @@ public StoragePoolHostDaoImpl() {
         PoolHostSearch.and("host_id", PoolHostSearch.entity().getHostId(), SearchCriteria.Op.EQ);
         PoolHostSearch.done();
 
+        LocalPathSearch = createSearchBuilder();
+        LocalPathSearch.and("local_path", LocalPathSearch.entity().getLocalPath(), SearchCriteria.Op.EQ);
+        LocalPathSearch.done();
     }
 
     @PostConstruct
@@ -117,6 +121,13 @@ public StoragePoolHostVO findByPoolHost(long poolId, long hostId) {
         return findOneIncludingRemovedBy(sc);
     }
 
+    @Override
+    public List<StoragePoolHostVO> findByLocalPath(String path) {
+        SearchCriteria<StoragePoolHostVO> sc = LocalPathSearch.create();
+        sc.setParameters("local_path", path);
+        return listBy(sc);
+    }
+
     @Override
     public List<StoragePoolHostVO> listByHostStatus(long poolId, Status hostStatus) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java

@@ -60,6 +60,8 @@ public interface PrimaryDataStoreDao extends GenericDao<StoragePoolVO, Long> {
 
     StoragePoolVO persist(StoragePoolVO pool, Map<String, String> details, List<String> tags, Boolean isTagARule);
 
+    StoragePoolVO persist(StoragePoolVO pool, Map<String, String> details, List<String> tags, Boolean isTagARule, boolean displayDetails);
+
     /**
      * Find pool by name.
      *
@@ -103,6 +105,8 @@ public interface PrimaryDataStoreDao extends GenericDao<StoragePoolVO, Long> {
 
     void updateDetails(long poolId, Map<String, String> details);
 
+    void removeDetails(long poolId);
+
     Map<String, String> getDetails(long poolId);
 
     List<String> searchForStoragePoolTags(long poolId);

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java

@@ -296,14 +296,19 @@ public StoragePoolVO listById(Integer id) {
     }
 
     @Override
-    @DB
     public StoragePoolVO persist(StoragePoolVO pool, Map<String, String> details, List<String> tags, Boolean isTagARule) {
+        return persist(pool, details, tags, isTagARule, true);
+    }
+
+    @Override
+    @DB
+    public StoragePoolVO persist(StoragePoolVO pool, Map<String, String> details, List<String> tags, Boolean isTagARule, boolean displayDetails) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();
         txn.start();
         pool = super.persist(pool);
         if (details != null) {
             for (Map.Entry<String, String> detail : details.entrySet()) {
-                StoragePoolDetailVO vo = new StoragePoolDetailVO(pool.getId(), detail.getKey(), detail.getValue(), true);
+                StoragePoolDetailVO vo = new StoragePoolDetailVO(pool.getId(), detail.getKey(), detail.getValue(), displayDetails);
                 _detailsDao.persist(vo);
             }
         }
@@ -570,6 +575,11 @@ public void updateDetails(long poolId, Map<String, String> details) {
         }
     }
 
+    @Override
+    public void removeDetails(long poolId) {
+        _detailsDao.removeDetails(poolId);
+    }
+
     @Override
     public Map<String, String> getDetails(long poolId) {
         return _detailsDao.listDetailsKeyPairs(poolId);

engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java

@@ -34,6 +34,7 @@
 import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
 import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClientConnectionPool;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil;
@@ -301,7 +302,11 @@ public boolean revertVMSnapshot(VMSnapshot vmSnapshot) {
                 srcSnapshotDestVolumeMap.put(srcSnapshotVolumeId, destVolumeId);
             }
 
-            String systemId = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID).getValue();
+            String systemId = null;
+            StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
+            if (systemIdDetail != null) {
+                systemId = systemIdDetail.getValue();
+            }
             if (systemId == null) {
                 throw new CloudRuntimeException("Failed to get the system id for PowerFlex storage pool for reverting VM snapshot: " + vmSnapshot.getName());
             }
@@ -380,7 +385,11 @@ public boolean deleteVMSnapshot(VMSnapshot vmSnapshot) {
         try {
             List<VolumeObjectTO> volumeTOs = vmSnapshotHelper.getVolumeTOList(vmSnapshot.getVmId());
             StoragePoolVO storagePool = vmSnapshotHelper.getStoragePoolForVM(userVm);
-            String systemId = storagePoolDetailsDao.findDetail(storagePool.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID).getValue();
+            String systemId = null;
+            StoragePoolDetailVO systemIdDetail = storagePoolDetailsDao.findDetail(storagePool.getId(), ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
+            if (systemIdDetail != null) {
+                systemId = systemIdDetail.getValue();
+            }
             if (systemId == null) {
                 throw new CloudRuntimeException("Failed to get the system id for PowerFlex storage pool for deleting VM snapshot: " + vmSnapshot.getName());
             }