Merge branch 'instance-from-backup' of https://github.com/shapeblue/cloudstack into instance

Author: abh1sar

.asf.yaml

@@ -54,10 +54,10 @@ github:
     - gpordeus
     - hsato03
     - bernardodemarco
-    - abh1sar
     - FelipeM525
     - lucas-a-martins
     - nicoschmdt
+    - abh1sar
 
   protected_branches: ~
 

CONTRIBUTING.md

@@ -4,9 +4,9 @@ Contributing to Apache CloudStack (ACS)
 Summary
 -------
 This document covers how to contribute to the ACS project. ACS uses GitHub PRs to manage code contributions.
-These instructions assume you have a GitHub.com account, so if you don't have one you will have to create one. Your proposed code changes will be published to your own fork of the ACS project and you will submit a Pull Request for your changes to be added.
+These instructions assume you have a GitHub.com account, so if you don't have one you will have to create one. Your proposed code changes will be published to your own fork of the ACS project, and you will submit a Pull Request for your changes to be added.
 
-_Lets get started!!!_
+_Let's get started!!!_
 
 Bug fixes
 ---------
@@ -26,7 +26,7 @@ No back porting / cherry-picking features to existing branches!
 
 PendingReleaseNotes file
 ------------------------
-When developing a new feature or making a (major) change to a existing feature you are encouraged to append this to the PendingReleaseNotes file so that the Release Manager can
+When developing a new feature or making a (major) change to an existing feature you are encouraged to append this to the PendingReleaseNotes file so that the Release Manager can
 use this file as a source of information when compiling the Release Notes for a new release.
 
 When adding information to the PendingReleaseNotes file make sure that you write a good and understandable description of the new feature or change which you have developed.
@@ -38,9 +38,9 @@ Fork the code
 
 In your browser, navigate to: [https://github.com/apache/cloudstack](https://github.com/apache/cloudstack)
 
-Fork the repository by clicking on the 'Fork' button on the top right hand side.  The fork will happen and you will be taken to your own fork of the repository.  Copy the Git repository URL by clicking on the clipboard next to the URL on the right hand side of the page under '**HTTPS** clone URL'.  You will paste this URL when doing the following `git clone` command.
+Fork the repository by clicking on the 'Fork' button on the top right hand side.  The fork will happen, and you will be taken to your own fork of the repository.  Copy the Git repository URL by clicking on the clipboard next to the URL on the right hand side of the page under '**HTTPS** clone URL'.  You will paste this URL when doing the following `git clone` command.
 
-On your computer, follow these steps to setup a local repository for working on ACS:
+On your computer, follow these steps to set up a local repository for working on ACS:
 
 ```bash
 $ git clone https://github.com/YOUR_ACCOUNT/cloudstack.git
@@ -92,9 +92,9 @@ $ git rebase main
 Make a GitHub Pull Request to contribute your changes
 -----------------------------------------------------
 
-When you are happy with your changes and you are ready to contribute them, you will create a Pull Request on GitHub to do so.  This is done by pushing your local changes to your forked repository (default remote name is `origin`) and then initiating a pull request on GitHub.
+When you are happy with your changes, and you are ready to contribute them, you will create a Pull Request on GitHub to do so.  This is done by pushing your local changes to your forked repository (default remote name is `origin`) and then initiating a pull request on GitHub.
 
-Please include JIRA id, detailed information about the bug/feature, what all tests are executed, how the reviewer can test this feature etc. Incase of UI PRs, a screenshot is preferred.
+Please include JIRA id, detailed information about the bug/feature, what all tests are executed, how the reviewer can test this feature etc. In case of UI PRs, a screenshot is preferred.
 
 > **IMPORTANT:** Make sure you have rebased your `feature_x` branch to include the latest code from `upstream/main` _before_ you do this.
 

api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java

@@ -149,6 +149,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations {
     @Param(description = "whether this pool is managed or not")
     private Boolean managed;
 
+    @SerializedName(ApiConstants.DETAILS)
+    @Param(description = "the storage pool details")
+    private Map<String, String> details;
+
     public Map<String, String> getCaps() {
         return caps;
     }
@@ -407,4 +411,12 @@ public Boolean getManaged() {
     public void setManaged(Boolean managed) {
         this.managed = managed;
     }
+
+    public Map<String, String> getDetails() {
+        return details;
+    }
+
+    public void setDetails(Map<String, String> details) {
+        this.details = details;
+    }
 }

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java

@@ -43,7 +43,6 @@
 import com.cloud.storage.StoragePoolHostVO;
 import com.cloud.storage.StorageService;
 import com.cloud.storage.dao.StoragePoolHostDao;
-import com.cloud.utils.Pair;
 import com.cloud.utils.exception.CloudRuntimeException;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@@ -60,6 +59,7 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 public class DefaultHostListener implements HypervisorHostListener {
     protected Logger logger = LogManager.getLogger(getClass());
@@ -133,9 +133,11 @@ private NicTO createNicTOFromNetworkAndOffering(NetworkVO networkVO, NetworkOffe
     @Override
     public boolean hostConnect(long hostId, long poolId) throws StorageConflictException {
         StoragePool pool = (StoragePool) this.dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
-        Pair<Map<String, String>, Boolean> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null);
+        Map<String, String> detailsMap = storagePoolDetailsDao.listDetailsKeyPairs(poolId);
+        Map<String, String> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null).first();
 
-        ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool, nfsMountOpts.first());
+        Optional.ofNullable(nfsMountOpts).ifPresent(detailsMap::putAll);
+        ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool, detailsMap);
         cmd.setWait(modifyStoragePoolCommandWait);
         HostVO host = hostDao.findById(hostId);
         logger.debug("Sending modify storage pool command to agent: {} for storage pool: {} with timeout {} seconds", host, pool, cmd.getWait());

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreatePrivateTemplateFromVolumeCommandWrapper.java

@@ -108,9 +108,7 @@ public Answer execute(final CreatePrivateTemplateFromVolumeCommand command, fina
             } else {
                 logger.debug("Converting RBD disk " + disk.getPath() + " into template " + command.getUniqueName());
 
-                final QemuImgFile srcFile =
-                        new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary.getSourceHost(), primary.getSourcePort(), primary.getAuthUserName(),
-                                primary.getAuthSecret(), disk.getPath()));
+                final QemuImgFile srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary, disk.getPath()));
                 srcFile.setFormat(PhysicalDiskFormat.RAW);
 
                 final QemuImgFile destFile = new QemuImgFile(tmpltPath + "/" + command.getUniqueName() + ".qcow2");

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java

@@ -161,11 +161,7 @@ private Map<String, String> getDiskFileInfo(KVMStoragePool pool, KVMPhysicalDisk
             QemuImg qemu = new QemuImg(0);
             QemuImgFile qemuFile = new QemuImgFile(disk.getPath(), disk.getFormat());
             if (StoragePoolType.RBD.equals(pool.getType())) {
-                String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(pool.getSourceHost(),
-                        pool.getSourcePort(),
-                        pool.getAuthUserName(),
-                        pool.getAuthSecret(),
-                        disk.getPath());
+                String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(pool, disk.getPath());
                 qemuFile = new QemuImgFile(rbdDestFile, disk.getFormat());
             }
             return qemu.info(qemuFile, secure);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java

@@ -410,9 +410,7 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk srcDisk, String destVolu
         KVMStoragePool srcPool = srcDisk.getPool();
 
         if (srcPool.getType() == StoragePoolType.RBD) {
-            srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool.getSourceHost(), srcPool.getSourcePort(),
-                                                                       srcPool.getAuthUserName(), srcPool.getAuthSecret(),
-                                                                       srcDisk.getPath()),srcDisk.getFormat());
+            srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool, srcDisk.getPath()), srcDisk.getFormat());
         } else {
             srcFile = new QemuImgFile(srcDisk.getPath(), srcDisk.getFormat());
         }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java

@@ -23,6 +23,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 public class KVMPhysicalDisk {
     private String path;
@@ -32,10 +33,17 @@ public class KVMPhysicalDisk {
     private String vmName;
     private boolean useAsTemplate;
 
-    public static String RBDStringBuilder(String monHost, int monPort, String authUserName, String authSecret, String image) {
-        String rbdOpts;
+    public static final String RBD_DEFAULT_DATA_POOL = "rbd_default_data_pool";
 
-        rbdOpts = "rbd:" + image;
+    public static String RBDStringBuilder(KVMStoragePool storagePool, String image) {
+        String monHost = storagePool.getSourceHost();
+        int monPort = storagePool.getSourcePort();
+        String authUserName = storagePool.getAuthUserName();
+        String authSecret = storagePool.getAuthSecret();
+        Map<String, String> details = storagePool.getDetails();
+        String dataPool = (details == null) ? null : details.get(RBD_DEFAULT_DATA_POOL);
+
+        String rbdOpts = "rbd:" + image;
         rbdOpts += ":mon_host=" + composeOptionForMonHosts(monHost, monPort);
 
         if (authUserName == null) {
@@ -46,6 +54,10 @@ public static String RBDStringBuilder(String monHost, int monPort, String authUs
             rbdOpts += ":key=" + authSecret;
         }
 
+        if (dataPool != null) {
+            rbdOpts += String.format(":rbd_default_data_pool=%s", dataPool);
+        }
+
         rbdOpts += ":rbd_default_format=2";
         rbdOpts += ":client_mount_timeout=30";
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java

@@ -53,28 +53,6 @@
 public class KVMStoragePoolManager {
     protected Logger logger = LogManager.getLogger(getClass());
 
-    private class StoragePoolInformation {
-        String name;
-        String host;
-        int port;
-        String path;
-        String userInfo;
-        boolean type;
-        StoragePoolType poolType;
-        Map<String, String> details;
-
-        public StoragePoolInformation(String name, String host, int port, String path, String userInfo, StoragePoolType poolType, Map<String, String> details, boolean type) {
-            this.name = name;
-            this.host = host;
-            this.port = port;
-            this.path = path;
-            this.userInfo = userInfo;
-            this.type = type;
-            this.poolType = poolType;
-            this.details = details;
-        }
-    }
-
     private KVMHAMonitor _haMonitor;
     private final Map<String, StoragePoolInformation> _storagePools = new ConcurrentHashMap<String, StoragePoolInformation>();
     private final Map<String, StorageAdaptor> _storageMapper = new HashMap<String, StorageAdaptor>();
@@ -303,14 +281,33 @@ public KVMStoragePool getStoragePool(StoragePoolType type, String uuid, boolean
         } catch (Exception e) {
             StoragePoolInformation info = _storagePools.get(uuid);
             if (info != null) {
-                pool = createStoragePool(info.name, info.host, info.port, info.path, info.userInfo, info.poolType, info.details, info.type);
+                pool = createStoragePool(info.getName(), info.getHost(), info.getPort(), info.getPath(), info.getUserInfo(), info.getPoolType(), info.getDetails(), info.isType());
             } else {
                 throw new CloudRuntimeException("Could not fetch storage pool " + uuid + " from libvirt due to " + e.getMessage());
             }
         }
+
+        if (pool instanceof LibvirtStoragePool) {
+            addPoolDetails(uuid, (LibvirtStoragePool) pool);
+        }
+
         return pool;
     }
 
+    /**
+     * As the class {@link LibvirtStoragePool} is constrained to the {@link org.libvirt.StoragePool} class, there is no way of saving a generic parameter such as the details, hence,
+     * this method was created to always make available the details of libvirt primary storages for when they are needed.
+     */
+    private void addPoolDetails(String uuid, LibvirtStoragePool pool) {
+        StoragePoolInformation storagePoolInformation = _storagePools.get(uuid);
+        Map<String, String> details = storagePoolInformation.getDetails();
+
+        if (MapUtils.isNotEmpty(details)) {
+            logger.trace("Adding the details {} to the pool with UUID {}.", details, uuid);
+            pool.setDetails(details);
+        }
+    }
+
     public KVMStoragePool getStoragePoolByURI(String uri) {
         URI storageUri = null;
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -667,9 +667,7 @@ public Answer createTemplateFromVolume(final CopyCommand cmd) {
             } else {
                 logger.debug("Converting RBD disk " + disk.getPath() + " into template " + templateName);
 
-                final QemuImgFile srcFile =
-                        new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary.getSourceHost(), primary.getSourcePort(), primary.getAuthUserName(),
-                                primary.getAuthSecret(), disk.getPath()));
+                final QemuImgFile srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary, disk.getPath()));
                 srcFile.setFormat(PhysicalDiskFormat.RAW);
 
                 final QemuImgFile destFile = new QemuImgFile(tmpltPath + "/" + templateName + ".qcow2");
@@ -1022,9 +1020,7 @@ public Answer backupSnapshot(final CopyCommand cmd) {
                     logger.debug("Attempting to create " + snapDir.getAbsolutePath() + " recursively for snapshot storage");
                     FileUtils.forceMkdir(snapDir);
 
-                    final QemuImgFile srcFile =
-                            new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primaryPool.getSourceHost(), primaryPool.getSourcePort(), primaryPool.getAuthUserName(),
-                                    primaryPool.getAuthSecret(), rbdSnapshot));
+                    final QemuImgFile srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primaryPool, rbdSnapshot));
                     srcFile.setFormat(snapshotDisk.getFormat());
 
                     final QemuImgFile destFile = new QemuImgFile(snapshotFile);