Merge remote-tracking branch 'apache/4.19' into fix-kvm-securebootvm-diskcontroller

Author: shwstppr

.github/workflows/ci.yml

@@ -232,7 +232,7 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid
+          python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid pynose
 
       - name: Install jacoco dependencies
         run: |

agent/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>org.apache.cloudstack</groupId>
         <artifactId>cloudstack</artifactId>
-        <version>4.19.2.0-SNAPSHOT</version>
+        <version>4.19.3.0-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>

api/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>org.apache.cloudstack</groupId>
         <artifactId>cloudstack</artifactId>
-        <version>4.19.2.0-SNAPSHOT</version>
+        <version>4.19.3.0-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>

api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java

@@ -27,6 +27,7 @@ public class RemoteInstanceTO implements Serializable {
 
     private Hypervisor.HypervisorType hypervisorType;
     private String instanceName;
+    private String instancePath;
 
     // VMware Remote Instances parameters (required for exporting OVA through ovftool)
     // TODO: cloud.agent.transport.Request#getCommands() cannot handle gsoc decode for polymorphic classes
@@ -44,9 +45,10 @@ public RemoteInstanceTO(String instanceName) {
         this.instanceName = instanceName;
     }
 
-    public RemoteInstanceTO(String instanceName, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) {
+    public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) {
         this.hypervisorType = Hypervisor.HypervisorType.VMware;
         this.instanceName = instanceName;
+        this.instancePath = instancePath;
         this.vcenterHost = vcenterHost;
         this.vcenterUsername = vcenterUsername;
         this.vcenterPassword = vcenterPassword;
@@ -61,6 +63,10 @@ public String getInstanceName() {
         return this.instanceName;
     }
 
+    public String getInstancePath() {
+        return this.instancePath;
+    }
+
     public String getVcenterUsername() {
         return vcenterUsername;
     }

api/src/main/java/com/cloud/exception/StorageAccessException.java

@@ -26,7 +26,7 @@
 public class StorageAccessException extends RuntimeException {
     private static final long serialVersionUID = SerialVersionUID.StorageAccessException;
 
-    public StorageAccessException(String message) {
-        super(message);
+    public StorageAccessException(String message, Exception causer) {
+        super(message, causer);
     }
 }

api/src/main/java/com/cloud/storage/MigrationOptions.java

@@ -24,6 +24,7 @@ public class MigrationOptions implements Serializable {
 
     private String srcPoolUuid;
     private Storage.StoragePoolType srcPoolType;
+    private Long srcPoolClusterId;
     private Type type;
     private ScopeType scopeType;
     private String srcBackingFilePath;
@@ -38,21 +39,23 @@ public enum Type {
     public MigrationOptions() {
     }
 
-    public MigrationOptions(String srcPoolUuid, Storage.StoragePoolType srcPoolType, String srcBackingFilePath, boolean copySrcTemplate, ScopeType scopeType) {
+    public MigrationOptions(String srcPoolUuid, Storage.StoragePoolType srcPoolType, String srcBackingFilePath, boolean copySrcTemplate, ScopeType scopeType, Long srcPoolClusterId) {
         this.srcPoolUuid = srcPoolUuid;
         this.srcPoolType = srcPoolType;
         this.type = Type.LinkedClone;
         this.scopeType = scopeType;
         this.srcBackingFilePath = srcBackingFilePath;
         this.copySrcTemplate = copySrcTemplate;
+        this.srcPoolClusterId = srcPoolClusterId;
     }
 
-    public MigrationOptions(String srcPoolUuid, Storage.StoragePoolType srcPoolType, String srcVolumeUuid, ScopeType scopeType) {
+    public MigrationOptions(String srcPoolUuid, Storage.StoragePoolType srcPoolType, String srcVolumeUuid, ScopeType scopeType, Long srcPoolClusterId) {
         this.srcPoolUuid = srcPoolUuid;
         this.srcPoolType = srcPoolType;
         this.type = Type.FullClone;
         this.scopeType = scopeType;
         this.srcVolumeUuid = srcVolumeUuid;
+        this.srcPoolClusterId = srcPoolClusterId;
     }
 
     public String getSrcPoolUuid() {
@@ -63,6 +66,10 @@ public Storage.StoragePoolType getSrcPoolType() {
         return srcPoolType;
     }
 
+    public Long getSrcPoolClusterId() {
+        return srcPoolClusterId;
+    }
+
     public ScopeType getScopeType() { return scopeType; }
 
     public String getSrcBackingFilePath() {

api/src/main/java/com/cloud/storage/Storage.java

@@ -135,34 +135,49 @@ public static enum TemplateType {
         ISODISK /* Template corresponding to a iso (non root disk) present in an OVA */
     }
 
+    public enum EncryptionSupport {
+        /**
+         * Encryption not supported.
+         */
+        Unsupported,
+        /**
+         * Will use hypervisor encryption driver (qemu -> luks)
+         */
+        Hypervisor,
+        /**
+         * Storage pool handles encryption and just provides an encrypted volume
+         */
+        Storage
+    }
+
     public static enum StoragePoolType {
-        Filesystem(false, true, true), // local directory
-        NetworkFilesystem(true, true, true), // NFS
-        IscsiLUN(true, false, false), // shared LUN, with a clusterfs overlay
-        Iscsi(true, false, false), // for e.g., ZFS Comstar
-        ISO(false, false, false), // for iso image
-        LVM(false, false, false), // XenServer local LVM SR
-        CLVM(true, false, false),
-        RBD(true, true, false), // http://libvirt.org/storage.html#StorageBackendRBD
-        SharedMountPoint(true, true, true),
-        VMFS(true, true, false), // VMware VMFS storage
-        PreSetup(true, true, false), // for XenServer, Storage Pool is set up by customers.
-        EXT(false, true, false), // XenServer local EXT SR
-        OCFS2(true, false, false),
-        SMB(true, false, false),
-        Gluster(true, false, false),
-        PowerFlex(true, true, true), // Dell EMC PowerFlex/ScaleIO (formerly VxFlexOS)
-        ManagedNFS(true, false, false),
-        Linstor(true, true, false),
-        DatastoreCluster(true, true, false), // for VMware, to abstract pool of clusters
-        StorPool(true, true, true),
-        FiberChannel(true, true, false); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-<wwnvalue>)
+        Filesystem(false, true, EncryptionSupport.Hypervisor), // local directory
+        NetworkFilesystem(true, true, EncryptionSupport.Hypervisor), // NFS
+        IscsiLUN(true, false, EncryptionSupport.Unsupported), // shared LUN, with a clusterfs overlay
+        Iscsi(true, false, EncryptionSupport.Unsupported), // for e.g., ZFS Comstar
+        ISO(false, false, EncryptionSupport.Unsupported), // for iso image
+        LVM(false, false, EncryptionSupport.Unsupported), // XenServer local LVM SR
+        CLVM(true, false, EncryptionSupport.Unsupported),
+        RBD(true, true, EncryptionSupport.Unsupported), // http://libvirt.org/storage.html#StorageBackendRBD
+        SharedMountPoint(true, true, EncryptionSupport.Hypervisor),
+        VMFS(true, true, EncryptionSupport.Unsupported), // VMware VMFS storage
+        PreSetup(true, true, EncryptionSupport.Unsupported), // for XenServer, Storage Pool is set up by customers.
+        EXT(false, true, EncryptionSupport.Unsupported), // XenServer local EXT SR
+        OCFS2(true, false, EncryptionSupport.Unsupported),
+        SMB(true, false, EncryptionSupport.Unsupported),
+        Gluster(true, false, EncryptionSupport.Unsupported),
+        PowerFlex(true, true, EncryptionSupport.Hypervisor), // Dell EMC PowerFlex/ScaleIO (formerly VxFlexOS)
+        ManagedNFS(true, false, EncryptionSupport.Unsupported),
+        Linstor(true, true, EncryptionSupport.Storage),
+        DatastoreCluster(true, true, EncryptionSupport.Unsupported), // for VMware, to abstract pool of clusters
+        StorPool(true, true, EncryptionSupport.Hypervisor),
+        FiberChannel(true, true, EncryptionSupport.Unsupported); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-<wwnvalue>)
 
         private final boolean shared;
         private final boolean overProvisioning;
-        private final boolean encryption;
+        private final EncryptionSupport encryption;
 
-        StoragePoolType(boolean shared, boolean overProvisioning, boolean encryption) {
+        StoragePoolType(boolean shared, boolean overProvisioning, EncryptionSupport encryption) {
             this.shared = shared;
             this.overProvisioning = overProvisioning;
             this.encryption = encryption;
@@ -177,6 +192,10 @@ public boolean supportsOverProvisioning() {
         }
 
         public boolean supportsEncryption() {
+            return encryption == EncryptionSupport.Hypervisor || encryption == EncryptionSupport.Storage;
+        }
+
+        public EncryptionSupport encryptionSupportMode() {
             return encryption;
         }
     }

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -133,13 +133,13 @@ Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account acc
     Snapshot allocSnapshotForVm(Long vmId, Long volumeId, String snapshotName) throws ResourceAllocationException;
 
     /**
-     *  Checks if the target storage supports the disk offering.
+     *  Checks if the storage pool supports the disk offering tags.
      *  This validation is consistent with the mechanism used to select a storage pool to deploy a volume when a virtual machine is deployed or when a data disk is allocated.
      *
      *  The scenarios when this method returns true or false is presented in the following table.
      *   <table border="1">
      *      <tr>
-     *          <th>#</th><th>Disk offering tags</th><th>Storage tags</th><th>Does the storage support the disk offering?</th>
+     *          <th>#</th><th>Disk offering diskOfferingTags</th><th>Storage diskOfferingTags</th><th>Does the storage support the disk offering?</th>
      *      </tr>
      *      <body>
      *      <tr>
@@ -163,7 +163,7 @@ Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account acc
      *      </body>
      *   </table>
      */
-    boolean doesTargetStorageSupportDiskOffering(StoragePool destPool, String diskOfferingTags);
+    boolean doesStoragePoolSupportDiskOfferingTags(StoragePool destPool, String diskOfferingTags);
 
     Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge);
 

api/src/main/java/org/apache/cloudstack/api/command/user/firewall/CreateFirewallRuleCmd.java

@@ -19,6 +19,8 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.acl.RoleType;
@@ -103,14 +105,13 @@ public String getProtocol() {
 
     @Override
     public List<String> getSourceCidrList() {
-        if (cidrlist != null) {
+        if (CollectionUtils.isNotEmpty(cidrlist) && !(cidrlist.size() == 1 && StringUtils.isBlank(cidrlist.get(0)))) {
             return cidrlist;
         } else {
-            List<String> oneCidrList = new ArrayList<String>();
+            List<String> oneCidrList = new ArrayList<>();
             oneCidrList.add(NetUtils.ALL_IP4_CIDRS);
             return oneCidrList;
         }
-
     }
 
     // ///////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java

@@ -41,6 +41,7 @@
 import org.apache.cloudstack.api.response.SecurityGroupResponse;
 import org.apache.cloudstack.api.response.ServiceOfferingResponse;
 import org.apache.cloudstack.api.response.TemplateResponse;
+import org.apache.cloudstack.api.response.UserDataResponse;
 import org.apache.cloudstack.api.response.UserResponse;
 import org.apache.cloudstack.api.response.UserVmResponse;
 import org.apache.cloudstack.api.response.VpcResponse;
@@ -151,6 +152,9 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements
     @Parameter(name = ApiConstants.USER_DATA, type = CommandType.BOOLEAN, description = "Whether to return the VMs' user data or not. By default, user data will not be returned.", since = "4.18.0.0")
     private Boolean showUserData;
 
+    @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, required = false, description = "the instances by userdata", since = "4.20.1")
+    private Long userdataId;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -245,6 +249,10 @@ protected boolean isViewDetailsEmpty() {
         return CollectionUtils.isEmpty(viewDetails);
     }
 
+    public Long getUserdataId() {
+        return userdataId;
+    }
+
     public EnumSet<VMDetails> getDetails() throws InvalidParameterValueException {
         if (isViewDetailsEmpty()) {
             if (_queryService.ReturnVmStatsOnVmList.value()) {