Merge branch 'apache:4.19' into 4.19

Author: Imvedansh

.github/workflows/ci.yml

@@ -232,7 +232,7 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid
+          python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid pynose
 
       - name: Install jacoco dependencies
         run: |

api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java

@@ -41,6 +41,7 @@
 import org.apache.cloudstack.api.response.SecurityGroupResponse;
 import org.apache.cloudstack.api.response.ServiceOfferingResponse;
 import org.apache.cloudstack.api.response.TemplateResponse;
+import org.apache.cloudstack.api.response.UserDataResponse;
 import org.apache.cloudstack.api.response.UserResponse;
 import org.apache.cloudstack.api.response.UserVmResponse;
 import org.apache.cloudstack.api.response.VpcResponse;
@@ -151,6 +152,9 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements
     @Parameter(name = ApiConstants.USER_DATA, type = CommandType.BOOLEAN, description = "Whether to return the VMs' user data or not. By default, user data will not be returned.", since = "4.18.0.0")
     private Boolean showUserData;
 
+    @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, required = false, description = "the instances by userdata", since = "4.20.1")
+    private Long userdataId;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -245,6 +249,10 @@ protected boolean isViewDetailsEmpty() {
         return CollectionUtils.isEmpty(viewDetails);
     }
 
+    public Long getUserdataId() {
+        return userdataId;
+    }
+
     public EnumSet<VMDetails> getDetails() throws InvalidParameterValueException {
         if (isViewDetailsEmpty()) {
             if (_queryService.ReturnVmStatsOnVmList.value()) {

api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java

@@ -27,12 +27,11 @@
 import org.apache.cloudstack.api.EntityReference;
 
 import com.cloud.network.Network;
-import com.cloud.projects.ProjectAccount;
 import com.cloud.serializer.Param;
 import com.google.gson.annotations.SerializedName;
 
 @SuppressWarnings("unused")
-@EntityReference(value = {Network.class, ProjectAccount.class})
+@EntityReference(value = {Network.class})
 public class NetworkResponse extends BaseResponseWithAssociatedNetwork implements ControlledEntityResponse, SetResourceIconResponse {
 
     @SerializedName(ApiConstants.ID)

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -5248,10 +5248,9 @@ public Outcome<VirtualMachine> migrateVmAwayThroughJobQueue(final String vmUuid,
             workJob = newVmWorkJobAndInfo.first();
             VmWorkMigrateAway workInfo = new VmWorkMigrateAway(newVmWorkJobAndInfo.second(), srcHostId);
 
-            workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo));
+            setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
         }
 
-        _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vmId);
 
         AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
 

engine/schema/src/main/java/com/cloud/vm/VMInstanceVO.java

@@ -502,7 +502,7 @@ public void setRemoved(Date removed) {
 
     @Override
     public String toString() {
-        return String.format("VM instance %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "instanceName", "uuid", "type"));
+        return String.format("VM instance %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "instanceName", "uuid", "type", "state"));
     }
 
     @Override

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImpl.java

@@ -22,8 +22,13 @@
 
 import javax.inject.Inject;
 
+import com.cloud.storage.VMTemplateStoragePoolVO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
+import com.cloud.template.TemplateManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper;
 import org.apache.log4j.Logger;
 
@@ -54,6 +59,10 @@ public class BasePrimaryDataStoreLifeCycleImpl {
     protected HostDao hostDao;
     @Inject
     protected StoragePoolHostDao storagePoolHostDao;
+    @Inject
+    private PrimaryDataStoreDao primaryDataStoreDao;
+    @Inject
+    private TemplateManager templateMgr;
 
     private List<HostVO> getPoolHostsList(ClusterScope clusterScope, HypervisorType hypervisorType) {
         List<HostVO> hosts;
@@ -103,4 +112,42 @@ public void changeStoragePoolScopeToCluster(DataStore store, ClusterScope cluste
         }
         dataStoreHelper.switchToCluster(store, clusterScope);
     }
+
+    private void evictTemplates(StoragePoolVO storagePoolVO) {
+        List<VMTemplateStoragePoolVO> unusedTemplatesInPool = templateMgr.getUnusedTemplatesInPool(storagePoolVO);
+        for (VMTemplateStoragePoolVO templatePoolVO : unusedTemplatesInPool) {
+            if (templatePoolVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
+                templateMgr.evictTemplateFromStoragePool(templatePoolVO);
+            }
+        }
+    }
+
+    private void deleteAgentStoragePools(StoragePool storagePool) {
+        List<StoragePoolHostVO> poolHostVOs = storagePoolHostDao.listByPoolId(storagePool.getId());
+        for (StoragePoolHostVO poolHostVO : poolHostVOs) {
+            DeleteStoragePoolCommand deleteStoragePoolCommand = new DeleteStoragePoolCommand(storagePool);
+            final Answer answer = agentMgr.easySend(poolHostVO.getHostId(), deleteStoragePoolCommand);
+            if (answer != null && answer.getResult()) {
+                s_logger.info("Successfully deleted storage pool: " + storagePool.getId() + " from host: " + poolHostVO.getHostId());
+            } else {
+                if (answer != null) {
+                    s_logger.error("Failed to delete storage pool: " + storagePool.getId() + " from host: " + poolHostVO.getHostId() + " , result: " + answer.getResult());
+                } else {
+                    s_logger.error("Failed to delete storage pool: " + storagePool.getId() + " from host: " + poolHostVO.getHostId());
+                }
+            }
+        }
+    }
+
+    protected boolean cleanupDatastore(DataStore store) {
+        StoragePool storagePool = (StoragePool)store;
+        StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(storagePool.getId());
+        if (storagePoolVO == null) {
+            return false;
+        }
+
+        evictTemplates(storagePoolVO);
+        deleteAgentStoragePools(storagePool);
+        return true;
+    }
 }

framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java

@@ -1052,12 +1052,24 @@ public boolean stop() {
         }
 
         if (_mshostId != null) {
-            final ManagementServerHostVO mshost = _mshostDao.findByMsid(_msId);
-            final ManagementServerStatusVO mshostStatus = mshostStatusDao.findByMsId(mshost.getUuid());
-            mshost.setState(ManagementServerHost.State.Down);
-            mshostStatus.setLastJvmStop(new Date());
-            _mshostDao.update(_mshostId, mshost);
-            mshostStatusDao.update(mshostStatus.getId(), mshostStatus);
+            ManagementServerHostVO mshost = _mshostDao.findByMsid(_msId);
+            if (mshost != null) {
+                ManagementServerStatusVO mshostStatus = mshostStatusDao.findByMsId(mshost.getUuid());
+                if (mshostStatus != null) {
+                    mshost.setState(ManagementServerHost.State.Down);
+                    mshostStatus.setLastJvmStop(new Date());
+                    _mshostDao.update(_mshostId, mshost);
+                    mshostStatusDao.update(mshostStatus.getId(), mshostStatus);
+                } else {
+                    s_logger.warn(String.format("Found a management server host [%s] without a status. This should never happen!", mshost));
+                    mshostStatus = new ManagementServerStatusVO();
+                    mshostStatus.setMsId(mshost.getUuid());
+                    mshostStatus.setLastSystemBoot(new Date());
+                    mshostStatus.setLastJvmStart(new Date());
+                    mshostStatus.setUpdated(new Date());
+                    mshostStatusDao.persist(mshostStatus);
+                }
+            }
         }
 
         _heartbeatScheduler.shutdownNow();

plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java

@@ -199,6 +199,9 @@ public boolean assignVMToBackupOffering(final VirtualMachine vm, final BackupOff
     public boolean removeVMFromBackupOffering(final VirtualMachine vm) {
         final VeeamClient client = getClient(vm.getDataCenterId());
         final VmwareDatacenter vmwareDC = findVmwareDatacenterForVM(vm);
+        if (vm.getBackupExternalId() == null) {
+            throw new CloudRuntimeException("The VM does not have a backup job assigned.");
+        }
         try {
             if (!client.removeVMFromVeeamJob(vm.getBackupExternalId(), vm.getInstanceName(), vmwareDC.getVcenterHost())) {
                 LOG.warn("Failed to remove VM from Veeam Job id: " + vm.getBackupExternalId());

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java

@@ -51,10 +51,6 @@
 import javax.naming.ConfigurationException;
 import javax.xml.parsers.ParserConfigurationException;
 
-import com.trilead.ssh2.SFTPException;
-import com.trilead.ssh2.SFTPv3Client;
-import com.trilead.ssh2.SFTPv3DirectoryEntry;
-import com.trilead.ssh2.SFTPv3FileAttributes;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.diagnostics.CopyToSecondaryStorageAnswer;
 import org.apache.cloudstack.diagnostics.CopyToSecondaryStorageCommand;
@@ -72,6 +68,7 @@
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
+import org.apache.maven.artifact.versioning.ComparableVersion;
 import org.apache.xmlrpc.XmlRpcException;
 import org.joda.time.Duration;
 import org.w3c.dom.Document;
@@ -153,6 +150,10 @@
 import com.cloud.vm.VirtualMachine.PowerState;
 import com.cloud.vm.VmDetailConstants;
 import com.trilead.ssh2.SCPClient;
+import com.trilead.ssh2.SFTPException;
+import com.trilead.ssh2.SFTPv3Client;
+import com.trilead.ssh2.SFTPv3DirectoryEntry;
+import com.trilead.ssh2.SFTPv3FileAttributes;
 import com.xensource.xenapi.Bond;
 import com.xensource.xenapi.Connection;
 import com.xensource.xenapi.Console;
@@ -627,7 +628,7 @@ public boolean cleanupHaltedVms(final Connection conn) throws XenAPIException, X
 
                 if (VmPowerState.HALTED.equals(vmRec.powerState) && vmRec.affinity.equals(host) && !isAlienVm(vm, conn)) {
                     try {
-                        vm.destroy(conn);
+                        destroyVm(vm, conn);
                     } catch (final Exception e) {
                         s_logger.warn("Catch Exception " + e.getClass().getName() + ": unable to destroy VM " + vmRec.nameLabel + " due to ", e);
                         success = false;
@@ -1452,7 +1453,7 @@ public VM createVmFromTemplate(final Connection conn, final VirtualMachineTO vmS
                 vm.setPVBootloader(conn, "pygrub");
                 vm.setPVBootloaderArgs(conn, CitrixHelper.getPVbootloaderArgs(guestOsTypeName));
             } else {
-                vm.destroy(conn);
+                destroyVm(vm, conn, true);
                 throw new CloudRuntimeException("Unable to handle boot loader type: " + vmSpec.getBootloader());
             }
         }
@@ -2038,7 +2039,7 @@ void forceShutdownVM(final Connection conn, final VM vm) {
             final Long domId = vm.getDomid(conn);
             callHostPlugin(conn, "vmopspremium", "forceShutdownVM", "domId", domId.toString());
             vm.powerStateReset(conn);
-            vm.destroy(conn);
+            destroyVm(vm, conn);
         } catch (final Exception e) {
             final String msg = "forceShutdown failed due to " + e.toString();
             s_logger.warn(msg, e);
@@ -3690,7 +3691,7 @@ public String handleVmStartFailure(final Connection conn, final String vmName, f
             }
             if (vm.getPowerState(conn) == VmPowerState.HALTED) {
                 try {
-                    vm.destroy(conn);
+                    destroyVm(vm, conn, true);
                 } catch (final Exception e) {
                     s_logger.warn("VM destroy failed due to ", e);
                 }
@@ -5208,7 +5209,7 @@ protected void startvmfailhandle(final Connection conn, final VM vm, final List<
                 }
                 if (vm.getPowerState(conn) == VmPowerState.HALTED) {
                     try {
-                        vm.destroy(conn);
+                        destroyVm(vm, conn, true);
                     } catch (final Exception e) {
                         final String msg = "VM destroy failed due to " + e.toString();
                         s_logger.warn(msg, e);
@@ -5875,4 +5876,23 @@ private void umountNfs(Connection conn, String remoteDir, String localDir) {
             s_logger.warn(errMsg);
         }
     }
+
+    public boolean isDestroyHaltedVms() {
+        ComparableVersion version = new ComparableVersion(getHost().getProductVersion());
+        if (version.compareTo(new ComparableVersion("8.0")) >= 0) {
+            return false;
+        }
+        return true;
+    }
+
+    public void destroyVm(VM vm, Connection connection, boolean forced) throws XenAPIException, XmlRpcException {
+        if (!isDestroyHaltedVms() && !forced) {
+            return;
+        }
+        vm.destroy(connection);
+    }
+
+    public void destroyVm(VM vm, Connection connection) throws XenAPIException, XmlRpcException {
+        destroyVm(vm, connection, false);
+    }
 }

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xen56/XenServer56FenceCommandWrapper.java

@@ -56,7 +56,7 @@ public Answer execute(final FenceCommand command, final XenServer56Resource xenS
             for (final VM vm : vms) {
                 s_logger.info("Fence command for VM " + command.getVmName());
                 vm.powerStateReset(conn);
-                vm.destroy(conn);
+                xenServer56.destroyVm(vm, conn);
             }
             return new FenceAnswer(command);
         } catch (final XmlRpcException e) {