Merge branch '4.20' into 4.20-routed-support-vxlan

Author: weizhouapache

.github/workflows/main-sonar-check.yml

@@ -54,7 +54,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-m2
 

.github/workflows/sonar-check.yml

@@ -56,7 +56,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-m2
 

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -448,6 +448,7 @@ public class ApiConstants {
     public static final String SENT = "sent";
     public static final String SENT_BYTES = "sentbytes";
     public static final String SERIAL = "serial";
+    public static final String SERVICE_IP = "serviceip";
     public static final String SERVICE_OFFERING_ID = "serviceofferingid";
     public static final String SESSIONKEY = "sessionkey";
     public static final String SHOW_CAPACITIES = "showcapacities";

api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java

@@ -74,6 +74,11 @@ public class ManagementServerResponse extends BaseResponse {
     @Param(description = "the running OS kernel version for this Management Server")
     private String kernelVersion;
 
+    @Deprecated
+    @SerializedName(ApiConstants.SERVICE_IP)
+    @Param(description = "the IP Address for this Management Server. This is deprecated, please use 'ipaddress' instead.")
+    private String serviceIp;
+
     @SerializedName(ApiConstants.IP_ADDRESS)
     @Param(description = "the IP Address for this Management Server")
     private String ipAddress;
@@ -122,6 +127,10 @@ public Date getLastBoot() {
         return lastBoot;
     }
 
+    public String getServiceIp() {
+        return serviceIp;
+    }
+
     public String getIpAddress() {
         return ipAddress;
     }
@@ -170,6 +179,10 @@ public void setKernelVersion(String kernelVersion) {
         this.kernelVersion = kernelVersion;
     }
 
+    public void setServiceIp(String serviceIp) {
+        this.serviceIp = serviceIp;
+    }
+
     public void setIpAddress(String ipAddress) {
         this.ipAddress = ipAddress;
     }

api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java

@@ -79,6 +79,14 @@ public class UnmanagedInstanceResponse extends BaseResponse {
     @Param(description = "the operating system of the virtual machine")
     private String operatingSystem;
 
+    @SerializedName(ApiConstants.BOOT_MODE)
+    @Param(description = "indicates the boot mode")
+    private String bootMode;
+
+    @SerializedName(ApiConstants.BOOT_TYPE)
+    @Param(description = "indicates the boot type")
+    private String bootType;
+
     @SerializedName(ApiConstants.DISK)
     @Param(description = "the list of disks associated with the virtual machine", responseObject = UnmanagedInstanceDiskResponse.class)
     private Set<UnmanagedInstanceDiskResponse> disks;
@@ -211,4 +219,20 @@ public void setNics(Set<NicResponse> nics) {
     public void addNic(NicResponse nic) {
         this.nics.add(nic);
     }
+
+    public String getBootMode() {
+        return bootMode;
+    }
+
+    public void setBootMode(String bootMode) {
+        this.bootMode = bootMode;
+    }
+
+    public String getBootType() {
+        return bootType;
+    }
+
+    public void setBootType(String bootType) {
+        this.bootType = bootType;
+    }
 }

api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java

@@ -61,6 +61,9 @@ public enum PowerState {
 
     private String vncPassword;
 
+    private String bootType;
+    private String bootMode;
+
     public String getName() {
         return name;
     }
@@ -196,6 +199,22 @@ public String toString() {
                         this, "name", "internalCSName", "hostName", "clusterName"));
     }
 
+    public String getBootType() {
+        return bootType;
+    }
+
+    public void setBootType(String bootType) {
+        this.bootType = bootType;
+    }
+
+    public String getBootMode() {
+        return bootMode;
+    }
+
+    public void setBootMode(String bootMode) {
+        this.bootMode = bootMode;
+    }
+
     public static class Disk {
         private String diskId;
 

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -61,7 +61,11 @@
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
 import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
 import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
 import org.apache.cloudstack.framework.ca.Certificate;
 import org.apache.cloudstack.framework.config.ConfigKey;
@@ -409,6 +413,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     ResourceCleanupService resourceCleanupService;
     @Inject
     VmWorkJobDao vmWorkJobDao;
+    @Inject
+    DataStoreProviderManager dataStoreProviderManager;
 
     private SingleCache<List<Long>> vmIdsInProgressCache;
 
@@ -1224,6 +1230,13 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
                                 planChangedByVolume = true;
                             }
                         }
+                        DataStoreProvider storeProvider = dataStoreProviderManager.getDataStoreProvider(pool.getStorageProviderName());
+                        if (storeProvider != null) {
+                            DataStoreDriver storeDriver = storeProvider.getDataStoreDriver();
+                            if (storeDriver instanceof PrimaryDataStoreDriver) {
+                                ((PrimaryDataStoreDriver)storeDriver).detachVolumeFromAllStorageNodes(vol);
+                            }
+                        }
                     }
                 }
 

plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java

@@ -26,6 +26,7 @@
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.ScopeType;
+import com.cloud.storage.Storage;
 import com.cloud.storage.StoragePoolHostVO;
 import com.cloud.storage.Volume;
 import com.cloud.storage.VolumeVO;
@@ -280,6 +281,7 @@ public Pair<Boolean, String> restoreBackedUpVolume(Backup backup, String volumeU
         restoredVolume.setPoolId(dataStore.getPoolId());
         restoredVolume.setPath(restoredVolume.getUuid());
         restoredVolume.setState(Volume.State.Copying);
+        restoredVolume.setFormat(Storage.ImageFormat.QCOW2);
         restoredVolume.setSize(backedUpVolumeSize);
         restoredVolume.setDiskOfferingId(volume.getDiskOfferingId());
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java

@@ -45,7 +45,7 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper<RestoreBa
     private static final String MOUNT_COMMAND = "sudo mount -t %s %s %s";
     private static final String UMOUNT_COMMAND = "sudo umount %s";
     private static final String FILE_PATH_PLACEHOLDER = "%s/%s";
-    private static final String ATTACH_DISK_COMMAND = " virsh attach-disk %s %s %s --cache none";
+    private static final String ATTACH_DISK_COMMAND = " virsh attach-disk %s %s %s --driver qemu --subdriver qcow2 --cache none";
     private static final String CURRRENT_DEVICE = "virsh domblklist --domain %s | tail -n 3 | head -n 1 | awk '{print $1}'";
     private static final String RSYNC_COMMAND = "rsync -az %s %s";
 

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java

@@ -43,6 +43,7 @@
 import javax.naming.ConfigurationException;
 import javax.persistence.EntityExistsException;
 
+import com.cloud.hypervisor.vmware.mo.VirtualMachineMO;
 import com.cloud.hypervisor.vmware.util.VmwareClient;
 import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd;
 import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd;
@@ -171,8 +172,11 @@
 import com.vmware.pbm.PbmProfile;
 import com.vmware.vim25.AboutInfo;
 import com.vmware.vim25.ManagedObjectReference;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 public class VmwareManagerImpl extends ManagerBase implements VmwareManager, VmwareStorageMount, Listener, VmwareDatacenterService, Configurable {
+    protected static Logger static_logger = LogManager.getLogger(VmwareManagerImpl.class);
 
     private static final long SECONDS_PER_MINUTE = 60;
     private static final int DEFAULT_PORTS_PER_DV_PORT_GROUP_VSPHERE4_x = 256;
@@ -249,7 +253,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
     private boolean _fullCloneFlag;
     private boolean _instanceNameFlag;
     private String _serviceConsoleName;
-    private String _managemetPortGroupName;
+    private String _managementPortGroupName;
     private String _defaultSystemVmNicAdapterType = VirtualEthernetCardType.E1000.toString();
     private String _recycleHungWorker = "false";
     private int _additionalPortRangeStart;
@@ -347,9 +351,9 @@ public boolean configure(String name, Map<String, Object> params) throws Configu
             _serviceConsoleName = "Service Console";
         }
 
-        _managemetPortGroupName = _configDao.getValue(Config.VmwareManagementPortGroup.key());
-        if (_managemetPortGroupName == null) {
-            _managemetPortGroupName = "Management Network";
+        _managementPortGroupName = _configDao.getValue(Config.VmwareManagementPortGroup.key());
+        if (_managementPortGroupName == null) {
+            _managementPortGroupName = "Management Network";
         }
 
         _defaultSystemVmNicAdapterType = _configDao.getValue(Config.VmwareSystemVmNicDeviceType.key());
@@ -614,13 +618,13 @@ public String getServiceConsolePortGroupName() {
 
     @Override
     public String getManagementPortGroupName() {
-        return _managemetPortGroupName;
+        return _managementPortGroupName;
     }
 
     @Override
     public String getManagementPortGroupByHost(HostMO hostMo) throws Exception {
         if (hostMo.getHostType() == VmwareHostType.ESXi) {
-            return _managemetPortGroupName;
+            return _managementPortGroupName;
         }
         return _serviceConsoleName;
     }
@@ -630,7 +634,7 @@ public void setupResourceStartupParams(Map<String, Object> params) {
         params.put("vmware.create.full.clone", _fullCloneFlag);
         params.put("vm.instancename.flag", _instanceNameFlag);
         params.put("service.console.name", _serviceConsoleName);
-        params.put("management.portgroup.name", _managemetPortGroupName);
+        params.put("management.portgroup.name", _managementPortGroupName);
         params.put("vmware.root.disk.controller", _rootDiskController);
         params.put("vmware.data.disk.controller", _dataDiskController);
         params.put("vmware.recycle.hung.wokervm", _recycleHungWorker);
@@ -1585,14 +1589,26 @@ public List<StoragePool> listVsphereStoragePolicyCompatibleStoragePools(ListVsph
         return compatiblePools;
     }
 
-    @Override
-    public List<UnmanagedInstanceTO> listVMsInDatacenter(ListVmwareDcVmsCmd cmd) {
+    private static class VcenterData {
+        public final String vcenter;
+        public final String datacenterName;
+        public final String username;
+        public final String password;
+
+        public VcenterData(String vcenter, String datacenterName, String username, String password) {
+            this.vcenter = vcenter;
+            this.datacenterName = datacenterName;
+            this.username = username;
+            this.password = password;
+        }
+    }
+
+    private VcenterData getVcenterData(ListVmwareDcVmsCmd cmd) {
         String vcenter = cmd.getVcenter();
         String datacenterName = cmd.getDatacenterName();
         String username = cmd.getUsername();
         String password = cmd.getPassword();
         Long existingVcenterId = cmd.getExistingVcenterId();
-        String keyword = cmd.getKeyword();
 
         if ((existingVcenterId == null && StringUtils.isBlank(vcenter)) ||
                 (existingVcenterId != null && StringUtils.isNotBlank(vcenter))) {
@@ -1613,34 +1629,69 @@ public List<UnmanagedInstanceTO> listVMsInDatacenter(ListVmwareDcVmsCmd cmd) {
             username = vmwareDc.getUser();
             password = vmwareDc.getPassword();
         }
+        VcenterData vmwaredc = new VcenterData(vcenter, datacenterName, username, password);
+        return vmwaredc;
+    }
+
+    private static VmwareContext getVmwareContext(String vcenter, String username, String password) throws Exception {
+        static_logger.debug(String.format("Connecting to the VMware vCenter %s", vcenter));
+        String serviceUrl = String.format("https://%s/sdk/vimService", vcenter);
+        VmwareClient vimClient = new VmwareClient(vcenter);
+        vimClient.connect(serviceUrl, username, password);
+        return new VmwareContext(vimClient, vcenter);
+    }
+
+    @Override
+    public List<UnmanagedInstanceTO> listVMsInDatacenter(ListVmwareDcVmsCmd cmd) {
+        VcenterData vmwareDC = getVcenterData(cmd);
+        String vcenter = vmwareDC.vcenter;
+        String username = vmwareDC.username;
+        String password = vmwareDC.password;
+        String datacenterName = vmwareDC.datacenterName;
+        String keyword = cmd.getKeyword();
+        String esxiHostName = cmd.getHostName();
+        String virtualMachineName = cmd.getInstanceName();
 
         try {
             logger.debug(String.format("Connecting to the VMware datacenter %s at vCenter %s to retrieve VMs",
                     datacenterName, vcenter));
-            String serviceUrl = String.format("https://%s/sdk/vimService", vcenter);
-            VmwareClient vimClient = new VmwareClient(vcenter);
-            vimClient.connect(serviceUrl, username, password);
-            VmwareContext context = new VmwareContext(vimClient, vcenter);
-
-            DatacenterMO dcMo = new DatacenterMO(context, datacenterName);
-            ManagedObjectReference dcMor = dcMo.getMor();
-            if (dcMor == null) {
-                String msg = String.format("Unable to find VMware datacenter %s in vCenter %s",
-                        datacenterName, vcenter);
-                logger.error(msg);
-                throw new InvalidParameterValueException(msg);
+            VmwareContext context = getVmwareContext(vcenter, username, password);
+            DatacenterMO dcMo = getDatacenterMO(context, vcenter, datacenterName);
+
+            List<UnmanagedInstanceTO> instances;
+            if (StringUtils.isNotBlank(esxiHostName) && StringUtils.isNotBlank(virtualMachineName)) {
+                ManagedObjectReference hostMor = dcMo.findHost(esxiHostName);
+                if (hostMor == null) {
+                    String errorMsg = String.format("Cannot find a host with name %s on vcenter %s", esxiHostName, vcenter);
+                    logger.error(errorMsg);
+                    throw new CloudRuntimeException(errorMsg);
+                }
+                HostMO hostMO = new HostMO(context, hostMor);
+                VirtualMachineMO vmMo = hostMO.findVmOnHyperHost(virtualMachineName);
+                instances = Collections.singletonList(VmwareHelper.getUnmanagedInstance(hostMO, vmMo));
+            } else {
+                instances = dcMo.getAllVmsOnDatacenter(keyword);
             }
-            List<UnmanagedInstanceTO> instances = dcMo.getAllVmsOnDatacenter();
-            return StringUtils.isBlank(keyword) ? instances :
-                    instances.stream().filter(x -> x.getName().toLowerCase().contains(keyword.toLowerCase())).collect(Collectors.toList());
+            return instances;
         } catch (Exception e) {
-            String errorMsg = String.format("Error retrieving stopped VMs from the VMware VC %s datacenter %s: %s",
+            String errorMsg = String.format("Error retrieving VMs from the VMware VC %s datacenter %s: %s",
                     vcenter, datacenterName, e.getMessage());
             logger.error(errorMsg, e);
             throw new CloudRuntimeException(errorMsg);
         }
     }
 
+    private static DatacenterMO getDatacenterMO(VmwareContext context, String vcenter, String datacenterName) throws Exception {
+        DatacenterMO dcMo = new DatacenterMO(context, datacenterName);
+        ManagedObjectReference dcMor = dcMo.getMor();
+        if (dcMor == null) {
+            String msg = String.format("Unable to find VMware datacenter %s in vCenter %s", datacenterName, vcenter);
+            static_logger.error(msg);
+            throw new InvalidParameterValueException(msg);
+        }
+        return dcMo;
+    }
+
     @Override
     public boolean hasNexusVSM(Long clusterId) {
         ClusterVSMMapVO vsmMapVo = null;
@@ -1693,7 +1744,7 @@ private void startTemplateCleanJobSchedule() {
     }
 
     /**
-     * This task is to cleanup templates from primary storage that are otherwise not cleaned by the {@link com.cloud.storage.StorageManagerImpl.StorageGarbageCollector}.
+     * This task is to cleanup templates from primary storage that are otherwise not cleaned by the {code}StorageGarbageCollector{code} from {@link com.cloud.storage.StorageManagerImpl}.
      * it is called at regular intervals when storage.template.cleanup.enabled == true
      * It collect all templates that
      * - are deleted from cloudstack