Merge branch 'main' into 4.21-reconcile-commands

Author: weizhouapache

.asf.yaml

@@ -54,10 +54,11 @@ github:
     - gpordeus
     - hsato03
     - bernardodemarco
-    - abh1sar
     - FelipeM525
     - lucas-a-martins
     - nicoschmdt
+    - abh1sar
+    - sudo87
 
   protected_branches: ~
 

CONTRIBUTING.md

@@ -4,9 +4,9 @@ Contributing to Apache CloudStack (ACS)
 Summary
 -------
 This document covers how to contribute to the ACS project. ACS uses GitHub PRs to manage code contributions.
-These instructions assume you have a GitHub.com account, so if you don't have one you will have to create one. Your proposed code changes will be published to your own fork of the ACS project and you will submit a Pull Request for your changes to be added.
+These instructions assume you have a GitHub.com account, so if you don't have one you will have to create one. Your proposed code changes will be published to your own fork of the ACS project, and you will submit a Pull Request for your changes to be added.
 
-_Lets get started!!!_
+_Let's get started!!!_
 
 Bug fixes
 ---------
@@ -26,7 +26,7 @@ No back porting / cherry-picking features to existing branches!
 
 PendingReleaseNotes file
 ------------------------
-When developing a new feature or making a (major) change to a existing feature you are encouraged to append this to the PendingReleaseNotes file so that the Release Manager can
+When developing a new feature or making a (major) change to an existing feature you are encouraged to append this to the PendingReleaseNotes file so that the Release Manager can
 use this file as a source of information when compiling the Release Notes for a new release.
 
 When adding information to the PendingReleaseNotes file make sure that you write a good and understandable description of the new feature or change which you have developed.
@@ -38,9 +38,9 @@ Fork the code
 
 In your browser, navigate to: [https://github.com/apache/cloudstack](https://github.com/apache/cloudstack)
 
-Fork the repository by clicking on the 'Fork' button on the top right hand side.  The fork will happen and you will be taken to your own fork of the repository.  Copy the Git repository URL by clicking on the clipboard next to the URL on the right hand side of the page under '**HTTPS** clone URL'.  You will paste this URL when doing the following `git clone` command.
+Fork the repository by clicking on the 'Fork' button on the top right hand side.  The fork will happen, and you will be taken to your own fork of the repository.  Copy the Git repository URL by clicking on the clipboard next to the URL on the right hand side of the page under '**HTTPS** clone URL'.  You will paste this URL when doing the following `git clone` command.
 
-On your computer, follow these steps to setup a local repository for working on ACS:
+On your computer, follow these steps to set up a local repository for working on ACS:
 
 ```bash
 $ git clone https://github.com/YOUR_ACCOUNT/cloudstack.git
@@ -92,9 +92,9 @@ $ git rebase main
 Make a GitHub Pull Request to contribute your changes
 -----------------------------------------------------
 
-When you are happy with your changes and you are ready to contribute them, you will create a Pull Request on GitHub to do so.  This is done by pushing your local changes to your forked repository (default remote name is `origin`) and then initiating a pull request on GitHub.
+When you are happy with your changes, and you are ready to contribute them, you will create a Pull Request on GitHub to do so.  This is done by pushing your local changes to your forked repository (default remote name is `origin`) and then initiating a pull request on GitHub.
 
-Please include JIRA id, detailed information about the bug/feature, what all tests are executed, how the reviewer can test this feature etc. Incase of UI PRs, a screenshot is preferred.
+Please include JIRA id, detailed information about the bug/feature, what all tests are executed, how the reviewer can test this feature etc. In case of UI PRs, a screenshot is preferred.
 
 > **IMPORTANT:** Make sure you have rebased your `feature_x` branch to include the latest code from `upstream/main` _before_ you do this.
 

agent/bindir/cloud-setup-agent.in

@@ -20,6 +20,19 @@ import os
 import logging
 import sys
 import socket
+
+# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ----
+# ---- We do this so cloud_utils can be looked up in the following order:
+# ---- 1) Sources directory
+# ---- 2) waf configured PYTHONDIR
+# ---- 3) System Python path
+for pythonpath in (
+        "@PYTHONDIR@",
+        os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"),
+    ):
+        if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath)
+# ---- End snippet of code ----
+
 from cloudutils.cloudException import CloudRuntimeException, CloudInternalException
 from cloudutils.utilities import initLoging, bash
 from cloudutils.configFileOps import  configFileOps

agent/bindir/libvirtqemuhook.in

@@ -20,6 +20,19 @@ import sys
 import os
 import subprocess
 from threading import Timer
+
+# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ----
+# ---- We do this so cloud_utils can be looked up in the following order:
+# ---- 1) Sources directory
+# ---- 2) waf configured PYTHONDIR
+# ---- 3) System Python path
+for pythonpath in (
+        "@PYTHONDIR@",
+        os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"),
+    ):
+        if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath)
+# ---- End snippet of code ----
+
 from xml.dom.minidom import parse
 from cloudutils.configFileOps import configFileOps
 from cloudutils.networkConfig import networkConfig

api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java

@@ -28,12 +28,11 @@
 import org.apache.cloudstack.api.EntityReference;
 
 import com.cloud.network.Network;
-import com.cloud.projects.ProjectAccount;
 import com.cloud.serializer.Param;
 import com.google.gson.annotations.SerializedName;
 
 @SuppressWarnings("unused")
-@EntityReference(value = {Network.class, ProjectAccount.class})
+@EntityReference(value = {Network.class})
 public class NetworkResponse extends BaseResponseWithAssociatedNetwork implements ControlledEntityResponse, SetResourceIconResponse {
 
     @SerializedName(ApiConstants.ID)

api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java

@@ -149,6 +149,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations {
     @Param(description = "whether this pool is managed or not")
     private Boolean managed;
 
+    @SerializedName(ApiConstants.DETAILS)
+    @Param(description = "the storage pool details")
+    private Map<String, String> details;
+
     public Map<String, String> getCaps() {
         return caps;
     }
@@ -407,4 +411,12 @@ public Boolean getManaged() {
     public void setManaged(Boolean managed) {
         this.managed = managed;
     }
+
+    public Map<String, String> getDetails() {
+        return details;
+    }
+
+    public void setDetails(Map<String, String> details) {
+        this.details = details;
+    }
 }

client/bindir/cloud-setup-management.in

@@ -16,13 +16,27 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import os
 import sys
+# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ----
+# ---- We do this so cloud_utils can be looked up in the following order:
+# ---- 1) Sources directory
+# ---- 2) waf configured PYTHONDIR
+# ---- 3) System Python path
+for pythonpath in (
+        "@PYTHONDIR@",
+        os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"),
+    ):
+        if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath)
+# ---- End snippet of code ----
+
 from cloudutils.syscfg import sysConfigFactory
 from cloudutils.utilities import initLoging, UnknownSystemException
 from cloudutils.cloudException import CloudRuntimeException, CloudInternalException
 from cloudutils.globalEnv import globalEnv
 from cloudutils.serviceConfigServer import cloudManagementConfig
 from optparse import OptionParser
+
 if __name__ == '__main__':
     initLoging("@MSLOGDIR@/setupManagement.log")
     glbEnv = globalEnv()

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImpl.java

@@ -22,23 +22,29 @@
 
 import javax.inject.Inject;
 
-import com.cloud.dc.dao.DataCenterDao;
 import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.DeleteStoragePoolCommand;
+import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.resource.ResourceManager;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.StoragePoolHostVO;
+import com.cloud.storage.VMTemplateStoragePoolVO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
 import com.cloud.storage.dao.StoragePoolHostDao;
+import com.cloud.template.TemplateManager;
 import com.cloud.utils.Pair;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -59,6 +65,10 @@ public class BasePrimaryDataStoreLifeCycleImpl {
     protected DataCenterDao zoneDao;
     @Inject
     protected StoragePoolHostDao storagePoolHostDao;
+    @Inject
+    private PrimaryDataStoreDao primaryDataStoreDao;
+    @Inject
+    private TemplateManager templateMgr;
 
     private List<HostVO> getPoolHostsList(ClusterScope clusterScope, HypervisorType hypervisorType) {
         List<HostVO> hosts;
@@ -81,7 +91,7 @@ public void changeStoragePoolScopeToZone(DataStore store, ClusterScope clusterSc
                 try {
                     storageMgr.connectHostToSharedPool(host, store.getId());
                 } catch (Exception e) {
-                    logger.warn("Unable to establish a connection between " + host + " and " + store, e);
+                    logger.warn("Unable to establish a connection between {} and {}", host, store, e);
                 }
             }
         }
@@ -99,7 +109,7 @@ public void changeStoragePoolScopeToCluster(DataStore store, ClusterScope cluste
 
                 if (answer != null) {
                     if (!answer.getResult()) {
-                        logger.debug("Failed to delete storage pool: " + answer.getResult());
+                        logger.debug("Failed to delete storage pool: {}", answer.getResult());
                     } else if (HypervisorType.KVM != hypervisorType) {
                         break;
                     }
@@ -108,4 +118,42 @@ public void changeStoragePoolScopeToCluster(DataStore store, ClusterScope cluste
         }
         dataStoreHelper.switchToCluster(store, clusterScope);
     }
+
+    private void evictTemplates(StoragePoolVO storagePoolVO) {
+        List<VMTemplateStoragePoolVO> unusedTemplatesInPool = templateMgr.getUnusedTemplatesInPool(storagePoolVO);
+        for (VMTemplateStoragePoolVO templatePoolVO : unusedTemplatesInPool) {
+            if (templatePoolVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
+                templateMgr.evictTemplateFromStoragePool(templatePoolVO);
+            }
+        }
+    }
+
+    private void deleteAgentStoragePools(StoragePool storagePool) {
+        List<StoragePoolHostVO> poolHostVOs = storagePoolHostDao.listByPoolId(storagePool.getId());
+        for (StoragePoolHostVO poolHostVO : poolHostVOs) {
+            DeleteStoragePoolCommand deleteStoragePoolCommand = new DeleteStoragePoolCommand(storagePool);
+            final Answer answer = agentMgr.easySend(poolHostVO.getHostId(), deleteStoragePoolCommand);
+            if (answer != null && answer.getResult()) {
+                logger.info("Successfully deleted storage pool: {} from host: {}", storagePool.getId(), poolHostVO.getHostId());
+            } else {
+                if (answer != null) {
+                    logger.error("Failed to delete storage pool: {} from host: {} , result: {}", storagePool.getId(), poolHostVO.getHostId(), answer.getResult());
+                } else {
+                    logger.error("Failed to delete storage pool: {} from host: {}", storagePool.getId(), poolHostVO.getHostId());
+                }
+            }
+        }
+    }
+
+    protected boolean cleanupDatastore(DataStore store) {
+        StoragePool storagePool = (StoragePool)store;
+        StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(storagePool.getId());
+        if (storagePoolVO == null) {
+            return false;
+        }
+
+        evictTemplates(storagePoolVO);
+        deleteAgentStoragePools(storagePool);
+        return true;
+    }
 }

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java

@@ -43,7 +43,6 @@
 import com.cloud.storage.StoragePoolHostVO;
 import com.cloud.storage.StorageService;
 import com.cloud.storage.dao.StoragePoolHostDao;
-import com.cloud.utils.Pair;
 import com.cloud.utils.exception.CloudRuntimeException;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@@ -60,6 +59,7 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 public class DefaultHostListener implements HypervisorHostListener {
     protected Logger logger = LogManager.getLogger(getClass());
@@ -133,9 +133,11 @@ private NicTO createNicTOFromNetworkAndOffering(NetworkVO networkVO, NetworkOffe
     @Override
     public boolean hostConnect(long hostId, long poolId) throws StorageConflictException {
         StoragePool pool = (StoragePool) this.dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
-        Pair<Map<String, String>, Boolean> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null);
+        Map<String, String> detailsMap = storagePoolDetailsDao.listDetailsKeyPairs(poolId);
+        Map<String, String> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null).first();
 
-        ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool, nfsMountOpts.first());
+        Optional.ofNullable(nfsMountOpts).ifPresent(detailsMap::putAll);
+        ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool, detailsMap);
         cmd.setWait(modifyStoragePoolCommandWait);
         HostVO host = hostDao.findById(hostId);
         logger.debug("Sending modify storage pool command to agent: {} for storage pool: {} with timeout {} seconds", host, pool, cmd.getWait());

framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java

@@ -1113,9 +1113,19 @@ public boolean stop() {
         if (_mshostId != null) {
             final ManagementServerHostVO mshost = _mshostDao.findByMsid(_msId);
             if (mshost != null) {
-                final ManagementServerStatusVO mshostStatus = mshostStatusDao.findByMsId(mshost.getUuid());
-                mshostStatus.setLastJvmStop(new Date());
-                mshostStatusDao.update(mshostStatus.getId(), mshostStatus);
+                ManagementServerStatusVO mshostStatus = mshostStatusDao.findByMsId(mshost.getUuid());
+                if (mshostStatus != null) {
+                    mshostStatus.setLastJvmStop(new Date());
+                    mshostStatusDao.update(mshostStatus.getId(), mshostStatus);
+                } else {
+                    logger.warn("Found a management server host [{}] without a status. This should never happen!", mshost);
+                    mshostStatus = new ManagementServerStatusVO();
+                    mshostStatus.setMsId(mshost.getUuid());
+                    mshostStatus.setLastSystemBoot(new Date());
+                    mshostStatus.setLastJvmStart(new Date());
+                    mshostStatus.setUpdated(new Date());
+                    mshostStatusDao.persist(mshostStatus);
+                }
 
                 ManagementServerHost.State msHostState = ManagementServerHost.State.Down;
                 if (ManagementServerHost.State.Maintenance.equals(mshost.getState()) || ManagementServerHost.State.PreparingForMaintenance.equals(mshost.getState())) {