Merge branch '4.20' into fix-arm-watchdog

nvazquez · nvazquez · commit 1b43424e8c46 · 2025-01-21T21:55:19.000-03:00
diff --git a/client/conf/server.properties.in b/client/conf/server.properties.in
@@ -32,6 +32,9 @@ session.timeout=30
 # Max allowed API request payload/content size in bytes
 request.content.size=1048576
 
+# Max allowed API request form keys
+request.max.form.keys=5000
+
 # Options to configure and enable HTTPS on the management server
 #
 # For the management server to pick up these configuration settings, the configured
diff --git a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java
@@ -82,6 +82,8 @@ public class ServerDaemon implements Daemon {
     private static final String ACCESS_LOG = "access.log";
     private static final String REQUEST_CONTENT_SIZE_KEY = "request.content.size";
     private static final int DEFAULT_REQUEST_CONTENT_SIZE = 1048576;
+    private static final String REQUEST_MAX_FORM_KEYS_KEY = "request.max.form.keys";
+    private static final int DEFAULT_REQUEST_MAX_FORM_KEYS = 5000;
 
     ////////////////////////////////////////////////////////
     /////////////// Server Configuration ///////////////////
@@ -94,6 +96,7 @@ public class ServerDaemon implements Daemon {
     private int httpsPort = 8443;
     private int sessionTimeout = 30;
     private int maxFormContentSize = DEFAULT_REQUEST_CONTENT_SIZE;
+    private int maxFormKeys = DEFAULT_REQUEST_MAX_FORM_KEYS;
     private boolean httpsEnable = false;
     private String accessLogFile = "access.log";
     private String bindInterface = null;
@@ -141,6 +144,7 @@ public void init(final DaemonContext context) {
             setAccessLogFile(properties.getProperty(ACCESS_LOG, "access.log"));
             setSessionTimeout(Integer.valueOf(properties.getProperty(SESSION_TIMEOUT, "30")));
             setMaxFormContentSize(Integer.valueOf(properties.getProperty(REQUEST_CONTENT_SIZE_KEY, String.valueOf(DEFAULT_REQUEST_CONTENT_SIZE))));
+            setMaxFormKeys(Integer.valueOf(properties.getProperty(REQUEST_MAX_FORM_KEYS_KEY, String.valueOf(DEFAULT_REQUEST_MAX_FORM_KEYS))));
         } catch (final IOException e) {
             logger.warn("Failed to read configuration from server.properties file", e);
         } finally {
@@ -192,6 +196,7 @@ public void start() throws Exception {
         // Extra config options
         server.setStopAtShutdown(true);
         server.setAttribute(ContextHandler.MAX_FORM_CONTENT_SIZE_KEY, maxFormContentSize);
+        server.setAttribute(ContextHandler.MAX_FORM_KEYS_KEY, maxFormKeys);
 
         // HTTPS Connector
         createHttpsConnector(httpConfig);
@@ -264,6 +269,7 @@ private Pair<SessionHandler,HandlerCollection> createHandlers() {
         webApp.setContextPath(contextPath);
         webApp.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
         webApp.setMaxFormContentSize(maxFormContentSize);
+        webApp.setMaxFormKeys(maxFormKeys);
 
         // GZIP handler
         final GzipHandler gzipHandler = new GzipHandler();
@@ -366,4 +372,8 @@ public void setSessionTimeout(int sessionTimeout) {
     public void setMaxFormContentSize(int maxFormContentSize) {
         this.maxFormContentSize = maxFormContentSize;
     }
+
+    public void setMaxFormKeys(int maxFormKeys) {
+        this.maxFormKeys = maxFormKeys;
+    }
 }
diff --git a/engine/schema/src/main/java/org/apache/cloudstack/network/dao/NetworkPermissionDao.java b/engine/schema/src/main/java/org/apache/cloudstack/network/dao/NetworkPermissionDao.java
@@ -40,6 +40,13 @@ public interface NetworkPermissionDao extends GenericDao<NetworkPermissionVO, Lo
      */
     void removeAllPermissions(long networkId);
 
+    /**
+     * Removes all network permissions associated with a given account.
+     *
+     * @param accountId The ID of the account from which all network permissions will be removed.
+     */
+    void removeAccountPermissions(long accountId);
+
     /**
      * Find a Network permission by networkId, accountName, and domainId
      *
diff --git a/engine/schema/src/main/java/org/apache/cloudstack/network/dao/NetworkPermissionDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/network/dao/NetworkPermissionDaoImpl.java
@@ -33,6 +33,7 @@ public class NetworkPermissionDaoImpl extends GenericDaoBase<NetworkPermissionVO
 
     private SearchBuilder<NetworkPermissionVO> NetworkAndAccountSearch;
     private SearchBuilder<NetworkPermissionVO> NetworkIdSearch;
+    private SearchBuilder<NetworkPermissionVO> accountSearch;
     private GenericSearchBuilder<NetworkPermissionVO, Long> FindNetworkIdsByAccount;
 
     protected NetworkPermissionDaoImpl() {
@@ -45,6 +46,10 @@ protected NetworkPermissionDaoImpl() {
         NetworkIdSearch.and("networkId", NetworkIdSearch.entity().getNetworkId(), SearchCriteria.Op.EQ);
         NetworkIdSearch.done();
 
+        accountSearch = createSearchBuilder();
+        accountSearch.and("accountId", accountSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
+        accountSearch.done();
+
         FindNetworkIdsByAccount = createSearchBuilder(Long.class);
         FindNetworkIdsByAccount.select(null, SearchCriteria.Func.DISTINCT, FindNetworkIdsByAccount.entity().getNetworkId());
         FindNetworkIdsByAccount.and("account", FindNetworkIdsByAccount.entity().getAccountId(), SearchCriteria.Op.IN);
@@ -69,6 +74,16 @@ public void removeAllPermissions(long networkId) {
         expunge(sc);
     }
 
+    @Override
+    public void removeAccountPermissions(long accountId) {
+        SearchCriteria<NetworkPermissionVO> sc = accountSearch.create();
+        sc.setParameters("accountId", accountId);
+        int networkPermissionRemoved = expunge(sc);
+        if (networkPermissionRemoved > 0) {
+            logger.debug(String.format("Removed [%s] network permission(s) for the account with Id [%s]", networkPermissionRemoved, accountId));
+        }
+    }
+
     @Override
     public NetworkPermissionVO findByNetworkAndAccount(long networkId, long accountId) {
         SearchCriteria<NetworkPermissionVO> sc = NetworkAndAccountSearch.create();
diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/api/ListSystemVMsUsageHistoryCmd.java b/plugins/metrics/src/main/java/org/apache/cloudstack/api/ListSystemVMsUsageHistoryCmd.java
@@ -26,7 +26,7 @@
 
 @APICommand(name = "listSystemVmsUsageHistory", description = "Lists System VM stats", responseObject = VmMetricsStatsResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.18.0",
-        authorized = {RoleType.Admin,  RoleType.ResourceAdmin, RoleType.DomainAdmin})
+        authorized = {RoleType.Admin})
 public class ListSystemVMsUsageHistoryCmd extends BaseResourceUsageHistoryCmd {
 
     /////////////////////////////////////////////////////
diff --git a/plugins/storage/volume/linstor/CHANGELOG.md b/plugins/storage/volume/linstor/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to Linstor CloudStack plugin will be documented in this file
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2025-01-20]
+
+### Fixed
+
+- Volume snapshots on zfs used the wrong dataset path to hide/unhide snapdev
+
 ## [2024-12-13]
 
 ### Fixed
diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java
@@ -46,11 +46,17 @@ public final class LinstorBackupSnapshotCommandWrapper
 {
     protected static Logger LOGGER = LogManager.getLogger(LinstorBackupSnapshotCommandWrapper.class);
 
+    private static String zfsDatasetName(String zfsFullSnapshotUrl) {
+        String zfsFullPath = zfsFullSnapshotUrl.substring(6);
+        int atPos = zfsFullPath.indexOf('@');
+        return atPos >= 0 ? zfsFullPath.substring(0, atPos) : zfsFullPath;
+    }
+
     private String zfsSnapdev(boolean hide, String zfsUrl) {
-        Script script = new Script("/usr/bin/zfs", Duration.millis(5000));
+        Script script = new Script("zfs", Duration.millis(5000));
         script.add("set");
         script.add("snapdev=" + (hide ? "hidden" : "visible"));
-        script.add(zfsUrl.substring(6));  // cutting zfs://
+        script.add(zfsDatasetName(zfsUrl));  // cutting zfs:// and @snapshotname
         return script.execute();
     }
 
@@ -134,10 +140,10 @@ public CopyCmdAnswer execute(LinstorBackupSnapshotCommand cmd, LibvirtComputingR
             LOGGER.info("Src: " + srcPath + " | " + src.getName());
             if (srcPath.startsWith("zfs://")) {
                 zfsHidden = true;
-                if (zfsSnapdev(false, srcPath) != null) {
+                if (zfsSnapdev(false, src.getPath()) != null) {
                     return new CopyCmdAnswer("Unable to unhide zfs snapshot device.");
                 }
-                srcPath = "/dev/" + srcPath.substring(6);
+                srcPath = "/dev/zvol/" + srcPath.substring(6);
             }
 
             secondaryPool = storagePoolMgr.getStoragePoolByURI(dstDataStore.getUrl());
diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
@@ -1407,6 +1407,7 @@ private Pair<List<Long>, Integer> searchForUserVMIdsAndCount(ListVMsCmd cmd) {
         if (networkId != null || vpcId != null) {
             SearchBuilder<NicVO> nicSearch = nicDao.createSearchBuilder();
             nicSearch.and("networkId", nicSearch.entity().getNetworkId(), Op.EQ);
+            nicSearch.and("removed", nicSearch.entity().getRemoved(), Op.NULL);
             if (vpcId != null) {
                 SearchBuilder<NetworkVO> networkSearch = networkDao.createSearchBuilder();
                 networkSearch.and("vpcId", networkSearch.entity().getVpcId(), Op.EQ);
diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java
@@ -1473,8 +1473,10 @@ private void migrateAwayVmWithVolumes(HostVO host, VMInstanceVO vm) {
         final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, null, offeringVO, null, null);
         plan.setMigrationPlan(true);
         DeployDestination dest = null;
+        DeploymentPlanner.ExcludeList avoids = new DeploymentPlanner.ExcludeList();
+        avoids.addHost(host.getId());
         try {
-            dest = deploymentManager.planDeployment(profile, plan, new DeploymentPlanner.ExcludeList(), null);
+            dest = deploymentManager.planDeployment(profile, plan, avoids, null);
         } catch (InsufficientServerCapacityException e) {
             throw new CloudRuntimeException(String.format("Maintenance failed, could not find deployment destination for VM: %s.", vm), e);
         }
diff --git a/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java b/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java
@@ -37,6 +37,7 @@
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.ObjectUtils;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
@@ -64,12 +65,16 @@
 import com.cloud.service.ServiceOfferingVO;
 import com.cloud.service.dao.ServiceOfferingDao;
 import com.cloud.utils.Pair;
+import com.cloud.utils.StringUtils;
 import com.cloud.utils.Ternary;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.UserVmDetailVO;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine.State;
 import com.cloud.vm.VirtualMachineProfileImpl;
+import com.cloud.vm.VmDetailConstants;
+import com.cloud.vm.dao.UserVmDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
 
 public class RollingMaintenanceManagerImpl extends ManagerBase implements RollingMaintenanceManager {
@@ -85,6 +90,8 @@ public class RollingMaintenanceManagerImpl extends ManagerBase implements Rollin
     @Inject
     private VMInstanceDao vmInstanceDao;
     @Inject
+    protected UserVmDetailsDao userVmDetailsDao;
+    @Inject
     private ServiceOfferingDao serviceOfferingDao;
     @Inject
     private ClusterDetailsDao clusterDetailsDao;
@@ -619,10 +626,19 @@ private Pair<Boolean, String> performCapacityChecksBeforeHostInMaintenance(Host
         int successfullyCheckedVmMigrations = 0;
         for (VMInstanceVO runningVM : vmsRunning) {
             boolean canMigrateVm = false;
+            Ternary<Integer, Integer, Integer> cpuSpeedAndRamSize = getComputeResourcesCpuSpeedAndRamSize(runningVM);
+            Integer cpu = cpuSpeedAndRamSize.first();
+            Integer speed = cpuSpeedAndRamSize.second();
+            Integer ramSize = cpuSpeedAndRamSize.third();
+            if (ObjectUtils.anyNull(cpu, speed, ramSize)) {
+                logger.warn("Cannot fetch compute resources for the VM {}, skipping it from the capacity check", runningVM);
+                continue;
+            }
+
             ServiceOfferingVO serviceOffering = serviceOfferingDao.findById(runningVM.getServiceOfferingId());
             for (Host hostInCluster : hostsInCluster) {
                 if (!checkHostTags(hostTags, hostTagsDao.getHostTags(hostInCluster.getId()), serviceOffering.getHostTag())) {
-                    logger.debug(String.format("Host tags mismatch between %s and %s Skipping it from the capacity check", host, hostInCluster));
+                    logger.debug("Host tags mismatch between {} and {} Skipping it from the capacity check", host, hostInCluster);
                     continue;
                 }
                 DeployDestination deployDestination = new DeployDestination(null, null, null, host);
@@ -632,13 +648,13 @@ private Pair<Boolean, String> performCapacityChecksBeforeHostInMaintenance(Host
                     affinityChecks = affinityChecks && affinityProcessor.check(vmProfile, deployDestination);
                 }
                 if (!affinityChecks) {
-                    logger.debug(String.format("Affinity check failed between %s and %s Skipping it from the capacity check", host, hostInCluster));
+                    logger.debug("Affinity check failed between {} and {} Skipping it from the capacity check", host, hostInCluster);
                     continue;
                 }
                 boolean maxGuestLimit = capacityManager.checkIfHostReachMaxGuestLimit(host);
-                boolean hostHasCPUCapacity = capacityManager.checkIfHostHasCpuCapability(hostInCluster.getId(), serviceOffering.getCpu(), serviceOffering.getSpeed());
-                int cpuRequested = serviceOffering.getCpu() * serviceOffering.getSpeed();
-                long ramRequested = serviceOffering.getRamSize() * 1024L * 1024L;
+                boolean hostHasCPUCapacity = capacityManager.checkIfHostHasCpuCapability(hostInCluster.getId(), cpu, speed);
+                int cpuRequested = cpu * speed;
+                long ramRequested = ramSize * 1024L * 1024L;
                 ClusterDetailsVO clusterDetailsCpuOvercommit = clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio");
                 ClusterDetailsVO clusterDetailsRamOvercommmt = clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio");
                 Float cpuOvercommitRatio = Float.parseFloat(clusterDetailsCpuOvercommit.getValue());
@@ -664,11 +680,42 @@ private Pair<Boolean, String> performCapacityChecksBeforeHostInMaintenance(Host
         return new Pair<>(true, "OK");
     }
 
+    protected Ternary<Integer, Integer, Integer> getComputeResourcesCpuSpeedAndRamSize(VMInstanceVO runningVM) {
+        ServiceOfferingVO serviceOffering = serviceOfferingDao.findById(runningVM.getServiceOfferingId());
+        Integer cpu = serviceOffering.getCpu();
+        Integer speed = serviceOffering.getSpeed();
+        Integer ramSize = serviceOffering.getRamSize();
+        if (!serviceOffering.isDynamic()) {
+            return new Ternary<>(cpu, speed, ramSize);
+        }
+
+        List<UserVmDetailVO> vmDetails = userVmDetailsDao.listDetails(runningVM.getId());
+        if (CollectionUtils.isEmpty(vmDetails)) {
+            return new Ternary<>(cpu, speed, ramSize);
+        }
+
+        for (UserVmDetailVO vmDetail : vmDetails) {
+            if (StringUtils.isBlank(vmDetail.getName()) || StringUtils.isBlank(vmDetail.getValue())) {
+                continue;
+            }
+
+            if (cpu == null && VmDetailConstants.CPU_NUMBER.equals(vmDetail.getName())) {
+                cpu = Integer.valueOf(vmDetail.getValue());
+            } else if (speed == null && VmDetailConstants.CPU_SPEED.equals(vmDetail.getName())) {
+                speed = Integer.valueOf(vmDetail.getValue());
+            } else if (ramSize == null && VmDetailConstants.MEMORY.equals(vmDetail.getName())) {
+                ramSize = Integer.valueOf(vmDetail.getValue());
+            }
+        }
+
+        return new Ternary<>(cpu, speed, ramSize);
+    }
+
     /**
      * Check hosts tags
      */
     private boolean checkHostTags(List<HostTagVO> hostTags, List<HostTagVO> hostInClusterTags, String offeringTag) {
-        if (CollectionUtils.isEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) {
+        if ((CollectionUtils.isEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) || StringUtils.isBlank(offeringTag)) {
             return true;
         } else if ((CollectionUtils.isNotEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) ||
                 (CollectionUtils.isEmpty(hostTags) && CollectionUtils.isNotEmpty(hostInClusterTags))) {
diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java
@@ -76,6 +76,7 @@
 import org.apache.cloudstack.framework.messagebus.PublishScope;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.network.RoutedIpv4Manager;
+import org.apache.cloudstack.network.dao.NetworkPermissionDao;
 import org.apache.cloudstack.region.gslb.GlobalLoadBalancerRuleDao;
 import org.apache.cloudstack.resourcedetail.UserDetailVO;
 import org.apache.cloudstack.resourcedetail.dao.UserDetailsDao;
@@ -303,6 +304,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
     private SSHKeyPairDao _sshKeyPairDao;
     @Inject
     private UserDataDao userDataDao;
+    @Inject
+    private NetworkPermissionDao networkPermissionDao;
 
     private List<QuerySelector> _querySelectors;
 
@@ -898,6 +901,9 @@ protected boolean cleanupAccount(AccountVO account, long callerUserId, Account c
             // delete the account from project accounts
             _projectAccountDao.removeAccountFromProjects(accountId);
 
+            // Delete account's network permissions
+            networkPermissionDao.removeAccountPermissions(accountId);
+
             if (account.getType() != Account.Type.PROJECT) {
                 // delete the account from group
                 _messageBus.publish(_name, MESSAGE_REMOVE_ACCOUNT_EVENT, PublishScope.LOCAL, accountId);
@@ -1943,22 +1949,23 @@ public boolean deleteUserAccount(long accountId) {
             return true;
         }
 
-        // Account that manages project(s) can't be removed
-        List<Long> managedProjectIds = _projectAccountDao.listAdministratedProjectIds(accountId);
-        if (!managedProjectIds.isEmpty()) {
-            StringBuilder projectIds = new StringBuilder();
-            for (Long projectId : managedProjectIds) {
-                projectIds.append(projectId).append(", ");
-            }
-
-            throw new InvalidParameterValueException(String.format("The account %s with id %d manages project(s) with ids %s and can't be removed", account, accountId, projectIds));
-        }
+        checkIfAccountManagesProjects(accountId);
 
         CallContext.current().putContextParameter(Account.class, account.getUuid());
 
         return deleteAccount(account, callerUserId, caller);
     }
 
+    protected void checkIfAccountManagesProjects(long accountId) {
+        List<Long> managedProjectIds = _projectAccountDao.listAdministratedProjectIds(accountId);
+        if (!CollectionUtils.isEmpty(managedProjectIds)) {
+            throw new InvalidParameterValueException(String.format(
+                    "Unable to delete account [%s], because it manages the following project(s): %s. Please, remove the account from these projects or demote it to a regular project role first.",
+                    accountId, managedProjectIds
+            ));
+        }
+    }
+
     protected boolean isDeleteNeeded(AccountVO account, long accountId, Account caller) {
         if (account == null) {
             logger.info(String.format("The account, identified by id %d, doesn't exist", accountId ));
diff --git a/server/src/test/java/com/cloud/resource/RollingMaintenanceManagerImplTest.java b/server/src/test/java/com/cloud/resource/RollingMaintenanceManagerImplTest.java
diff --git a/server/src/test/java/com/cloud/user/AccountManagerImplTest.java b/server/src/test/java/com/cloud/user/AccountManagerImplTest.java
diff --git a/server/src/test/java/com/cloud/user/AccountManagetImplTestBase.java b/server/src/test/java/com/cloud/user/AccountManagetImplTestBase.java
diff --git a/ui/src/views/setting/ConfigurationValue.vue b/ui/src/views/setting/ConfigurationValue.vue

Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,9 @@ session.timeout=30`
`32`	`32`	`# Max allowed API request payload/content size in bytes`
`33`	`33`	`request.content.size=1048576`
`34`	`34`
	`35`	`+# Max allowed API request form keys`
	`36`	`+request.max.form.keys=5000`
	`37`	`+`
`35`	`38`	`# Options to configure and enable HTTPS on the management server`
`36`	`39`	`#`
`37`	`40`	`# For the management server to pick up these configuration settings, the configured`