CPU to Memory weight based algorithm to order cluster

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -90,9 +90,11 @@ public class ApiConstants {
     public static final String CONVERT_INSTANCE_HOST_ID = "convertinstancehostid";
     public static final String CONVERT_INSTANCE_STORAGE_POOL_ID = "convertinstancepoolid";
     public static final String ENABLED_REVOCATION_CHECK = "enabledrevocationcheck";
+    public static final String COMBINED_CAPACITY_ORDERING = "COMBINED";
     public static final String CONTROLLER = "controller";
     public static final String CONTROLLER_UNIT = "controllerunit";
     public static final String COPY_IMAGE_TAGS = "copyimagetags";
+    public static final String CPU_OVERCOMMIT_RATIO = "cpuOvercommitRatio";
     public static final String CSR = "csr";
     public static final String PRIVATE_KEY = "privatekey";
     public static final String DATASTORE_HOST = "datastorehost";
@@ -120,6 +122,7 @@ public class ApiConstants {
     public static final String COMMAND = "command";
     public static final String CMD_EVENT_TYPE = "cmdeventtype";
     public static final String COMPONENT = "component";
+    public static final String CPU = "CPU";
     public static final String CPU_CORE_PER_SOCKET = "cpucorepersocket";
     public static final String CPU_NUMBER = "cpunumber";
     public static final String CPU_SPEED = "cpuspeed";
@@ -338,6 +341,7 @@ public class ApiConstants {
     public static final String MAX_BACKUPS = "maxbackups";
     public static final String MAX_CPU_NUMBER = "maxcpunumber";
     public static final String MAX_MEMORY = "maxmemory";
+    public static final String MEMORY_OVERCOMMIT_RATIO = "memoryOvercommitRatio";
     public static final String MIN_CPU_NUMBER = "mincpunumber";
     public static final String MIN_MEMORY = "minmemory";
     public static final String MIGRATION_TYPE = "migrationtype";
@@ -433,6 +437,7 @@ public class ApiConstants {
     public static final String PUBLIC_END_PORT = "publicendport";
     public static final String PUBLIC_ZONE = "publiczone";
     public static final String PURGE_RESOURCES = "purgeresources";
+    public static final String RAM = "RAM";
     public static final String RECEIVED_BYTES = "receivedbytes";
     public static final String RECONNECT = "reconnect";
     public static final String RECOVER = "recover";

engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java

@@ -65,6 +65,11 @@ public interface ConfigurationManager {
             "allow.non.rfc1918.compliant.ips", "Advanced", "false",
             "Allows non-compliant RFC 1918 IPs for Shared, Isolated networks and VPCs", true, null);
 
+    ConfigKey<Float> HostCapacityTypeCpuMemoryWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Float.class, "host.capacityType.to.order.clusters.cputomemoryweight",
+            "0.5",
+            "CPU to Memory weight used for COMBINED capacityTye to order cluster and host",
+            true, ConfigKey.Scope.Global);
+
     /**
      * @param offering
      * @return

engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java

@@ -30,7 +30,7 @@ public interface CapacityDao extends GenericDao<CapacityVO, Long> {
 
     List<CapacityVO> listByHostIdTypes(Long hostId, List<Short> capacityTypes);
 
-    List<Long> listClustersInZoneOrPodByHostCapacities(long id, long vmId, int requiredCpu, long requiredRam, short capacityTypeForOrdering, boolean isZone);
+    List<Long> listClustersInZoneOrPodByHostCapacities(long id, long vmId, int requiredCpu, long requiredRam, boolean isZone);
 
     List<Long> listHostsWithEnoughCapacity(int requiredCpu, long requiredRam, Long clusterId, String hostType);
 
@@ -48,7 +48,7 @@ public interface CapacityDao extends GenericDao<CapacityVO, Long> {
 
     List<SummedCapacity> findFilteredCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId, List<Long> hostIds, List<Long> poolIds);
 
-    List<Long> listPodsByHostCapacities(long zoneId, int requiredCpu, long requiredRam, short capacityType);
+    List<Long> listPodsByHostCapacities(long zoneId, int requiredCpu, long requiredRam);
 
     Pair<List<Long>, Map<Long, Double>> orderPodsByAggregateCapacity(long zoneId, short capacityType);
 
@@ -65,4 +65,10 @@ List<SummedCapacity> listCapacitiesGroupedByLevelAndType(Integer capacityType, L
     float findClusterConsumption(Long clusterId, short capacityType, long computeRequested);
 
     Pair<List<Long>, Map<Long, Double>> orderHostsByFreeCapacity(Long zoneId, Long clusterId, short capacityType);
+
+    List<CapacityVO> listHostCapacityByCapacityTypes(Long zoneId, Long clusterId, List<Short> capacityTypes);
+
+    List<CapacityVO> listPodCapacityByCapacityTypes(Long zoneId, List<Short> capacityTypes);
+
+    List<CapacityVO> listClusterCapacityByCapacityTypes(Long zoneId, Long podId, long vmId, List<Short> capacityTypes);
 }

engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java

@@ -684,7 +684,7 @@
     }
 
     @Override
-    public List<Long> listClustersInZoneOrPodByHostCapacities(long id, long vmId, int requiredCpu, long requiredRam, short capacityTypeForOrdering, boolean isZone) {
+    public List<Long> listClustersInZoneOrPodByHostCapacities(long id, long vmId, int requiredCpu, long requiredRam, boolean isZone) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();
         PreparedStatement pstmt = null;
         List<Long> result = new ArrayList<Long>();
@@ -1068,7 +1068,65 @@
     }
 
     @Override
-    public List<Long> listPodsByHostCapacities(long zoneId, int requiredCpu, long requiredRam, short capacityType) {
+    public List<CapacityVO> listHostCapacityByCapacityTypes(Long zoneId, Long clusterId, List<Short> capacityTypes) {
+        SearchBuilder<CapacityVO> sb = createSearchBuilder();
+        sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ);
+        sb.and("capacityTypes", sb.entity().getCapacityType(), SearchCriteria.Op.IN);
+        sb.and("capacityState", sb.entity().getCapacityState(), Op.EQ);
+        sb.done();
+
+        SearchCriteria<CapacityVO> sc = sb.create();
+        sc.setParameters("capacityState", "Enabled");
+        if (zoneId != null) {
+            sc.setParameters("zoneId", zoneId);
+        }
+        if (clusterId != null) {
+            sc.setParameters("clusterId", clusterId);
+        }
+        sc.setParameters("capacityTypes", capacityTypes.toArray());
+        return listBy(sc);
+    }
+
+    @Override
+    public List<CapacityVO> listPodCapacityByCapacityTypes(Long zoneId, List<Short> capacityTypes) {
+        SearchBuilder<CapacityVO> sb = createSearchBuilder();
+        sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        sb.and("capacityTypes", sb.entity().getCapacityType(), SearchCriteria.Op.IN);
+        sb.and("capacityState", sb.entity().getCapacityState(), Op.EQ);
+        sb.done();
+        SearchCriteria<CapacityVO> sc = sb.create();
+        sc.setParameters("capacityState", "Enabled");
+        if (zoneId != null) {
+            sc.setParameters("zoneId", zoneId);
+        }
+        sc.setParameters("capacityTypes", capacityTypes.toArray());
+        return listBy(sc);
+    }
+
+    @Override
+    public List<CapacityVO> listClusterCapacityByCapacityTypes(Long zoneId, Long podId, long vmId, List<Short> capacityTypes) {
+        SearchBuilder<CapacityVO> sb = createSearchBuilder();
+        sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ);
+        sb.and("capacityTypes", sb.entity().getCapacityType(), SearchCriteria.Op.IN);
+        sb.and("capacityState", sb.entity().getCapacityState(), Op.EQ);
+        sb.done();
+
+        SearchCriteria<CapacityVO> sc = sb.create();
+        sc.setParameters("capacityState", "Enabled");
+        if (zoneId != null) {
+            sc.setParameters("zoneId", zoneId);
+        }
+        if (podId != null) {
+            sc.setParameters("podId", podId);
+        }
+        sc.setParameters("capacityTypes", capacityTypes.toArray());
+        return listBy(sc);
+    }
+
+    @Override
+    public List<Long> listPodsByHostCapacities(long zoneId, int requiredCpu, long requiredRam) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();
         PreparedStatement pstmt = null;
         List<Long> result = new ArrayList<Long>();

engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql

@@ -166,3 +166,9 @@ SET `sort_key` = CASE
     ELSE `sort_key`
 END;
 -- End: Changes for Guest OS category cleanup
+
+-- Update description for configuration: host.capacityType.to.order.clusters
+UPDATE `cloud`.`configuration` SET
+    `description` = 'The host capacity type (CPU, RAM or COMBINED) is used by deployment planner to order clusters during VM resource allocation'
+WHERE `name` = 'host.capacityType.to.order.clusters'
+  AND `description` = 'The host capacity type (CPU or RAM) is used by deployment planner to order clusters during VM resource allocation';

engine/schema/src/test/java/com/cloud/capacity/dao/CapacityDaoImplTest.java

@@ -205,11 +205,11 @@ public void testListClustersInZoneOrPodByHostCapacitiesEmpty() throws Exception
         when(pstmt.executeQuery()).thenReturn(resultSet);
         when(resultSet.next()).thenReturn(false);
 
-        List<Long> resultZone = capacityDao.listClustersInZoneOrPodByHostCapacities(1L, 123L, 2, 2048L, (short)0, true);
+        List<Long> resultZone = capacityDao.listClustersInZoneOrPodByHostCapacities(1L, 123L, 2, 2048L, true);
         assertNotNull(resultZone);
         assertTrue(resultZone.isEmpty());
 
-        List<Long> resultPod = capacityDao.listClustersInZoneOrPodByHostCapacities(1L, 123L, 2, 2048L, (short)0, false);
+        List<Long> resultPod = capacityDao.listClustersInZoneOrPodByHostCapacities(1L, 123L, 2, 2048L, false);
         assertNotNull(resultPod);
         assertTrue(resultPod.isEmpty());
     }
@@ -281,7 +281,7 @@ public void testListPodsByHostCapacitiesEmptyResult() throws Exception {
         when(pstmt.executeQuery()).thenReturn(resultSet);
         when(resultSet.next()).thenReturn(false);
 
-        List<Long> result = capacityDao.listPodsByHostCapacities(1L, 2, 1024L, (short)0);
+        List<Long> result = capacityDao.listPodsByHostCapacities(1L, 2, 1024L);
         assertNotNull(result);
         assertTrue(result.isEmpty());
     }

plugins/deployment-planners/implicit-dedication/src/test/java/org/apache/cloudstack/implicitplanner/ImplicitPlannerTest.java

@@ -359,7 +359,7 @@ private void initializeForTest(VirtualMachineProfileImpl vmProfile, DataCenterDe
         clustersWithEnoughCapacity.add(3L);
         when(
             capacityDao.listClustersInZoneOrPodByHostCapacities(dataCenterId, 12L, noOfCpusInOffering * cpuSpeedInOffering, ramInOffering * 1024L * 1024L,
-                Capacity.CAPACITY_TYPE_CPU, true)).thenReturn(clustersWithEnoughCapacity);
+                    true)).thenReturn(clustersWithEnoughCapacity);
 
         Map<Long, Double> clusterCapacityMap = new HashMap<Long, Double>();
         clusterCapacityMap.put(1L, 2048D);

server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java

@@ -30,10 +30,12 @@
 import javax.naming.ConfigurationException;
 
 import com.cloud.agent.manager.allocator.HostAllocator;
+import com.cloud.capacity.Capacity;
 import com.cloud.capacity.CapacityManager;
 import com.cloud.capacity.CapacityVO;
 import com.cloud.capacity.dao.CapacityDao;
 import com.cloud.configuration.Config;
+import com.cloud.configuration.ConfigurationManager;
 import com.cloud.dc.ClusterDetailsDao;
 import com.cloud.dc.dao.ClusterDao;
 import com.cloud.deploy.DeploymentPlan;
@@ -64,9 +66,11 @@
 import com.cloud.vm.dao.UserVmDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
 
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 
+import org.jetbrains.annotations.NotNull;
 import org.springframework.stereotype.Component;
 
 /**
@@ -295,7 +299,7 @@
             Collections.shuffle(hosts);
         } else if (vmAllocationAlgorithm.equals("userdispersing")) {
             hosts = reorderHostsByNumberOfVms(plan, hosts, account);
-        }else if(vmAllocationAlgorithm.equals("firstfitleastconsumed")){
+        } else if(vmAllocationAlgorithm.equals("firstfitleastconsumed")){
             hosts = reorderHostsByCapacity(plan, hosts);
         }
 
@@ -372,13 +376,7 @@
     private List<? extends Host> reorderHostsByCapacity(DeploymentPlan plan, List<? extends Host> hosts) {
         Long zoneId = plan.getDataCenterId();
         Long clusterId = plan.getClusterId();
-        //Get capacity by which we should reorder
-        String capacityTypeToOrder = _configDao.getValue(Config.HostCapacityTypeToOrderClusters.key());
-        short capacityType = CapacityVO.CAPACITY_TYPE_CPU;
-        if("RAM".equalsIgnoreCase(capacityTypeToOrder)){
-            capacityType = CapacityVO.CAPACITY_TYPE_MEMORY;
-        }
-        Pair<List<Long>, Map<Long, Double>> result = _capacityDao.orderHostsByFreeCapacity(zoneId, clusterId, capacityType);
+        Pair<List<Long>, Map<Long, Double>> result = getOrderedHostsByCapacity(zoneId, clusterId);
         List<Long> hostIdsByFreeCapacity = result.first();
         Map<Long, String> sortedHostByCapacity = result.second().entrySet()
                 .stream()
@@ -407,6 +405,54 @@
         return reorderedHosts;
     }
 
+    private Pair<List<Long>, Map<Long, Double>> getOrderedHostsByCapacity(Long zoneId, Long clusterId) {
+        double cpuToMemoryWeight = ConfigurationManager.HostCapacityTypeCpuMemoryWeight.value();
+        // Get capacity by which we should reorder
+        short capacityType = getCapacityType(cpuToMemoryWeight);
+        if (capacityType >= 0) { // for CPU or RAM
+            return _capacityDao.orderHostsByFreeCapacity(zoneId, clusterId, capacityType);
+        }
+        List<CapacityVO> capacities = _capacityDao.listHostCapacityByCapacityTypes(zoneId, clusterId,
+                List.of(Capacity.CAPACITY_TYPE_CPU, Capacity.CAPACITY_TYPE_MEMORY));
+        Map<Long, Double> hostByComputedCapacity = getHostByCombinedCapacities(capacities, cpuToMemoryWeight);
+        return new Pair<>(new ArrayList<>(hostByComputedCapacity.keySet()), hostByComputedCapacity);
+    }
+
+    short getCapacityType(double cpuToMemoryWeight) {
+        String capacityTypeToOrder = _configDao.getValue(Config.HostCapacityTypeToOrderClusters.key());
+        short capacityType = CapacityVO.CAPACITY_TYPE_CPU;
+        if (ApiConstants.RAM.equalsIgnoreCase(capacityTypeToOrder)){
+            capacityType = CapacityVO.CAPACITY_TYPE_MEMORY;
+        } else if (ApiConstants.COMBINED_CAPACITY_ORDERING.equalsIgnoreCase(capacityTypeToOrder)) {
+            capacityType = -1;
+        }
+
+        if (cpuToMemoryWeight == 1) {
+            capacityType = CapacityVO.CAPACITY_TYPE_CPU;
+        }
+        if (cpuToMemoryWeight == 0) {
+            capacityType = CapacityVO.CAPACITY_TYPE_MEMORY;
+        }
+        return capacityType;
+    }
+
+
+    @NotNull
+    private static Map<Long, Double> getHostByCombinedCapacities(List<CapacityVO> capacities, double cpuToMemoryWeight) {
+        Map<Long, Double> hostByComputedCapacity = new HashMap<>();
+        for (CapacityVO capacityVO : capacities) {
+            long hostId = capacityVO.getHostOrPoolId();
+            double applicableWeight = capacityVO.getCapacityType() == Capacity.CAPACITY_TYPE_CPU ? cpuToMemoryWeight : 1 - cpuToMemoryWeight;
+            double capacityMetric = applicableWeight * (capacityVO.getTotalCapacity() - (capacityVO.getUsedCapacity() + capacityVO.getReservedCapacity()))/capacityVO.getTotalCapacity();
+            hostByComputedCapacity.merge(hostId, capacityMetric, Double::sum);
+        }
+
+        return hostByComputedCapacity.entrySet()
+                .stream()
+                .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder()))
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (e1, e2) -> e1, LinkedHashMap::new));
+    }
+
     private List<? extends Host> reorderHostsByNumberOfVms(DeploymentPlan plan, List<? extends Host> hosts, Account account) {
         if (account == null) {
             return hosts;

server/src/main/java/com/cloud/configuration/Config.java

@@ -897,8 +897,9 @@ public enum Config {
             String.class,
             "host.capacityType.to.order.clusters",
             "CPU",
-            "The host capacity type (CPU or RAM) is used by deployment planner to order clusters during VM resource allocation",
-            "CPU,RAM"),
+            "The host capacity type (CPU, RAM, COMBINED) is used by deployment planner to order clusters during VM resource allocation",
+            "CPU,RAM,COMBINED"),
+
     ApplyAllocationAlgorithmToPods(
             "Advanced",
             ManagementServer.class,

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -8252,7 +8252,7 @@ public ConfigKey<?>[] getConfigKeys() {
                 BYTES_MAX_READ_LENGTH, BYTES_MAX_WRITE_LENGTH, ADD_HOST_ON_SERVICE_RESTART_KVM, SET_HOST_DOWN_TO_MAINTENANCE,
                 VM_SERVICE_OFFERING_MAX_CPU_CORES, VM_SERVICE_OFFERING_MAX_RAM_SIZE, MIGRATE_VM_ACROSS_CLUSTERS,
                 ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN, ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN,
-                ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS, DELETE_QUERY_BATCH_SIZE, AllowNonRFC1918CompliantIPs
+                ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS, DELETE_QUERY_BATCH_SIZE, AllowNonRFC1918CompliantIPs, HostCapacityTypeCpuMemoryWeight
         };
     }