Calculate memory_mb_max_unit based on instance's memory_mb

Previously the memory_mb_max_unit was relying on the
overallMemoryUsage reported by the ESXi, which was wrong, because
the host could still have a big VM with reserved memory, that was
not consuming any RAM and thus not reflected in the memory usage.

We are now looking for the instances spawned on that ESXi and
subtract their configured memory_mb from the total RAM capacity
of the host, resulting in a realistic memory_mb_max_unit.

Change-Id: I72821c97d80f9f675dca943d65f94e36824d81b6

Author: leust

nova/tests/unit/virt/vmwareapi/test_driver_api.py

@@ -2377,8 +2377,8 @@ def test_get_available_resource(self):
                 stats['supported_instances'])
         memory_mbmax_unit = stats['stats']['memory_mb_max_unit']
         vcpus_max_unit = stats['stats']['vcpus_max_unit']
-        # memory_mb_used = fake.HostSystem.overallMemoryUsage = 500
-        self.assertEqual(memory_mbmax_unit, 1024 - 500)
+        # memory_mb_used = fake.HostSystem.hardware.memorySize = 1024
+        self.assertEqual(memory_mbmax_unit, 1024)
         # vcpus_max_unit = fake.HostSystem.summary.hardware.numCpuThreads
         self.assertEqual(vcpus_max_unit, 16)
 
@@ -2910,3 +2910,24 @@ def test_host_state_service_disabled(self, mock_service):
             self.assertEqual(2, mock_save.call_count)
             self.assertFalse(service.disabled)
             self.assertFalse(self.conn._vc_state._auto_service_disabled)
+
+    @mock.patch.object(vmops.VMwareVMOps,
+                       'get_available_memory_per_host')
+    def test_get_cluster_stats(self, mock_get_mem_per_host):
+        mem_per_host = {
+            'host-1': 1024,
+            'host-2': 4096,
+            'host-3': 0,
+        }
+        mock_get_mem_per_host.return_value = mem_per_host
+
+        host_stats = {
+            'node-001': {'vcpus': 8},
+            'node-002': {'vcpus': 16},
+            self.conn._nodename: {'vcpus': 24}
+        }
+
+        result = self.conn._get_cluster_stats(host_stats)
+
+        self.assertEqual(16, result['vcpus_max_unit'])
+        self.assertEqual(4096, result['memory_mb_max_unit'])

nova/tests/unit/virt/vmwareapi/test_vmops.py

@@ -4203,3 +4203,83 @@ def test_stack_vm_to_host_if_needed_equal_hosts(
         # were returned into is not changed
         self._test_stack_vm_to_host_if_needed(
             requested_mb, hosts, expected)
+
+    @ddt.unpack
+    @ddt.data(
+        # test 1
+        ([
+             {'host': dict(memory_size=1024),
+              'instances_mem': [512]},
+             {'host': dict(memory_size=4096,
+                           in_maintenance_mode=True),
+              'instances_mem': []},
+             {'host': dict(memory_size=8192,
+                           connection_state='N/A'),
+              'instances_mem': []}
+         ],
+         # available memory per host
+         # expect only hosts in valid state to be returned
+         [512]),
+
+        # test 2
+        ([
+             {'host': dict(memory_size=1024),
+              'instances_mem': [256, 256, 256]},
+             {'host': dict(memory_size=1024),
+              'instances_mem': [512, 512, 512]}
+         ],
+         # available memory per host
+         # second host is over-provisioned, but we should still
+         # return 0 available memory MB.
+         [256, 0])
+
+    )
+    def test_get_available_memory_per_host(self,
+                                           hosts_with_instances,
+                                           expected_results):
+        def _fake_host(in_maintenance_mode=False,
+                       connection_state="connected",
+                       memory_size=0):
+            host = vmwareapi_fake.HostSystem()
+            host.set("summary.runtime", mock.Mock(
+                inMaintenanceMode=in_maintenance_mode,
+                connectionState=connection_state))
+            host.set("summary.hardware", mock.Mock(
+                memorySize=memory_size * units.Mi))
+            return host
+
+        hosts = []
+        instances_ret = []
+
+        for hi in hosts_with_instances:
+            host = _fake_host(**hi['host'])
+            hosts.append(host)
+            for mem in hi['instances_mem']:
+                instances_ret.append(
+                    (uuidutils.generate_uuid(),
+                     {'runtime.host': host.mo_id,
+                      'config.hardware.memoryMB': mem}))
+
+        hosts_ret = (
+            [h.mo_id for h in hosts],
+            None
+        )
+
+        def _mock_with_ret(vim, ret_res):
+            return mock.Mock(__enter__=mock.Mock(return_value=ret_res),
+                             __exit__=mock.Mock(return_value=None))
+
+        with test.nested(
+                mock.patch.object(self._session, "_call_method",
+                                  return_value=hosts),
+                mock.patch.object(vutil, 'WithRetrieval', _mock_with_ret),
+                mock.patch.object(self._vmops, '_list_instances_in_cluster',
+                                  return_value=instances_ret),
+                mock.patch.object(vm_util,
+                                  'get_hosts_and_reservations_for_cluster',
+                                  return_value=hosts_ret)
+        ) as (mock_call_method, mock_with_retrieval, mock_list_instances,
+              mock_get_hosts_and_res):
+
+            result = self._vmops.get_available_memory_per_host()
+            self.assertEqual(list(result.values()), expected_results)

nova/virt/vmwareapi/driver.py

@@ -405,7 +405,11 @@ def _get_cluster_stats(self, host_stats):
         # memory_mb_max_unit represents the free memory of the freest host
         # available in the cluster. Can be used by the scheduler to determine
         # if the cluster can accommodate a big VM.
-        memory_mb_max_unit = 0
+        # It's calculated by looking at the configured memoryMB of instances
+        # in that host, not taking into account the actual memory usage.
+        memory_mb_max_unit = max(
+            mem for mem in
+            self._vmops.get_available_memory_per_host().values())
         # vcpus_max_unit represents the vCPUs of a physical node from the
         # cluster. It can be used to determine if a VM can be powered-on in
         # this cluster, by comparing its flavor.vcpus to this value.
@@ -414,11 +418,6 @@ def _get_cluster_stats(self, host_stats):
             # Skip the cluster node
             if nodename == self._nodename:
                 continue
-            memory_mb_free = (resources['memory_mb'] -
-                              resources['memory_mb_used'])
-            if memory_mb_max_unit < memory_mb_free:
-                memory_mb_max_unit = memory_mb_free
-
             vcpus = resources['vcpus']
             if vcpus_max_unit < vcpus:
                 vcpus_max_unit = vcpus

nova/virt/vmwareapi/vmops.py

@@ -4808,3 +4808,56 @@ def check_can_live_migrate_source(self, instance):
                 raise exception.MigrationPreCheckError(
                         reason=("Found non-configdrive CD-ROM. "
                                 "Can only migrate configdrive"))
+
+    def get_available_memory_per_host(self):
+        """Retrieves per-host available RAM.
+
+        The value is calculated by subtracting the total configured
+        memory_mb of the instances residing on that host from the
+        total memory size of the host.
+
+        Returns a dict containing available RAM information per host.
+        {
+            "host-ref-value": 1024
+        }
+        """
+        (host_mors, _) = vm_util.get_hosts_and_reservations_for_cluster(
+                            self._session, self._cluster)
+
+        ram_per_host = {}
+        # initialize ram_per_host with the hosts in the expected state
+        # and their total memorySize.
+        result = self._session._call_method(vim_util,
+                            "get_properties_for_a_collection_of_objects",
+                            "HostSystem", host_mors,
+                            ["summary.runtime", "summary.hardware"])
+        with vutil.WithRetrieval(self._session.vim, result) as objects:
+            for obj in objects:
+                host_props = vutil.propset_dict(obj.propSet)
+                runtime_summary = host_props['summary.runtime']
+                if (runtime_summary.inMaintenanceMode or
+                        runtime_summary.connectionState != "connected"):
+                    continue
+                host_ref_value = vutil.get_moref_value(obj.obj)
+                hardware_summary = host_props.get("summary.hardware")
+                mem_size = getattr(hardware_summary, "memorySize", 0)
+                ram_per_host[host_ref_value] = mem_size // units.Mi
+
+        props = ['config.hardware.memoryMB', 'runtime.host']
+        vms = self._list_instances_in_cluster(additional_properties=props)
+
+        for (vm_uuid, vm_props) in vms:
+            host_obj = vm_props.get('runtime.host')
+            if not host_obj:
+                continue
+            host_ref_value = vutil.get_moref_value(host_obj)
+            if host_ref_value not in ram_per_host:
+                continue
+
+            vm_mb = vm_props.get('config.hardware.memoryMB', 0)
+
+            # make sure the minimum available memory >= 0
+            ram_per_host[host_ref_value] = \
+                max(ram_per_host[host_ref_value] - vm_mb, 0)
+
+        return ram_per_host