Merge pull request #78 from stackhpc/upstream/2023.1-2024-06-25

Synchronise 2023.1 with upstream

Author: markgoddard

doc/source/admin/cpu-topologies.rst

@@ -772,6 +772,11 @@ while there could be other tools outside Nova to manage the governor, like
 tuned. That being said, we also provide a way to automatically change the
 governors on the fly, as explained below.
 
+.. important::
+   Some OS platforms don't support `cpufreq` resources in sysfs, so the
+   ``governor`` strategy could be not available. Please verify if your OS
+   supports scaling govenors before modifying the configuration option.
+
 If the strategy is set to ``governor``, a couple of config options are provided
 to define which exact CPU govenor to use for each of the up and down states :
 

nova/cmd/manage.py

@@ -3195,7 +3195,7 @@ class ImagePropertyCommands:
         'instance_uuid', metavar='<instance_uuid>',
         help='UUID of the instance')
     @args(
-        'property', metavar='<image_property>',
+        'image_property', metavar='<image_property>',
         help='Image property to show')
     def show(self, instance_uuid=None, image_property=None):
         """Show value of a given instance image property.
@@ -3213,10 +3213,10 @@ def show(self, instance_uuid=None, image_property=None):
             with context.target_cell(ctxt, im.cell_mapping) as cctxt:
                 instance = objects.Instance.get_by_uuid(
                     cctxt, instance_uuid, expected_attrs=['system_metadata'])
-                image_property = instance.system_metadata.get(
+                property_value = instance.system_metadata.get(
                     f'image_{image_property}')
-                if image_property:
-                    print(image_property)
+                if property_value:
+                    print(property_value)
                     return 0
                 else:
                     print(f'Image property {image_property} not found '

nova/cmd/status.py

@@ -266,7 +266,7 @@ def _check_machine_type_set(self):
 your environment does not contain libvirt based compute hosts.
 Use the `nova-manage machine_type list_unset` command to list these instances.
 For more details see the following:
-https://docs.openstack.org/latest/nova/admin/hw_machine_type.html"""))
+https://docs.openstack.org/nova/latest/admin/hw-machine-type.html"""))
             return upgradecheck.Result(upgradecheck.Code.WARNING, msg)
 
         return upgradecheck.Result(upgradecheck.Code.SUCCESS)
@@ -276,7 +276,7 @@ def _check_service_user_token(self):
             msg = (_("""
 Service user token configuration is required for all Nova services.
 For more details see the following:
-https://docs.openstack.org/latest/nova/admin/configuration/service-user-token.html"""))  # noqa
+https://docs.openstack.org/nova/latest/admin/configuration/service-user-token.html"""))  # noqa
             return upgradecheck.Result(upgradecheck.Code.FAILURE, msg)
         return upgradecheck.Result(upgradecheck.Code.SUCCESS)
 

nova/compute/manager.py

@@ -9095,12 +9095,16 @@ def _live_migration_cleanup_flags(self, migrate_data, migr_ctxt=None):
                                    objects.LibvirtVPMEMDevice)):
                         has_vpmem = True
                         break
+            power_management_possible = (
+                'dst_numa_info' in migrate_data and
+                migrate_data.dst_numa_info is not None)
             # No instance booting at source host, but instance dir
             # must be deleted for preparing next block migration
             # must be deleted for preparing next live migration w/o shared
             # storage
             # vpmem must be cleaned
-            do_cleanup = not migrate_data.is_shared_instance_path or has_vpmem
+            do_cleanup = (not migrate_data.is_shared_instance_path or
+                          has_vpmem or power_management_possible)
             destroy_disks = not migrate_data.is_shared_block_storage
         elif isinstance(migrate_data, migrate_data_obj.HyperVLiveMigrateData):
             # NOTE(claudiub): We need to cleanup any zombie Planned VM.

nova/network/neutron.py

@@ -3610,6 +3610,7 @@ def _get_subnets_from_port(self, context, port, client=None):
                            'gateway': network_model.IP(
                                 address=subnet['gateway_ip'],
                                 type='gateway'),
+                           'enable_dhcp': False,
             }
             if subnet.get('ipv6_address_mode'):
                 subnet_dict['ipv6_address_mode'] = subnet['ipv6_address_mode']
@@ -3626,22 +3627,14 @@ def _get_subnets_from_port(self, context, port, client=None):
                         subnet_dict['dhcp_server'] = ip_pair['ip_address']
                         break
 
-            # NOTE(arnaudmorin): If enable_dhcp is set on subnet, but, for
-            # some reason neutron did not have any DHCP port yet, we still
-            # want the network_info to be populated with a valid dhcp_server
-            # value. This is mostly useful for the metadata API (which is
-            # relying on this value to give network_data to the instance).
-            #
-            # This will also help some providers which are using external
-            # DHCP servers not handled by neutron.
-            # In this case, neutron will never create any DHCP port in the
-            # subnet.
-            #
-            # Also note that we cannot set the value to None because then the
-            # value would be discarded by the metadata API.
-            # So the subnet gateway will be used as fallback.
-            if subnet.get('enable_dhcp') and 'dhcp_server' not in subnet_dict:
-                subnet_dict['dhcp_server'] = subnet['gateway_ip']
+            # NOTE(stblatzheim): If enable_dhcp is set on subnet, but subnet
+            # has ovn native dhcp and no dhcp-agents. Network owner will be
+            # network:distributed
+            # Just rely on enable_dhcp flag given by neutron
+            # Fix for https://bugs.launchpad.net/nova/+bug/2055245
+
+            if subnet.get('enable_dhcp'):
+                subnet_dict['enable_dhcp'] = True
 
             subnet_object = network_model.Subnet(**subnet_dict)
             for dns in subnet.get('dns_nameservers', []):

nova/objects/instance_numa.py

@@ -293,6 +293,12 @@ def cpu_pinning(self):
             cell.cpu_pinning.values() for cell in self.cells
             if cell.cpu_pinning]))
 
+    @property
+    def cpuset_reserved(self):
+        return set(itertools.chain.from_iterable([
+            cell.cpuset_reserved for cell in self.cells
+            if cell.cpuset_reserved]))
+
     def clear_host_pinning(self):
         """Clear any data related to how instance is pinned to the host.
 

nova/scheduler/utils.py

@@ -1080,6 +1080,17 @@ def validate_weigher(weigher):
 _SUPPORTS_SOFT_ANTI_AFFINITY = None
 
 
+def reset_globals():
+    global _SUPPORTS_AFFINITY
+    _SUPPORTS_AFFINITY = None
+    global _SUPPORTS_ANTI_AFFINITY
+    _SUPPORTS_ANTI_AFFINITY = None
+    global _SUPPORTS_SOFT_AFFINITY
+    _SUPPORTS_SOFT_AFFINITY = None
+    global _SUPPORTS_SOFT_ANTI_AFFINITY
+    _SUPPORTS_SOFT_ANTI_AFFINITY = None
+
+
 def _get_group_details(context, instance_uuid, user_group_hosts=None):
     """Provide group_hosts and group_policies sets related to instances if
     those instances are belonging to a group and if corresponding filters are

nova/test.py

@@ -62,6 +62,7 @@
 from nova.objects import base as objects_base
 from nova import quota
 from nova.scheduler.client import report
+from nova.scheduler import utils as scheduler_utils
 from nova.tests import fixtures as nova_fixtures
 from nova.tests.unit import matchers
 from nova import utils
@@ -310,6 +311,19 @@ def setUp(self):
         if self.STUB_COMPUTE_ID:
             self.useFixture(nova_fixtures.ComputeNodeIdFixture())
 
+        # Reset globals indicating affinity filter support. Some tests may set
+        # self.flags(enabled_filters=...) which could make the affinity filter
+        # support globals get set to a non-default configuration which affects
+        # all other tests.
+        scheduler_utils.reset_globals()
+
+        # Wait for bare greenlets spawn_n()'ed from a GreenThreadPoolExecutor
+        # to finish before moving on from the test. When greenlets from a
+        # previous test remain running, they may attempt to access structures
+        # (like the database) that have already been torn down and can cause
+        # the currently running test to fail.
+        self.useFixture(nova_fixtures.GreenThreadPoolShutdownWait())
+
     def _setup_cells(self):
         """Setup a normal cellsv2 environment.
 

nova/tests/fixtures/filesystem.py

@@ -39,10 +39,14 @@ def _setUp(self):
 
 
 class SysFileSystemFixture(TempFileSystemFixture):
-    """Creates a fake /sys filesystem"""
+    def __init__(self, cpus_supported=None, cpufreq_enabled=True):
+        """Instantiates a fake sysfs.
 
-    def __init__(self, cpus_supported=None):
+        :param cpus_supported: number of devices/system/cpu (default: 10)
+        :param cpufreq_enabled: cpufreq subdir created (default: True)
+        """
         self.cpus_supported = cpus_supported or 10
+        self.cpufreq_enabled = cpufreq_enabled
 
     def _setUp(self):
         super()._setUp()
@@ -73,9 +77,12 @@ def _setUp(self):
 
         for cpu_nr in range(self.cpus_supported):
             cpu_dir = os.path.join(self.cpu_path_mock % {'core': cpu_nr})
-            os.makedirs(os.path.join(cpu_dir, 'cpufreq'))
-            filesystem.write_sys(
-                os.path.join(cpu_dir, 'cpufreq/scaling_governor'),
-                data='powersave')
+            os.makedirs(cpu_dir)
+            filesystem.write_sys(os.path.join(cpu_dir, 'online'), data='1')
+            if self.cpufreq_enabled:
+                os.makedirs(os.path.join(cpu_dir, 'cpufreq'))
+                filesystem.write_sys(
+                    os.path.join(cpu_dir, 'cpufreq/scaling_governor'),
+                    data='powersave')
         filesystem.write_sys(core.AVAILABLE_PATH,
                              f'0-{self.cpus_supported - 1}')

nova/tests/fixtures/nova.py

@@ -1938,3 +1938,38 @@ def setUp(self):
             'nova.compute.manager.ComputeManager.'
             '_ensure_existing_node_identity',
             mock.DEFAULT))
+
+
+class GreenThreadPoolShutdownWait(fixtures.Fixture):
+    """Always wait for greenlets in greenpool to finish.
+
+    We use the futurist.GreenThreadPoolExecutor, for example, in compute
+    manager to run live migration jobs. It runs those jobs in bare greenlets
+    created by eventlet.spawn_n(). Bare greenlets cannot be killed the same
+    way as GreenThreads created by eventlet.spawn().
+
+    Because they cannot be killed, in the test environment we must either let
+    them run to completion or move on while they are still running (which can
+    cause test failures as the leaked greenlets attempt to access structures
+    that have already been torn down).
+
+    When a compute service is stopped by Service.stop(), the compute manager's
+    cleanup_host() method is called and while cleaning up, the compute manager
+    calls the GreenThreadPoolExecutor.shutdown() method with wait=False. This
+    means that a test running GreenThreadPoolExecutor jobs will not wait for
+    the bare greenlets to finish running -- it will instead move on immediately
+    while greenlets are still running.
+
+    This fixture will ensure GreenThreadPoolExecutor.shutdown() is always
+    called with wait=True in an effort to reduce the number of leaked bare
+    greenlets.
+
+    See https://bugs.launchpad.net/nova/+bug/1946339 for details.
+    """
+
+    def setUp(self):
+        super().setUp()
+        real_shutdown = futurist.GreenThreadPoolExecutor.shutdown
+        self.useFixture(fixtures.MockPatch(
+            'futurist.GreenThreadPoolExecutor.shutdown',
+            lambda self, wait: real_shutdown(self, wait=True)))