Enable cpus when an instance is spawning

By this patch, we now automatically power down or up cores
when an instance is either stopped or started.

Also, by default, we now powersave or offline dedicated cores when
starting the compute service.

Implements: blueprint libvirt-cpu-state-mgmt
Change-Id: Id645fd1ba909683af903f3b8f11c7f06db3401cb

Author: sbauza

nova/conf/libvirt.py

@@ -1482,6 +1482,11 @@
     cfg.BoolOpt('cpu_power_management',
                 default=False,
                 help='Use libvirt to manage CPU cores performance.'),
+    cfg.StrOpt('cpu_power_management_strategy',
+               choices=['cpu_state', 'governor'],
+               default='cpu_state',
+               help='Tuning strategy to reduce CPU power consumption when '
+                    'unused'),
     cfg.StrOpt('cpu_power_governor_low',
                default='powersave',
                help='Governor to use in order '

nova/tests/fixtures/__init__.py

@@ -16,6 +16,8 @@
 from .cinder import CinderFixture  # noqa: F401
 from .conf import ConfFixture  # noqa: F401, F403
 from .cyborg import CyborgFixture  # noqa: F401
+from .filesystem import SysFileSystemFixture  # noqa: F401
+from .filesystem import TempFileSystemFixture  # noqa: F401
 from .glance import GlanceFixture  # noqa: F401
 from .libvirt import LibvirtFixture  # noqa: F401
 from .libvirt_imagebackend import LibvirtImageBackendFixture  # noqa: F401

nova/tests/fixtures/filesystem.py

@@ -0,0 +1,81 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import shutil
+import tempfile
+from unittest import mock
+
+import fixtures
+
+from nova import filesystem
+from nova.virt.libvirt.cpu import core
+
+
+SYS = 'sys'
+
+
+class TempFileSystemFixture(fixtures.Fixture):
+    """Creates a fake / filesystem"""
+
+    def _setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory(prefix='fake_fs')
+        # NOTE(sbauza): I/O disk errors may raise an exception here, as we
+        # don't ignore them. If that's causing a problem in our CI jobs, the
+        # recommended solution is to use shutil.rmtree instead of cleanup()
+        # with ignore_errors parameter set to True (or wait for the minimum
+        # python version to be 3.10 as TemporaryDirectory will provide
+        # ignore_cleanup_errors parameter)
+        self.addCleanup(self.temp_dir.cleanup)
+
+
+class SysFileSystemFixture(TempFileSystemFixture):
+    """Creates a fake /sys filesystem"""
+
+    def __init__(self, cpus_supported=None):
+        self.cpus_supported = cpus_supported or 10
+
+    def _setUp(self):
+        super()._setUp()
+        self.sys_path = os.path.join(self.temp_dir.name, SYS)
+        self.addCleanup(shutil.rmtree, self.sys_path, ignore_errors=True)
+
+        sys_patcher = mock.patch(
+            'nova.filesystem.SYS',
+            new_callable=mock.PropertyMock(return_value=self.sys_path))
+        self.sys_mock = sys_patcher.start()
+        self.addCleanup(sys_patcher.stop)
+
+        avail_path_patcher = mock.patch(
+            'nova.virt.libvirt.cpu.core.AVAILABLE_PATH',
+            new_callable=mock.PropertyMock(
+                return_value=os.path.join(self.sys_path,
+                                          'devices/system/cpu/present')))
+        self.avail_path_mock = avail_path_patcher.start()
+        self.addCleanup(avail_path_patcher.stop)
+
+        cpu_path_patcher = mock.patch(
+            'nova.virt.libvirt.cpu.core.CPU_PATH_TEMPLATE',
+            new_callable=mock.PropertyMock(
+                return_value=os.path.join(self.sys_path,
+                                          'devices/system/cpu/cpu%(core)s')))
+        self.cpu_path_mock = cpu_path_patcher.start()
+        self.addCleanup(cpu_path_patcher.stop)
+
+        for cpu_nr in range(self.cpus_supported):
+            cpu_dir = os.path.join(self.cpu_path_mock % {'core': cpu_nr})
+            os.makedirs(os.path.join(cpu_dir, 'cpufreq'))
+            filesystem.write_sys(
+                os.path.join(cpu_dir, 'cpufreq/scaling_governor'),
+                data='powersave')
+        filesystem.write_sys(core.AVAILABLE_PATH,
+                             f'0-{self.cpus_supported - 1}')

nova/tests/functional/libvirt/test_power_manage.py

@@ -0,0 +1,270 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from unittest import mock
+
+import fixtures
+
+from nova import context as nova_context
+from nova import exception
+from nova import objects
+from nova.tests import fixtures as nova_fixtures
+from nova.tests.fixtures import libvirt as fakelibvirt
+from nova.tests.functional.libvirt import base
+from nova.virt import hardware
+from nova.virt.libvirt import cpu
+
+
+class PowerManagementTestsBase(base.ServersTestBase):
+
+    ADDITIONAL_FILTERS = ['NUMATopologyFilter']
+
+    ADMIN_API = True
+
+    def setUp(self):
+        super(PowerManagementTestsBase, self).setUp()
+
+        self.ctxt = nova_context.get_admin_context()
+
+        # Mock the 'NUMATopologyFilter' filter, as most tests need to inspect
+        # this
+        host_manager = self.scheduler.manager.host_manager
+        numa_filter_class = host_manager.filter_cls_map['NUMATopologyFilter']
+        host_pass_mock = mock.Mock(wraps=numa_filter_class().host_passes)
+        _p = mock.patch('nova.scheduler.filters'
+                        '.numa_topology_filter.NUMATopologyFilter.host_passes',
+                        side_effect=host_pass_mock)
+        self.mock_filter = _p.start()
+        self.addCleanup(_p.stop)
+
+        # for the sake of resizing, we need to patch the two methods below
+        self.useFixture(fixtures.MockPatch(
+            'nova.virt.libvirt.LibvirtDriver._get_instance_disk_info',
+             return_value=[]))
+        self.useFixture(fixtures.MockPatch('os.rename'))
+
+        self.useFixture(nova_fixtures.PrivsepFixture())
+
+        # Defining the main flavor for 4 vCPUs all pinned
+        self.extra_spec = {
+            'hw:cpu_policy': 'dedicated',
+            'hw:cpu_thread_policy': 'prefer',
+        }
+        self.pcpu_flavor_id = self._create_flavor(
+            vcpu=4, extra_spec=self.extra_spec)
+
+    def _assert_server_cpus_state(self, server, expected='online'):
+        inst = objects.Instance.get_by_uuid(self.ctxt, server['id'])
+        if not inst.numa_topology:
+            self.fail('Instance should have a NUMA topology in order to know '
+                      'its physical CPUs')
+        instance_pcpus = inst.numa_topology.cpu_pinning
+        self._assert_cpu_set_state(instance_pcpus, expected=expected)
+        return instance_pcpus
+
+    def _assert_cpu_set_state(self, cpu_set, expected='online'):
+        for i in cpu_set:
+            core = cpu.Core(i)
+            if expected == 'online':
+                self.assertTrue(core.online, f'{i} is not online')
+            elif expected == 'offline':
+                self.assertFalse(core.online, f'{i} is online')
+            elif expected == 'powersave':
+                self.assertEqual('powersave', core.governor)
+            elif expected == 'performance':
+                self.assertEqual('performance', core.governor)
+
+
+class PowerManagementTests(PowerManagementTestsBase):
+    """Test suite for a single host with 9 dedicated cores and 1 used for OS"""
+
+    def setUp(self):
+        super(PowerManagementTests, self).setUp()
+
+        self.useFixture(nova_fixtures.SysFileSystemFixture())
+
+        # Definining the CPUs to be pinned.
+        self.flags(cpu_dedicated_set='1-9', cpu_shared_set=None,
+                   group='compute')
+        self.flags(vcpu_pin_set=None)
+        self.flags(cpu_power_management=True, group='libvirt')
+
+        self.flags(allow_resize_to_same_host=True)
+        self.host_info = fakelibvirt.HostInfo(cpu_nodes=1, cpu_sockets=1,
+                                         cpu_cores=5, cpu_threads=2)
+        self.compute1 = self.start_compute(host_info=self.host_info,
+                                           hostname='compute1')
+
+        # All cores are shutdown at startup, let's check.
+        cpu_dedicated_set = hardware.get_cpu_dedicated_set()
+        self._assert_cpu_set_state(cpu_dedicated_set, expected='offline')
+
+    def test_hardstop_compute_service_if_wrong_opt(self):
+        self.flags(cpu_dedicated_set=None, cpu_shared_set=None,
+                   group='compute')
+        self.flags(vcpu_pin_set=None)
+        self.flags(cpu_power_management=True, group='libvirt')
+        self.assertRaises(exception.InvalidConfiguration,
+                          self.start_compute, host_info=self.host_info,
+                                              hostname='compute2')
+
+    def test_create_server(self):
+        server = self._create_server(
+            flavor_id=self.pcpu_flavor_id,
+            expected_state='ACTIVE')
+        # Let's verify that the pinned CPUs are now online
+        self._assert_server_cpus_state(server, expected='online')
+
+        # Verify that the unused CPUs are still offline
+        inst = objects.Instance.get_by_uuid(self.ctxt, server['id'])
+        instance_pcpus = inst.numa_topology.cpu_pinning
+        cpu_dedicated_set = hardware.get_cpu_dedicated_set()
+        unused_cpus = cpu_dedicated_set - instance_pcpus
+        self._assert_cpu_set_state(unused_cpus, expected='offline')
+
+    def test_stop_start_server(self):
+        server = self._create_server(
+            flavor_id=self.pcpu_flavor_id,
+            expected_state='ACTIVE')
+
+        server = self._stop_server(server)
+        # Let's verify that the pinned CPUs are now stopped...
+        self._assert_server_cpus_state(server, expected='offline')
+
+        server = self._start_server(server)
+        # ...and now, they should be back.
+        self._assert_server_cpus_state(server, expected='online')
+
+    def test_resize(self):
+        server = self._create_server(
+            flavor_id=self.pcpu_flavor_id,
+            expected_state='ACTIVE')
+        server_pcpus = self._assert_server_cpus_state(server,
+                                                      expected='online')
+
+        new_flavor_id = self._create_flavor(
+            vcpu=5, extra_spec=self.extra_spec)
+        self._resize_server(server, new_flavor_id)
+        server2_pcpus = self._assert_server_cpus_state(server,
+                                                       expected='online')
+        # Even if the resize is not confirmed yet, the original guest is now
+        # destroyed so the cores are now offline.
+        self._assert_cpu_set_state(server_pcpus, expected='offline')
+
+        # let's revert the resize
+        self._revert_resize(server)
+        # So now the original CPUs will be online again, while the previous
+        # cores should be back offline.
+        self._assert_cpu_set_state(server_pcpus, expected='online')
+        self._assert_cpu_set_state(server2_pcpus, expected='offline')
+
+    def test_changing_strategy_fails(self):
+        # As a reminder, all cores have been shutdown before.
+        # Now we want to change the strategy and then we restart the service
+        self.flags(cpu_power_management_strategy='governor', group='libvirt')
+        # See, this is not possible as we would have offline CPUs.
+        self.assertRaises(exception.InvalidConfiguration,
+                          self.restart_compute_service, hostname='compute1')
+
+
+class PowerManagementTestsGovernor(PowerManagementTestsBase):
+    """Test suite for speific governor usage (same 10-core host)"""
+
+    def setUp(self):
+        super(PowerManagementTestsGovernor, self).setUp()
+
+        self.useFixture(nova_fixtures.SysFileSystemFixture())
+
+        # Definining the CPUs to be pinned.
+        self.flags(cpu_dedicated_set='1-9', cpu_shared_set=None,
+                   group='compute')
+        self.flags(vcpu_pin_set=None)
+        self.flags(cpu_power_management=True, group='libvirt')
+        self.flags(cpu_power_management_strategy='governor', group='libvirt')
+
+        self.flags(allow_resize_to_same_host=True)
+        self.host_info = fakelibvirt.HostInfo(cpu_nodes=1, cpu_sockets=1,
+                                         cpu_cores=5, cpu_threads=2)
+        self.compute1 = self.start_compute(host_info=self.host_info,
+                                           hostname='compute1')
+
+    def test_create(self):
+        cpu_dedicated_set = hardware.get_cpu_dedicated_set()
+        # With the governor strategy, cores are still online but run with a
+        # powersave governor.
+        self._assert_cpu_set_state(cpu_dedicated_set, expected='powersave')
+
+        # Now, start an instance
+        server = self._create_server(
+            flavor_id=self.pcpu_flavor_id,
+            expected_state='ACTIVE')
+        # When pinned cores are run, the governor state is now performance
+        self._assert_server_cpus_state(server, expected='performance')
+
+    def test_changing_strategy_fails(self):
+        # Arbitratly set a core governor strategy to be performance
+        cpu.Core(1).set_high_governor()
+        # and then forget about it while changing the strategy.
+        self.flags(cpu_power_management_strategy='cpu_state', group='libvirt')
+        # This time, this wouldn't be acceptable as some core would have a
+        # difference performance while Nova would only online/offline it.
+        self.assertRaises(exception.InvalidConfiguration,
+                          self.restart_compute_service, hostname='compute1')
+
+
+class PowerManagementMixedInstances(PowerManagementTestsBase):
+    """Test suite for a single host with 6 dedicated cores, 3 shared and one
+    OS-restricted.
+    """
+
+    def setUp(self):
+        super(PowerManagementMixedInstances, self).setUp()
+
+        self.useFixture(nova_fixtures.SysFileSystemFixture())
+
+        # Definining 6 CPUs to be dedicated, not all of them in a series.
+        self.flags(cpu_dedicated_set='1-3,5-7', cpu_shared_set='4,8-9',
+                   group='compute')
+        self.flags(vcpu_pin_set=None)
+        self.flags(cpu_power_management=True, group='libvirt')
+
+        self.host_info = fakelibvirt.HostInfo(cpu_nodes=1, cpu_sockets=1,
+                                         cpu_cores=5, cpu_threads=2)
+        self.compute1 = self.start_compute(host_info=self.host_info,
+                                           hostname='compute1')
+
+        # Make sure only 6 are offline now
+        cpu_dedicated_set = hardware.get_cpu_dedicated_set()
+        self._assert_cpu_set_state(cpu_dedicated_set, expected='offline')
+
+        # cores 4 and 8-9 should be online
+        self._assert_cpu_set_state({4, 8, 9}, expected='online')
+
+    def test_standard_server_works_and_passes(self):
+
+        std_flavor_id = self._create_flavor(vcpu=2)
+        self._create_server(flavor_id=std_flavor_id, expected_state='ACTIVE')
+
+        # Since this is an instance with floating vCPUs on the shared set, we
+        # can only lookup the host CPUs and see they haven't changed state.
+        cpu_dedicated_set = hardware.get_cpu_dedicated_set()
+        self._assert_cpu_set_state(cpu_dedicated_set, expected='offline')
+        self._assert_cpu_set_state({4, 8, 9}, expected='online')
+
+        # We can now try to boot an instance with pinned CPUs to test the mix
+        pinned_server = self._create_server(
+            flavor_id=self.pcpu_flavor_id,
+            expected_state='ACTIVE')
+        # We'll see that its CPUs are now online
+        self._assert_server_cpus_state(pinned_server, expected='online')
+        # but it doesn't change the shared set
+        self._assert_cpu_set_state({4, 8, 9}, expected='online')