hardware: Allow 'hw:cpu_realtime_mask' to be omitted

By offloading emulator threads to other host cores (via the
'hw:emulator_threads_policy' extra spec), it's possible to allocate all
guest cores to realtime. Enable this.

Part of blueprint use-pcpu-and-vcpu-in-one-instance

Change-Id: I00805cf9cca9657955c7e1ef3a76e384adaa78f1
Signed-off-by: Stephen Finucane <[email protected]>

Author: stephenfin

doc/source/user/flavors.rst

@@ -601,8 +601,9 @@ CPU real-time policy
   .. important::
 
      While most of your instance vCPUs can run with a real-time policy, you must
-     mark at least one vCPU as non-real-time, to be used for both non-real-time
-     guest processes and emulator overhead (housekeeping) processes.
+     either mark at least one vCPU as non-real-time to be account for emulator
+     overhead (housekeeping) or explicitly configure an :ref:`emulator thread
+     policy <extra-specs-emulator-threads-policy>`.
 
   .. important::
 
@@ -633,6 +634,13 @@ CPU real-time policy
      The ``hw:cpu_realtime_mask`` option is only valid if ``hw:cpu_realtime``
      is set to ``yes``.
 
+  .. versionchanged:: 22.0.0 (Victoria)
+
+     Previously, it was necessary to specify ``hw:cpu_realtime_mask`` when
+     ``hw:cpu_realtime`` was set to yes. Starting in Victoria, it is possible
+     to omit this when an emulator thread policy is configured using the
+     ``hw:emulator_threads_policy`` extra spec.
+
 .. _extra-specs-emulator-threads-policy:
 
 Emulator threads policy

nova/exception.py

@@ -1894,9 +1894,8 @@ class LibguestfsCannotReadKernel(Invalid):
 
 
 class RealtimeMaskNotFoundOrInvalid(Invalid):
-    msg_fmt = _("Realtime policy needs vCPU(s) mask configured with at least "
-                "1 RT vCPU and 1 ordinary vCPU. See hw:cpu_realtime_mask "
-                "or hw_cpu_realtime_mask")
+    msg_fmt = _("Use of realtime CPUs requires either one or more "
+                "non-realtime CPU(s) or offloaded emulator threads.")
 
 
 class OsInfoNotFound(NotFound):

nova/tests/unit/virt/libvirt/test_driver.py

@@ -3027,6 +3027,34 @@ def test_get_guest_memory_backing_config_realtime(self):
         self.assertTrue(membacking.locked)
         self.assertFalse(membacking.sharedpages)
 
+    def test_get_guest_memory_backing_config_realtime_invalid_share(self):
+        """Test behavior when there is no pool of shared CPUS on which to place
+        the emulator threads, isolating them from the instance CPU processes.
+        """
+        extra_specs = {
+            "hw:cpu_realtime": "yes",
+            "hw:cpu_policy": "dedicated",
+            "hw:emulator_threads_policy": "share",
+        }
+        flavor = objects.Flavor(
+            name='m1.small',
+            memory_mb=6,
+            vcpus=28,
+            root_gb=496,
+            ephemeral_gb=8128,
+            swap=33550336,
+            extra_specs=extra_specs)
+        image_meta = objects.ImageMeta.from_dict(self.test_image_meta)
+        drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
+
+        # this should fail because there is nowhere to place the emulator
+        # threads
+        self.assertRaises(
+            exception.RealtimeMaskNotFoundOrInvalid,
+            drvr._get_guest_memory_backing_config,
+            None, None, flavor, image_meta,
+        )
+
     def _test_sev_enabled(self, expected=None, host_sev_enabled=False,
                           enc_extra_spec=None, enc_image_prop=None,
                           hw_machine_type=None, hw_firmware_type=None):

nova/tests/unit/virt/test_hardware.py

@@ -3887,6 +3887,18 @@ def test_invalid_mask_no_exclusion_wo_emulator_policy(self):
             exception.RealtimeMaskNotFoundOrInvalid,
             hw.get_realtime_cpu_constraint, flavor, image)
 
+    def test_all_cpus_w_emulator_policy(self):
+        # The mask has no exclusion but there's an emulator thread policy
+        flavor = objects.Flavor(
+            vcpus=3, memory_mb=2048, extra_specs={
+                'hw:cpu_realtime': 'true',
+                'hw:emulator_threads_policy': 'isolate'
+            },
+        )
+        image = objects.ImageMeta.from_dict({"properties": {}})
+        rt = hw.get_realtime_cpu_constraint(flavor, image)
+        self.assertEqual({0, 1, 2}, rt)
+
     def test_invalid_mask_rt_cpus_out_of_range(self):
         # The mask is not just an exclusion mask, and the RT range specifies
         # an invalid vCPU number.

nova/virt/hardware.py

@@ -1705,16 +1705,19 @@ def get_realtime_cpu_constraint(
     # Image masks are used ahead of flavor masks as they will have more
     # specific requirements
     mask = image_mask or flavor_mask
-    if not mask:
-        raise exception.RealtimeMaskNotFoundOrInvalid()
 
     vcpus_set = set(range(flavor.vcpus))
-    vcpus_rt = parse_cpu_spec("0-%d,%s" % (flavor.vcpus - 1, mask))
+    if mask:
+        vcpus_rt = parse_cpu_spec("0-%d,%s" % (flavor.vcpus - 1, mask))
+    else:
+        vcpus_rt = set(range(flavor.vcpus))
 
     if not vcpus_rt:
         raise exception.RealtimeMaskNotFoundOrInvalid()
 
-    if vcpus_set == vcpus_rt:
+    # TODO(stephenfin): Do this check in numa_get_constraints instead
+    emu_policy = get_emulator_thread_policy_constraint(flavor)
+    if vcpus_set == vcpus_rt and not emu_policy:
         raise exception.RealtimeMaskNotFoundOrInvalid()
 
     if not vcpus_rt.issubset(vcpus_set):

nova/virt/libvirt/driver.py

@@ -5432,15 +5432,33 @@ def _set_qemu_guest_agent(self, guest, flavor, instance, image_meta):
 
     def _get_guest_memory_backing_config(
             self, inst_topology, numatune, flavor, image_meta):
+        wantsrealtime = hardware.is_realtime_enabled(flavor)
+        if (
+            wantsrealtime and
+            hardware.get_emulator_thread_policy_constraint(flavor) ==
+                fields.CPUEmulatorThreadsPolicy.SHARE and
+            not CONF.compute.cpu_shared_set
+        ):
+            # NOTE(stephenfin) Yes, it's horrible that we're doing this here,
+            # but the shared policy unfortunately has different behavior
+            # depending on whether the '[compute] cpu_shared_set' is configured
+            # or not and we need it to be configured. Also note that we have
+            # already handled other conditions, such as no emulator thread
+            # policy being configured whatsoever, at the API level.
+            LOG.warning(
+                'Instance is requesting real-time CPUs with pooled '
+                'emulator threads, but a shared CPU pool has not been '
+                'configured on this host.'
+            )
+            raise exception.RealtimeMaskNotFoundOrInvalid()
+
         wantsmempages = False
         if inst_topology:
             for cell in inst_topology.cells:
                 if cell.pagesize:
                     wantsmempages = True
                     break
 
-        wantsrealtime = hardware.is_realtime_enabled(flavor)
-
         wantsfilebacked = CONF.libvirt.file_backed_memory > 0
 
         if wantsmempages and wantsfilebacked:

releasenotes/notes/bug-1884231-16acf297d88b122e.yaml

@@ -4,3 +4,8 @@ fixes:
     Previously, it was possible to specify values for the
     ``hw:cpu_realtime_mask`` extra spec that were not within the range of valid
     instances cores. This value is now correctly validated.
+features:
+  - |
+    It is now possible to allocate all cores in an instance to realtime and
+    omit the ``hw:cpu_realtime_mask`` extra spec. This requires specifying the
+    ``hw:emulator_threads_policy`` extra spec.