Add locked_memory extra spec and image property

SeanMooney · ricolin · commit 572c2b18e27f · 2022-08-24T21:47:32.000+08:00
This change adds a new hw:locked_memory extra spec and hw_locked_memory
image property to contol preventing guest memory from swapping.

This change adds docs and extend the flavor
validators for the new extra spec.
Also add new image property.

Blueprint: libvirt-viommu-device
Change-Id: Id3779594f0078a5045031aded2ed68ee4301abbd
diff --git a/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json b/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json
@@ -4,5 +4,5 @@
         "hw_architecture": "x86_64"
     },
     "nova_object.name": "ImageMetaPropsPayload",
-    "nova_object.version": "1.10"
+    "nova_object.version": "1.11"
 }
diff --git a/doc/source/admin/libvirt-misc.rst b/doc/source/admin/libvirt-misc.rst
@@ -138,3 +138,33 @@ For example, to hide your signature from the guest OS, run:
 .. code:: console
 
    $ openstack flavor set $FLAVOR --property hw:hide_hypervisor_id=true
+
+
+.. _extra-spec-locked_memory:
+
+Locked memory allocation
+------------------------
+
+.. versionadded:: 26.0.0 (Zed)
+
+Locking memory marks the guest memory allocations as unmovable and
+unswappable. It is implicitly enabled in a number of cases such as SEV or
+realtime guests but can also be enabled explictly using the
+``hw:locked_memory`` extra spec (or use ``hw_locked_memory`` image property).
+``hw:locked_memory`` (also ``hw_locked_memory`` image property) accept
+boolean values in string format like 'true' or 'false' value.
+It will raise `FlavorImageLockedMemoryConflict` exception if both flavor and
+image property are specified but with different boolean values.
+This will only be allowed if you have also set ``hw:mem_page_size``,
+so we can ensure that the scheduler can actually account for this correctly
+and prevent out of memory events. Otherwise, will raise `LockMemoryForbidden`
+exception.
+
+.. code:: console
+
+   $ openstack flavor set FLAVOR-NAME \
+       --property hw:locked_memory=BOOLEAN_VALUE
+
+.. note::
+
+   This is currently only supported by the libvirt driver.
diff --git a/nova/api/validation/extra_specs/hw.py b/nova/api/validation/extra_specs/hw.py
@@ -163,6 +163,18 @@
             'pattern': r'(large|small|any|\d+([kKMGT]i?)?(b|bit|B)?)',
         },
     ),
+    base.ExtraSpecValidator(
+        name='hw:locked_memory',
+        description=(
+            'Determine if **guest** (instance) memory should be locked '
+            'preventing swaping. This is required in rare cases for device '
+            'DMA transfers. Only supported by the libvirt virt driver.'
+        ),
+        value={
+            'type': bool,
+            'description': 'Whether to lock **guest** (instance) memory.',
+        },
+    ),
 ]
 
 numa_validators = [
diff --git a/nova/exception.py b/nova/exception.py
@@ -1861,6 +1861,17 @@ class MemoryPageSizeNotSupported(Invalid):
     msg_fmt = _("Page size %(pagesize)s is not supported by the host.")
 
 
+class LockMemoryForbidden(Forbidden):
+    msg_fmt = _("locked_memory value in image or flavor is forbidden when "
+                "mem_page_size is not set.")
+
+
+class FlavorImageLockedMemoryConflict(NovaException):
+    msg_fmt = _("locked_memory value in image (%(image)s) and flavor "
+                "(%(flavor)s) conflict. A consistent value is expected if "
+                "both specified.")
+
+
 class CPUPinningInvalid(Invalid):
     msg_fmt = _("CPU set to pin %(requested)s must be a subset of "
                 "free CPU set %(available)s")
diff --git a/nova/notifications/objects/image.py b/nova/notifications/objects/image.py
@@ -128,7 +128,8 @@ class ImageMetaPropsPayload(base.NotificationPayloadBase):
     # Version 1.9: Added 'hw_emulation_architecture' field
     # Version 1.10: Added 'hw_ephemeral_encryption' and
     #                     'hw_ephemeral_encryption_format' fields
-    VERSION = '1.10'
+    # Version 1.11: Added 'hw_locked_memory' field
+    VERSION = '1.11'
 
     SCHEMA = {
         k: ('image_meta_props', k) for k in image_meta.ImageMetaProps.fields}
diff --git a/nova/objects/image_meta.py b/nova/objects/image_meta.py
@@ -190,14 +190,17 @@ class ImageMetaProps(base.NovaObject):
     # Version 1.31: Added 'hw_emulation_architecture' field
     # Version 1.32: Added 'hw_ephemeral_encryption' and
     #                     'hw_ephemeral_encryption_format' fields
+    # Version 1.33: Added 'hw_locked_memory' field
     # NOTE(efried): When bumping this version, the version of
     # ImageMetaPropsPayload must also be bumped. See its docstring for details.
-    VERSION = '1.32'
+    VERSION = '1.33'
 
     def obj_make_compatible(self, primitive, target_version):
         super(ImageMetaProps, self).obj_make_compatible(primitive,
                                                         target_version)
         target_version = versionutils.convert_version_to_tuple(target_version)
+        if target_version < (1, 33):
+            primitive.pop('hw_locked_memory', None)
         if target_version < (1, 32):
             primitive.pop('hw_ephemeral_encryption', None)
             primitive.pop('hw_ephemeral_encryption_format', None)
@@ -368,6 +371,10 @@ def obj_make_compatible(self, primitive, target_version):
         # image with a network boot image
         'hw_ipxe_boot': fields.FlexibleBooleanField(),
 
+        # string - make sure ``locked`` element is present in the
+        # ``memoryBacking``.
+        'hw_locked_memory': fields.FlexibleBooleanField(),
+
         # There are sooooooooooo many possible machine types in
         # QEMU - several new ones with each new release - that it
         # is not practical to enumerate them all. So we use a free
diff --git a/nova/tests/functional/notification_sample_tests/test_instance.py b/nova/tests/functional/notification_sample_tests/test_instance.py
@@ -1231,7 +1231,7 @@ def test_rebuild_server(self):
                     'nova_object.data': {},
                     'nova_object.name': 'ImageMetaPropsPayload',
                     'nova_object.namespace': 'nova',
-                    'nova_object.version': '1.10',
+                    'nova_object.version': '1.11',
                 },
                 'image.size': 58145823,
                 'image.tags': [],
@@ -1327,7 +1327,7 @@ def test_rebuild_server_with_trusted_cert(self):
                     'nova_object.data': {},
                     'nova_object.name': 'ImageMetaPropsPayload',
                     'nova_object.namespace': 'nova',
-                    'nova_object.version': '1.10',
+                    'nova_object.version': '1.11',
                 },
                 'image.size': 58145823,
                 'image.tags': [],
diff --git a/nova/tests/unit/notifications/objects/test_notification.py b/nova/tests/unit/notifications/objects/test_notification.py
@@ -386,7 +386,7 @@ def test_payload_is_not_generated_if_notification_format_is_unversioned(
     # ImageMetaProps, so when you see a fail here for that reason, you must
     # *also* bump the version of ImageMetaPropsPayload. See its docstring for
     # more information.
-    'ImageMetaPropsPayload': '1.10-44cf0030dc94a1a60ba7a0e222e854d6',
+    'ImageMetaPropsPayload': '1.11-938809cd33367c52cbc814fb9b6783dc',
     'InstanceActionNotification': '1.0-a73147b93b520ff0061865849d3dfa56',
     'InstanceActionPayload': '1.8-4fa3da9cbf0761f1f700ae578f36dc2f',
     'InstanceActionRebuildNotification':
diff --git a/nova/tests/unit/objects/test_image_meta.py b/nova/tests/unit/objects/test_image_meta.py
@@ -108,6 +108,7 @@ def test_normal_props(self):
                  'hw_video_model': 'vga',
                  'hw_video_ram': '512',
                  'hw_qemu_guest_agent': 'yes',
+                 'hw_locked_memory': 'true',
                  'trait:CUSTOM_TRUSTED': 'required',
                  # Fill sane values for the rest here
                  }
@@ -116,6 +117,7 @@ def test_normal_props(self):
         self.assertEqual('vga', virtprops.hw_video_model)
         self.assertEqual(512, virtprops.hw_video_ram)
         self.assertTrue(virtprops.hw_qemu_guest_agent)
+        self.assertTrue(virtprops.hw_locked_memory)
         self.assertIsNotNone(virtprops.traits_required)
         self.assertIn('CUSTOM_TRUSTED', virtprops.traits_required)
 
@@ -285,6 +287,28 @@ def test_set_numa_cpus_no_count(self):
         self.assertEqual([set([0, 1, 2, 3])],
                          virtprops.hw_numa_cpus)
 
+    def test_locked_memory_prop(self):
+        props = {'hw_locked_memory': 'true'}
+        virtprops = objects.ImageMetaProps.from_dict(props)
+        self.assertTrue(virtprops.hw_locked_memory)
+
+    def test_obj_make_compatible_hw_locked_memory(self):
+        """Check 'hw_locked_memory' compatibility."""
+        # assert that 'hw_locked_memory' is supported
+        # on a suitably new version
+        obj = objects.ImageMetaProps(
+            hw_locked_memory='true',
+        )
+        primitive = obj.obj_to_primitive('1.33')
+        self.assertIn('hw_locked_memory',
+            primitive['nova_object.data'])
+        self.assertTrue(primitive['nova_object.data']['hw_locked_memory'])
+
+        # and is absent on older versions
+        primitive = obj.obj_to_primitive('1.32')
+        self.assertNotIn('hw_locked_memory',
+                         primitive['nova_object.data'])
+
     def test_get_unnumbered_trait_fields(self):
         """Tests that only valid un-numbered required traits are parsed from
         the properties.
diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py
@@ -1072,7 +1072,7 @@ def obj_name(cls):
     'HyperVLiveMigrateData': '1.4-e265780e6acfa631476c8170e8d6fce0',
     'IDEDeviceBus': '1.0-29d4c9f27ac44197f01b6ac1b7e16502',
     'ImageMeta': '1.8-642d1b2eb3e880a367f37d72dd76162d',
-    'ImageMetaProps': '1.32-4967d35948af08b710b8b861f3fff0f9',
+    'ImageMetaProps': '1.33-6b7a29f769e6b8eee3f05832d78c85a2',
     'Instance': '2.7-d187aec68cad2e4d8b8a03a68e4739ce',
     'InstanceAction': '1.2-9a5abc87fdd3af46f45731960651efb5',
     'InstanceActionEvent': '1.4-5b1f361bd81989f8bb2c20bb7e8a4cb4',
diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py
@@ -3131,6 +3131,41 @@ def test_get_guest_memory_backing_config_realtime(self):
         self.assertTrue(membacking.locked)
         self.assertFalse(membacking.sharedpages)
 
+    def test_get_guest_memory_backing_config_locked_flavor(self):
+        extra_specs = {
+            "hw:locked_memory": "True",
+            "hw:mem_page_size": 1000,
+        }
+        flavor = objects.Flavor(
+            name='m1.small', memory_mb=6, vcpus=28, root_gb=496,
+            ephemeral_gb=8128, swap=33550336, extra_specs=extra_specs)
+        image_meta = objects.ImageMeta.from_dict(self.test_image_meta)
+        drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
+        membacking = drvr._get_guest_memory_backing_config(
+            None, None, flavor, image_meta)
+        self.assertTrue(membacking.locked)
+
+    def test_get_guest_memory_backing_config_locked_image_meta(self):
+        extra_specs = {}
+        flavor = objects.Flavor(
+            name='m1.small',
+            memory_mb=6,
+            vcpus=28,
+            root_gb=496,
+            ephemeral_gb=8128,
+            swap=33550336,
+            extra_specs=extra_specs)
+        image_meta = objects.ImageMeta.from_dict({
+            "disk_format": "raw",
+            "properties": {
+                "hw_locked_memory": "True",
+                "hw_mem_page_size": 1000,
+            }})
+        drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
+        membacking = drvr._get_guest_memory_backing_config(
+            None, None, flavor, image_meta)
+        self.assertTrue(membacking.locked)
+
     def test_get_guest_memory_backing_config_realtime_invalid_share(self):
         """Test behavior when there is no pool of shared CPUS on which to place
         the emulator threads, isolating them from the instance CPU processes.
diff --git a/nova/tests/unit/virt/test_hardware.py b/nova/tests/unit/virt/test_hardware.py
@@ -2814,6 +2814,54 @@ def test_flavor_smaller_than_image_meta(self):
                           flavor, image_meta)
 
 
+class VirtLockMemoryTestCase(test.NoDBTestCase):
+    def _test_get_locked_memory_constraint(self, spec=None, props=None):
+        flavor = objects.Flavor(vcpus=16, memory_mb=2048,
+                                extra_specs=spec or {})
+        image_meta = objects.ImageMeta.from_dict({"properties": props or {}})
+        return hw.get_locked_memory_constraint(flavor, image_meta)
+
+    def test_get_locked_memory_constraint_image(self):
+        self.assertTrue(
+            self._test_get_locked_memory_constraint(
+                spec={"hw:mem_page_size": "small"},
+                props={"hw_locked_memory": "True"}))
+
+    def test_get_locked_memory_conflict(self):
+        ex = self.assertRaises(
+            exception.FlavorImageLockedMemoryConflict,
+            self._test_get_locked_memory_constraint,
+            spec={
+                "hw:locked_memory": "False",
+                "hw:mem_page_size": "small"
+            },
+            props={"hw_locked_memory": "True"}
+        )
+        ex_msg = ("locked_memory value in image (True) and flavor (False) "
+                  "conflict. A consistent value is expected if both "
+                  "specified.")
+        self.assertEqual(ex_msg, str(ex))
+
+    def test_get_locked_memory_constraint_forbidden(self):
+        self.assertRaises(
+            exception.LockMemoryForbidden,
+            self._test_get_locked_memory_constraint,
+            {"hw:locked_memory": "True"})
+
+        self.assertRaises(
+            exception.LockMemoryForbidden,
+            self._test_get_locked_memory_constraint,
+            {},
+            {"hw_locked_memory": "True"})
+
+    def test_get_locked_memory_constraint_image_false(self):
+        # False value of locked_memory will not raise LockMemoryForbidden
+        self.assertFalse(
+            self._test_get_locked_memory_constraint(
+                spec=None,
+                props={"hw_locked_memory": "False"}))
+
+
 class VirtMemoryPagesTestCase(test.NoDBTestCase):
     def test_cell_instance_pagesize(self):
         cell = objects.InstanceNUMACell(
diff --git a/nova/virt/hardware.py b/nova/virt/hardware.py
@@ -1337,6 +1337,48 @@ def _get_constraint_mappings_from_flavor(flavor, key, func):
     return hw_numa_map or None
 
 
+def get_locked_memory_constraint(
+    flavor: 'objects.Flavor',
+    image_meta: 'objects.ImageMeta',
+) -> ty.Optional[bool]:
+    """Validate and return the requested locked memory.
+
+    :param flavor: ``nova.objects.Flavor`` instance
+    :param image_meta: ``nova.objects.ImageMeta`` instance
+    :raises: exception.LockMemoryForbidden if mem_page_size is not set
+        while provide locked_memory value in image or flavor.
+    :returns: The locked memory flag requested.
+    """
+    mem_page_size_flavor, mem_page_size_image = _get_flavor_image_meta(
+        'mem_page_size', flavor, image_meta)
+
+    locked_memory_flavor, locked_memory_image = _get_flavor_image_meta(
+        'locked_memory', flavor, image_meta)
+
+    if locked_memory_flavor is not None:
+        # locked_memory_image is boolean type already
+        locked_memory_flavor = strutils.bool_from_string(locked_memory_flavor)
+
+        if locked_memory_image is not None and (
+                locked_memory_flavor != locked_memory_image
+        ):
+            # We don't allow provide different value to flavor and image
+            raise exception.FlavorImageLockedMemoryConflict(
+                image=locked_memory_image, flavor=locked_memory_flavor)
+
+        locked_memory = locked_memory_flavor
+
+    else:
+        locked_memory = locked_memory_image
+
+    if locked_memory and not (
+            mem_page_size_flavor or mem_page_size_image
+    ):
+        raise exception.LockMemoryForbidden()
+
+    return locked_memory
+
+
 def _get_numa_cpu_constraint(
     flavor: 'objects.Flavor',
     image_meta: 'objects.ImageMeta',
@@ -2107,6 +2149,8 @@ def numa_get_constraints(flavor, image_meta):
     pagesize = _get_numa_pagesize_constraint(flavor, image_meta)
     vpmems = get_vpmems(flavor)
 
+    get_locked_memory_constraint(flavor, image_meta)
+
     # If 'hw:cpu_dedicated_mask' is not found in flavor extra specs, the
     # 'dedicated_cpus' variable is None, while we hope it being an empty set.
     dedicated_cpus = dedicated_cpus or set()
diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py
@@ -6342,6 +6342,11 @@ def _get_guest_memory_backing_config(
                 membacking = vconfig.LibvirtConfigGuestMemoryBacking()
             membacking.locked = True
 
+        if hardware.get_locked_memory_constraint(flavor, image_meta):
+            if not membacking:
+                membacking = vconfig.LibvirtConfigGuestMemoryBacking()
+            membacking.locked = True
+
         return membacking
 
     def _get_memory_backing_hugepages_support(self, inst_topology, numatune):
diff --git a/releasenotes/notes/new_locked_memory_option-b68a031779366828.yaml b/releasenotes/notes/new_locked_memory_option-b68a031779366828.yaml
@@ -0,0 +1,13 @@
+---
+features:
+  - |
+    Add new ``hw:locked_memory`` extra spec and ``hw_locked_memory`` image
+    property to lock memory on libvirt guest. Locking memory marks the guest
+    memory allocations as unmovable and unswappable.
+    ``hw:locked_memory`` extra spec and ``hw_locked_memory`` image property
+    accept boolean values in string format like 'Yes' or 'false' value.
+    Exception `LockMemoryForbidden` will raise, if you set lock memory value
+    but not set either flavor extra spec
+    ``hw:mem_page_size`` or image property ``hw_mem_page_size``,
+    so we can ensure that the scheduler can actually account for this correctly
+    and prevent out of memory events.

Original file line number	Diff line number	Diff line change
`@@ -4,5 +4,5 @@`
`4`	`4`	`"hw_architecture": "x86_64"`
`5`	`5`	`},`
`6`	`6`	`"nova_object.name": "ImageMetaPropsPayload",`
`7`		`- "nova_object.version": "1.10"`
	`7`	`+ "nova_object.version": "1.11"`
`8`	`8`	`}`