Merge "libvirt: Add vIOMMU device to guest"

Zuul · openstack-gerrit · commit 457806e2064d · 2022-09-01T19:56:09.000Z
diff --git a/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json b/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json
@@ -4,5 +4,5 @@
         "hw_architecture": "x86_64"
     },
     "nova_object.name": "ImageMetaPropsPayload",
-    "nova_object.version": "1.11"
+    "nova_object.version": "1.12"
 }
diff --git a/doc/source/admin/pci-passthrough.rst b/doc/source/admin/pci-passthrough.rst
@@ -429,3 +429,61 @@ the PF and its VFs are made available for consumption.
 
 
 For deeper technical details please read the `nova specification. <https://specs.openstack.org/openstack/nova-specs/specs/zed/approved/pci-device-tracking-in-placement.html>`_
+
+
+Virtual IOMMU support
+---------------------
+
+With provided :nova:extra-spec:`hw:viommu_model` flavor extra spec or equivalent
+image metadata property ``hw_viommu_model`` and with the guest CPU architecture
+and OS allows, we can enable vIOMMU in libvirt driver.
+
+.. note::
+
+    Enable vIOMMU might introduce significant performance overhead.
+    You can see performance comparision table from
+    `AMD vIOMMU session on KVM Forum 2021`_.
+    For the above reason, vIOMMU should only be enabled for workflow that
+    require it.
+
+.. _`AMD vIOMMU session on KVM Forum 2021`: https://static.sched.com/hosted_files/kvmforum2021/da/vIOMMU%20KVM%20Forum%202021%20-%20v4.pdf
+
+Here are four possible values allowed for ``hw:viommu_model``
+(and ``hw_viommu_model``):
+
+**virtio**
+    Supported on Libvirt since 8.3.0, for Q35 and ARM virt guests.
+
+**smmuv3**
+    Supported on Libvirt since 5.5.0, for ARM virt guests.
+**intel**
+    Supported for for Q35 guests.
+
+**auto**
+    This option will translate to ``virtio`` if Libvirt supported,
+    else ``intel`` on X86 (Q35) and ``smmuv3`` on AArch64.
+
+For the viommu attributes:
+
+* ``intremap``, ``caching_mode``, and ``iotlb``
+  options for viommu (These attributes are driver attributes defined in
+  `Libvirt IOMMU Domain`_) will direcly enabled.
+
+* ``eim`` will directly enabled if machine type is Q35.
+  ``eim`` is driver attribute defined in `Libvirt IOMMU Domain`_.
+
+.. note::
+
+    eim(Extended Interrupt Mode) attribute (with possible values on and off)
+    can be used to configure Extended Interrupt Mode.
+    A q35 domain with split I/O APIC (as described in hypervisor features),
+    and both interrupt remapping and EIM turned on for the IOMMU, will be
+    able to use more than 255 vCPUs. Since 3.4.0 (QEMU/KVM only).
+
+* ``aw_bits`` attribute can used to set the address width to allow mapping
+  larger iova addresses in the guest. Since Qemu current supported
+  values are 39 and 48, we directly set this to larger width (48)
+  if Libvirt supported.
+  ``aw_bits`` is driver attribute defined in `Libvirt IOMMU Domain`_.
+
+.. _`Libvirt IOMMU Domain`: https://libvirt.org/formatdomain.html#iommu-devices
diff --git a/nova/api/validation/extra_specs/hw.py b/nova/api/validation/extra_specs/hw.py
@@ -511,6 +511,22 @@
             ],
         },
     ),
+    base.ExtraSpecValidator(
+        name='hw:viommu_model',
+        description=(
+            'This can be used to set model for virtual IOMMU device.'
+        ),
+        value={
+            'type': str,
+            'enum': [
+                'intel',
+                'smmuv3',
+                'virtio',
+                'auto'
+            ],
+            'description': 'model for vIOMMU',
+        },
+    ),
 ]
 
 ephemeral_encryption_validators = [
diff --git a/nova/exception.py b/nova/exception.py
@@ -207,6 +207,16 @@ class Invalid(NovaException):
     code = 400
 
 
+class InvalidVIOMMUMachineType(Invalid):
+    msg_fmt = _("vIOMMU is not supported by Current machine type %(mtype)s "
+                "(Architecture: %(arch)s).")
+
+
+class InvalidVIOMMUArchitecture(Invalid):
+    msg_fmt = _("vIOMMU required either x86 or AArch64 architecture, "
+                "but given architecture %(arch)s.")
+
+
 class InvalidConfiguration(Invalid):
     msg_fmt = _("Configuration is Invalid.")
 
diff --git a/nova/notifications/objects/image.py b/nova/notifications/objects/image.py
@@ -129,7 +129,8 @@ class ImageMetaPropsPayload(base.NotificationPayloadBase):
     # Version 1.10: Added 'hw_ephemeral_encryption' and
     #                     'hw_ephemeral_encryption_format' fields
     # Version 1.11: Added 'hw_locked_memory' field
-    VERSION = '1.11'
+    # Version 1.12: Added 'hw_viommu_model' field
+    VERSION = '1.12'
 
     SCHEMA = {
         k: ('image_meta_props', k) for k in image_meta.ImageMetaProps.fields}
diff --git a/nova/objects/fields.py b/nova/objects/fields.py
@@ -616,6 +616,16 @@ def coerce(self, obj, attr, value):
         return super(VIFModel, self).coerce(obj, attr, value)
 
 
+class VIOMMUModel(BaseNovaEnum):
+
+    INTEL = 'intel'
+    SMMUV3 = 'smmuv3'
+    VIRTIO = 'virtio'
+    AUTO = 'auto'
+
+    ALL = (INTEL, SMMUV3, VIRTIO, AUTO)
+
+
 class VMMode(BaseNovaEnum):
     """Represents possible vm modes for instances.
 
@@ -1301,6 +1311,10 @@ class VIFModelField(BaseEnumField):
     AUTO_TYPE = VIFModel()
 
 
+class VIOMMUModelField(BaseEnumField):
+    AUTO_TYPE = VIOMMUModel()
+
+
 class VMModeField(BaseEnumField):
     AUTO_TYPE = VMMode()
 
diff --git a/nova/objects/image_meta.py b/nova/objects/image_meta.py
@@ -191,14 +191,17 @@ class ImageMetaProps(base.NovaObject):
     # Version 1.32: Added 'hw_ephemeral_encryption' and
     #                     'hw_ephemeral_encryption_format' fields
     # Version 1.33: Added 'hw_locked_memory' field
+    # Version 1.34: Added 'hw_viommu_model' field
     # NOTE(efried): When bumping this version, the version of
     # ImageMetaPropsPayload must also be bumped. See its docstring for details.
-    VERSION = '1.33'
+    VERSION = '1.34'
 
     def obj_make_compatible(self, primitive, target_version):
         super(ImageMetaProps, self).obj_make_compatible(primitive,
                                                         target_version)
         target_version = versionutils.convert_version_to_tuple(target_version)
+        if target_version < (1, 34):
+            primitive.pop('hw_viommu_model', None)
         if target_version < (1, 33):
             primitive.pop('hw_locked_memory', None)
         if target_version < (1, 32):
@@ -446,6 +449,9 @@ def obj_make_compatible(self, primitive, target_version):
         # name of a NIC device model eg virtio, e1000, rtl8139
         'hw_vif_model': fields.VIFModelField(),
 
+        # name of IOMMU device model eg virtio, intel, smmuv3, or auto
+        'hw_viommu_model': fields.VIOMMUModelField(),
+
         # "xen" vs "hvm"
         'hw_vm_mode': fields.VMModeField(),
 
diff --git a/nova/tests/functional/notification_sample_tests/test_instance.py b/nova/tests/functional/notification_sample_tests/test_instance.py
@@ -1231,7 +1231,7 @@ def test_rebuild_server(self):
                     'nova_object.data': {},
                     'nova_object.name': 'ImageMetaPropsPayload',
                     'nova_object.namespace': 'nova',
-                    'nova_object.version': '1.11',
+                    'nova_object.version': '1.12',
                 },
                 'image.size': 58145823,
                 'image.tags': [],
@@ -1327,7 +1327,7 @@ def test_rebuild_server_with_trusted_cert(self):
                     'nova_object.data': {},
                     'nova_object.name': 'ImageMetaPropsPayload',
                     'nova_object.namespace': 'nova',
-                    'nova_object.version': '1.11',
+                    'nova_object.version': '1.12',
                 },
                 'image.size': 58145823,
                 'image.tags': [],
diff --git a/nova/tests/unit/api/validation/extra_specs/test_validators.py b/nova/tests/unit/api/validation/extra_specs/test_validators.py
@@ -74,6 +74,10 @@ def test_value__str(self):
             ('hw:pci_numa_affinity_policy', 'preferred'),
             ('hw:pci_numa_affinity_policy', 'socket'),
             ('hw:cpu_policy', 'mixed'),
+            ('hw:viommu_model', 'auto'),
+            ('hw:viommu_model', 'intel'),
+            ('hw:viommu_model', 'smmuv3'),
+            ('hw:viommu_model', 'virtio'),
         )
         for key, value in valid_specs:
             validators.validate(key, value)
@@ -92,6 +96,7 @@ def test_value__str(self):
             ('hw:pci_numa_affinity_policy', 'requird'),
             ('hw:pci_numa_affinity_policy', 'prefrred'),
             ('hw:pci_numa_affinity_policy', 'socet'),
+            ('hw:viommu_model', 'autt'),
         )
         for key, value in invalid_specs:
             with testtools.ExpectedException(exception.ValidationError):
diff --git a/nova/tests/unit/notifications/objects/test_notification.py b/nova/tests/unit/notifications/objects/test_notification.py
@@ -386,7 +386,7 @@ def test_payload_is_not_generated_if_notification_format_is_unversioned(
     # ImageMetaProps, so when you see a fail here for that reason, you must
     # *also* bump the version of ImageMetaPropsPayload. See its docstring for
     # more information.
-    'ImageMetaPropsPayload': '1.11-938809cd33367c52cbc814fb9b6783dc',
+    'ImageMetaPropsPayload': '1.12-b9c64832d7772c1973e913bacbe0e8f9',
     'InstanceActionNotification': '1.0-a73147b93b520ff0061865849d3dfa56',
     'InstanceActionPayload': '1.8-4fa3da9cbf0761f1f700ae578f36dc2f',
     'InstanceActionRebuildNotification':
diff --git a/nova/tests/unit/objects/test_image_meta.py b/nova/tests/unit/objects/test_image_meta.py
@@ -538,3 +538,19 @@ def test_obj_make_compatible_socket_policy(self):
             hw_pci_numa_affinity_policy=fields.PCINUMAAffinityPolicy.SOCKET)
         self.assertRaises(exception.ObjectActionError,
                           obj.obj_to_primitive, '1.27')
+
+    def test_obj_make_compatible_viommu_model(self):
+        """Check 'hw_viommu_model' compatibility."""
+        # assert that 'hw_viommu_model' is supported on a suitably new version
+        obj = objects.ImageMetaProps(
+            hw_viommu_model=objects.fields.VIOMMUModel.VIRTIO,
+        )
+        primitive = obj.obj_to_primitive('1.34')
+        self.assertIn('hw_viommu_model', primitive['nova_object.data'])
+        self.assertEqual(
+            objects.fields.VIOMMUModel.VIRTIO,
+            primitive['nova_object.data']['hw_viommu_model'])
+
+        # and is absent on older versions
+        primitive = obj.obj_to_primitive('1.33')
+        self.assertNotIn('hw_viommu_model', primitive['nova_object.data'])
diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py
@@ -1072,7 +1072,7 @@ def obj_name(cls):
     'HyperVLiveMigrateData': '1.4-e265780e6acfa631476c8170e8d6fce0',
     'IDEDeviceBus': '1.0-29d4c9f27ac44197f01b6ac1b7e16502',
     'ImageMeta': '1.8-642d1b2eb3e880a367f37d72dd76162d',
-    'ImageMetaProps': '1.33-6b7a29f769e6b8eee3f05832d78c85a2',
+    'ImageMetaProps': '1.34-29b3a6b7fe703f36bfd240d914f16c21',
     'Instance': '2.7-d187aec68cad2e4d8b8a03a68e4739ce',
     'InstanceAction': '1.2-9a5abc87fdd3af46f45731960651efb5',
     'InstanceActionEvent': '1.4-5b1f361bd81989f8bb2c20bb7e8a4cb4',
diff --git a/nova/tests/unit/virt/libvirt/test_config.py b/nova/tests/unit/virt/libvirt/test_config.py
@@ -16,6 +16,7 @@
 from oslo_utils.fixture import uuidsentinel as uuids
 from oslo_utils import units
 
+from nova import exception
 from nova.objects import fields as obj_fields
 from nova import test
 from nova.tests.fixtures import libvirt_data as fake_libvirt_data
@@ -70,6 +71,23 @@ def test_config_parse(self):
         obj = config.LibvirtConfigObject(root_name="demo")
         obj.parse_str(inxml)
 
+    def test_parse_on_off_str(self):
+        obj = config.LibvirtConfigObject(root_name="demo")
+        self.assertTrue(obj.parse_on_off_str('on'))
+        self.assertFalse(obj.parse_on_off_str('off'))
+        self.assertFalse(obj.parse_on_off_str(None))
+        self.assertRaises(exception.InvalidInput, obj.parse_on_off_str, 'foo')
+
+    def test_get_yes_no_str(self):
+        obj = config.LibvirtConfigObject(root_name="demo")
+        self.assertEqual('yes', obj.get_yes_no_str(True))
+        self.assertEqual('no', obj.get_yes_no_str(False))
+
+    def test_get_on_off_str(self):
+        obj = config.LibvirtConfigObject(root_name="demo")
+        self.assertEqual('on', obj.get_on_off_str(True))
+        self.assertEqual('off', obj.get_on_off_str(False))
+
 
 class LibvirtConfigCapsTest(LibvirtConfigBaseTest):
 
@@ -2365,6 +2383,13 @@ def test_feature_pmu(self):
             xml = obj.to_xml()
             self.assertXmlEqual(xml, "<pmu state='off'/>")
 
+    def test_feature_ioapic(self):
+        obj = config.LibvirtConfigGuestFeatureIOAPIC()
+        obj.driver = "libvirt"
+
+        xml = obj.to_xml()
+        self.assertXmlEqual(xml, "<ioapic driver='libvirt'/>")
+
 
 class LibvirtConfigGuestTest(LibvirtConfigBaseTest):
 
@@ -3993,6 +4018,28 @@ def test_config_vpmem(self):
           </memory>""")
 
 
+class LibvirtConfigGuestIOMMUTest(LibvirtConfigBaseTest):
+
+    def test_config_iommu(self):
+        obj = config.LibvirtConfigGuestIOMMU()
+        obj.model = "intel"
+        obj.interrupt_remapping = True
+        obj.caching_mode = True
+        obj.aw_bits = 48
+        obj.eim = True
+        obj.iotlb = True
+
+        xml = obj.to_xml()
+        self.assertXmlEqual(
+            xml,
+            """
+<iommu model='intel'>
+  <driver intremap='on' caching_mode='on' aw_bits='48' eim='on' iotlb='on'/>
+</iommu>
+            """,
+        )
+
+
 class LibvirtConfigDomainCapsVideoModelsTests(LibvirtConfigBaseTest):
 
     def test_parse_video_model(self):
diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py
diff --git a/nova/virt/libvirt/config.py b/nova/virt/libvirt/config.py
diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py
diff --git a/releasenotes/notes/guest-iommu-device-4795c3a060aca424.yaml b/releasenotes/notes/guest-iommu-device-4795c3a060aca424.yaml

Original file line number	Diff line number	Diff line change
`@@ -4,5 +4,5 @@`
`4`	`4`	`"hw_architecture": "x86_64"`
`5`	`5`	`},`
`6`	`6`	`"nova_object.name": "ImageMetaPropsPayload",`
`7`		`- "nova_object.version": "1.11"`
	`7`	`+ "nova_object.version": "1.12"`
`8`	`8`	`}`