tests: Add functional test for vDPA device

Add a simple test to validate behavior with vDPA devices. Most of this
is simply fleshing out the fixtures we use to fake out vDPA devices and
generally tweaking things to make them better.

Change-Id: I1423d8a9652751b667463f90c69eae1a054dd776
Signed-off-by: Stephen Finucane <[email protected]>
Co-authored-by: Sean Mooney <[email protected]>

Author: stephenfin

nova/pci/stats.py

@@ -441,7 +441,7 @@ def _filter_pools(self, pools, request, numa_cells):
 
         if after_count < before_count:
             LOG.debug(
-                'Dropped %d devices due to mismatched PCI attribute(s)',
+                'Dropped %d device(s) due to mismatched PCI attribute(s)',
                 before_count - after_count
             )
 
@@ -458,7 +458,7 @@ def _filter_pools(self, pools, request, numa_cells):
 
         if after_count < before_count:
             LOG.debug(
-                'Dropped %d devices as they are on the wrong NUMA node(s)',
+                'Dropped %d device(s) as they are on the wrong NUMA node(s)',
                 before_count - after_count
             )
 
@@ -474,7 +474,7 @@ def _filter_pools(self, pools, request, numa_cells):
 
         if after_count < before_count:
             LOG.debug(
-                'Dropped %d devices as they are PFs which we have not '
+                'Dropped %d device(s) as they are PFs which we have not '
                 'requested',
                 before_count - after_count
             )
@@ -491,7 +491,7 @@ def _filter_pools(self, pools, request, numa_cells):
 
         if after_count < before_count:
             LOG.debug(
-                'Dropped %d devices as they are VDPA devices which we have '
+                'Dropped %d device(s) as they are VDPA devices which we have '
                 'not requested',
                 before_count - after_count
             )

nova/tests/functional/integrated_helpers.py

@@ -413,6 +413,24 @@ def _reboot_server(self, server, hard=False, expected_state='ACTIVE'):
         fake_notifier.wait_for_versioned_notifications('instance.reboot.end')
         return self._wait_for_state_change(server, expected_state)
 
+    def _attach_interface(self, server, port_uuid):
+        """attach a neutron port to a server."""
+        body = {
+            "interfaceAttachment": {
+                "port_id": port_uuid
+            }
+        }
+        attachment = self.api.attach_interface(server['id'], body)
+        fake_notifier.wait_for_versioned_notifications(
+            'instance.interface_attach.end')
+        return attachment
+
+    def _detach_interface(self, server, port_uuid):
+        """detach a neutron port form a server."""
+        self.api.detach_interface(server['id'], port_uuid)
+        fake_notifier.wait_for_versioned_notifications(
+            'instance.interface_detach.end')
+
     def _rebuild_server(self, server, image_uuid, expected_state='ACTIVE'):
         """Rebuild a server."""
         self.api.post_server_action(

nova/tests/functional/libvirt/base.py

@@ -85,7 +85,7 @@ def _setup_scheduler_service(self):
         return self.start_service('scheduler')
 
     def _get_connection(
-        self, host_info=None, pci_info=None, mdev_info=None,
+        self, host_info=None, pci_info=None, mdev_info=None, vdpa_info=None,
         libvirt_version=None, qemu_version=None, hostname=None,
     ):
         if not host_info:
@@ -107,12 +107,14 @@ def _get_connection(
             host_info=host_info,
             pci_info=pci_info,
             mdev_info=mdev_info,
+            vdpa_info=vdpa_info,
             hostname=hostname)
         return fake_connection
 
     def start_compute(
         self, hostname='compute1', host_info=None, pci_info=None,
-        mdev_info=None, libvirt_version=None, qemu_version=None,
+        mdev_info=None, vdpa_info=None, libvirt_version=None,
+        qemu_version=None,
     ):
         """Start a compute service.
 
@@ -129,8 +131,8 @@ def start_compute(
 
         def _start_compute(hostname, host_info):
             fake_connection = self._get_connection(
-                host_info, pci_info, mdev_info, libvirt_version, qemu_version,
-                hostname,
+                host_info, pci_info, mdev_info, vdpa_info, libvirt_version,
+                qemu_version, hostname,
             )
             # This is fun. Firstly we need to do a global'ish mock so we can
             # actually start the service.
@@ -299,8 +301,8 @@ class LibvirtNeutronFixture(nova_fixtures.NeutronFixture):
                 'subnet_id': subnet_4['id']
             }
         ],
-        'binding:vif_type': 'hw_veb',
         'binding:vif_details': {'vlan': 42},
+        'binding:vif_type': 'hw_veb',
         'binding:vnic_type': 'direct',
     }
 

nova/tests/functional/libvirt/test_pci_sriov_servers.py

@@ -23,6 +23,7 @@
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
+from oslo_utils.fixture import uuidsentinel as uuids
 from oslo_utils import units
 
 import nova
@@ -275,9 +276,7 @@ def fake_create(cls, xml, host):
         self.assertNotIn('binding:profile', port)
 
         # create a server using the VF via neutron
-        flavor_id = self._create_flavor()
         self._create_server(
-            flavor_id=flavor_id,
             networks=[
                 {'port': base.LibvirtNeutronFixture.network_4_port_1['id']},
             ],
@@ -548,9 +547,7 @@ def test_create_server_after_change_in_nonsriov_pf_to_sriov_pf(self):
         self.neutron.create_port({'port': self.neutron.network_4_port_1})
 
         # create a server using the VF via neutron
-        flavor_id = self._create_flavor()
         self._create_server(
-            flavor_id=flavor_id,
             networks=[
                 {'port': base.LibvirtNeutronFixture.network_4_port_1['id']},
             ],
@@ -672,6 +669,222 @@ def test_detach_direct_physical(self):
             self.neutron.sriov_pf_port2['id'])
 
 
+class VDPAServersTest(_PCIServersTestBase):
+
+    # this is needed for os_compute_api:os-migrate-server:migrate policy
+    ADMIN_API = True
+    microversion = 'latest'
+
+    # Whitelist both the PF and VF; in reality, you probably wouldn't do this
+    # but we want to make sure that the PF is correctly taken off the table
+    # once any VF is used
+    PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps(x) for x in (
+        {
+            'vendor_id': '15b3',
+            'product_id': '101d',
+            'physical_network': 'physnet4',
+        },
+        {
+            'vendor_id': '15b3',
+            'product_id': '101e',
+            'physical_network': 'physnet4',
+        },
+    )]
+    # No need for aliases as these test will request SRIOV via neutron
+    PCI_ALIAS = []
+
+    NUM_PFS = 1
+    NUM_VFS = 4
+
+    FAKE_LIBVIRT_VERSION = 6_009_000  # 6.9.0
+    FAKE_QEMU_VERSION = 5_001_000  # 5.1.0
+
+    def setUp(self):
+        super().setUp()
+
+        # The ultimate base class _IntegratedTestBase uses NeutronFixture but
+        # we need a bit more intelligent neutron for these tests. Applying the
+        # new fixture here means that we re-stub what the previous neutron
+        # fixture already stubbed.
+        self.neutron = self.useFixture(base.LibvirtNeutronFixture(self))
+
+    def start_compute(self):
+        vf_ratio = self.NUM_VFS // self.NUM_PFS
+
+        pci_info = fakelibvirt.HostPCIDevicesInfo(
+            num_pci=0, num_pfs=0, num_vfs=0)
+        vdpa_info = fakelibvirt.HostVDPADevicesInfo()
+
+        pci_info.add_device(
+            dev_type='PF',
+            bus=0x6,
+            slot=0x0,
+            function=0,
+            iommu_group=40,  # totally arbitrary number
+            numa_node=0,
+            vf_ratio=vf_ratio,
+            vend_id='15b3',
+            vend_name='Mellanox Technologies',
+            prod_id='101d',
+            prod_name='MT2892 Family [ConnectX-6 Dx]',
+            driver_name='mlx5_core')
+
+        for idx in range(self.NUM_VFS):
+            vf = pci_info.add_device(
+                dev_type='VF',
+                bus=0x6,
+                slot=0x0,
+                function=idx + 1,
+                iommu_group=idx + 41,  # totally arbitrary number + offset
+                numa_node=0,
+                vf_ratio=vf_ratio,
+                parent=(0x6, 0x0, 0),
+                vend_id='15b3',
+                vend_name='Mellanox Technologies',
+                prod_id='101e',
+                prod_name='ConnectX Family mlx5Gen Virtual Function',
+                driver_name='mlx5_core')
+            vdpa_info.add_device(f'vdpa_vdpa{idx}', idx, vf)
+
+        return super().start_compute(
+            pci_info=pci_info, vdpa_info=vdpa_info,
+            libvirt_version=self.FAKE_LIBVIRT_VERSION,
+            qemu_version=self.FAKE_QEMU_VERSION)
+
+    def create_vdpa_port(self):
+        vdpa_port = {
+            'id': uuids.vdpa_port,
+            'network_id': self.neutron.network_4['id'],
+            'status': 'ACTIVE',
+            'mac_address': 'b5:bc:2e:e7:51:ee',
+            'fixed_ips': [
+                {
+                    'ip_address': '192.168.4.6',
+                    'subnet_id': self.neutron.subnet_4['id']
+                }
+            ],
+            'binding:vif_details': {},
+            'binding:vif_type': 'ovs',
+            'binding:vnic_type': 'vdpa',
+        }
+
+        # create the port
+        self.neutron.create_port({'port': vdpa_port})
+        return vdpa_port
+
+    def test_create_server(self):
+        """Create an instance using a neutron-provisioned vDPA VIF."""
+
+        orig_create = nova.virt.libvirt.guest.Guest.create
+
+        def fake_create(cls, xml, host):
+            tree = etree.fromstring(xml)
+            elem = tree.find('./devices/interface/[@type="vdpa"]')
+
+            # compare source device
+            # the MAC address is derived from the neutron port, while the
+            # source dev path assumes we attach vDPA devs in order
+            expected = """
+                <interface type="vdpa">
+                  <mac address="b5:bc:2e:e7:51:ee"/>
+                  <model type="virtio"/>
+                  <source dev="/dev/vhost-vdpa-3"/>
+                </interface>"""
+            actual = etree.tostring(elem, encoding='unicode')
+
+            self.assertXmlEqual(expected, actual)
+
+            return orig_create(xml, host)
+
+        self.stub_out(
+            'nova.virt.libvirt.guest.Guest.create',
+            fake_create,
+        )
+
+        hostname = self.start_compute()
+        num_pci = self.NUM_PFS + self.NUM_VFS
+
+        # both the PF and VF with vDPA capabilities (dev_type=vdpa) should have
+        # been counted
+        self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci)
+
+        # create the port
+        vdpa_port = self.create_vdpa_port()
+
+        # ensure the binding details are currently unset
+        port = self.neutron.show_port(vdpa_port['id'])['port']
+        self.assertNotIn('binding:profile', port)
+
+        # create a server using the vDPA device via neutron
+        self._create_server(networks=[{'port': vdpa_port['id']}])
+
+        # ensure there is one less VF available and that the PF is no longer
+        # usable
+        self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2)
+
+        # ensure the binding details sent to "neutron" were correct
+        port = self.neutron.show_port(vdpa_port['id'])['port']
+        self.assertIn('binding:profile', port)
+        self.assertEqual(
+            {
+                'pci_vendor_info': '15b3:101e',
+                'pci_slot': '0000:06:00.4',
+                'physical_network': 'physnet4',
+            },
+            port['binding:profile'],
+        )
+
+    def _test_common(self, op, *args, **kwargs):
+        self.start_compute()
+
+        # create the port and a server, with the port attached to the server
+        vdpa_port = self.create_vdpa_port()
+        server = self._create_server(networks=[{'port': vdpa_port['id']}])
+
+        # attempt the unsupported action and ensure it fails
+        ex = self.assertRaises(
+            client.OpenStackApiException,
+            op, server, *args, **kwargs)
+        self.assertIn(
+            'not supported for instance with vDPA ports',
+            ex.response.text)
+
+    def test_attach_interface(self):
+        self.start_compute()
+
+        # create the port and a server, but don't attach the port to the server
+        # yet
+        vdpa_port = self.create_vdpa_port()
+        server = self._create_server(networks='none')
+
+        # attempt to attach the port to the server
+        ex = self.assertRaises(
+            client.OpenStackApiException,
+            self._attach_interface, server, vdpa_port['id'])
+        self.assertIn(
+            'not supported for instance with vDPA ports',
+            ex.response.text)
+
+    def test_detach_interface(self):
+        self._test_common(self._detach_interface, uuids.vdpa_port)
+
+    def test_shelve(self):
+        self._test_common(self._shelve_server)
+
+    def test_suspend(self):
+        self._test_common(self._suspend_server)
+
+    def test_evacute(self):
+        self._test_common(self._evacuate_server)
+
+    def test_resize(self):
+        flavor_id = self._create_flavor()
+        self._test_common(self._resize_server, flavor_id)
+
+    def test_cold_migrate(self):
+        self._test_common(self._migrate_server)
+
+
 class PCIServersTest(_PCIServersTestBase):
 
     ADMIN_API = True