Merge "tests: Add functional test for vDPA device"

Author: Zuul

nova/pci/stats.py

@@ -441,7 +441,7 @@ def _filter_pools(self, pools, request, numa_cells):
 
         if after_count < before_count:
             LOG.debug(
-                'Dropped %d devices due to mismatched PCI attribute(s)',
+                'Dropped %d device(s) due to mismatched PCI attribute(s)',
                 before_count - after_count
             )
 
@@ -458,7 +458,7 @@ def _filter_pools(self, pools, request, numa_cells):
 
         if after_count < before_count:
             LOG.debug(
-                'Dropped %d devices as they are on the wrong NUMA node(s)',
+                'Dropped %d device(s) as they are on the wrong NUMA node(s)',
                 before_count - after_count
             )
 
@@ -474,7 +474,7 @@ def _filter_pools(self, pools, request, numa_cells):
 
         if after_count < before_count:
             LOG.debug(
-                'Dropped %d devices as they are PFs which we have not '
+                'Dropped %d device(s) as they are PFs which we have not '
                 'requested',
                 before_count - after_count
             )
@@ -491,7 +491,7 @@ def _filter_pools(self, pools, request, numa_cells):
 
         if after_count < before_count:
             LOG.debug(
-                'Dropped %d devices as they are VDPA devices which we have '
+                'Dropped %d device(s) as they are VDPA devices which we have '
                 'not requested',
                 before_count - after_count
             )

nova/tests/functional/integrated_helpers.py

@@ -413,6 +413,24 @@ def _reboot_server(self, server, hard=False, expected_state='ACTIVE'):
         fake_notifier.wait_for_versioned_notifications('instance.reboot.end')
         return self._wait_for_state_change(server, expected_state)
 
+    def _attach_interface(self, server, port_uuid):
+        """attach a neutron port to a server."""
+        body = {
+            "interfaceAttachment": {
+                "port_id": port_uuid
+            }
+        }
+        attachment = self.api.attach_interface(server['id'], body)
+        fake_notifier.wait_for_versioned_notifications(
+            'instance.interface_attach.end')
+        return attachment
+
+    def _detach_interface(self, server, port_uuid):
+        """detach a neutron port form a server."""
+        self.api.detach_interface(server['id'], port_uuid)
+        fake_notifier.wait_for_versioned_notifications(
+            'instance.interface_detach.end')
+
     def _rebuild_server(self, server, image_uuid, expected_state='ACTIVE'):
         """Rebuild a server."""
         self.api.post_server_action(

nova/tests/functional/libvirt/base.py

@@ -85,7 +85,7 @@ def _setup_scheduler_service(self):
         return self.start_service('scheduler')
 
     def _get_connection(
-        self, host_info=None, pci_info=None, mdev_info=None,
+        self, host_info=None, pci_info=None, mdev_info=None, vdpa_info=None,
         libvirt_version=None, qemu_version=None, hostname=None,
     ):
         if not host_info:
@@ -107,12 +107,14 @@ def _get_connection(
             host_info=host_info,
             pci_info=pci_info,
             mdev_info=mdev_info,
+            vdpa_info=vdpa_info,
             hostname=hostname)
         return fake_connection
 
     def start_compute(
         self, hostname='compute1', host_info=None, pci_info=None,
-        mdev_info=None, libvirt_version=None, qemu_version=None,
+        mdev_info=None, vdpa_info=None, libvirt_version=None,
+        qemu_version=None,
     ):
         """Start a compute service.
 
@@ -129,8 +131,8 @@ def start_compute(
 
         def _start_compute(hostname, host_info):
             fake_connection = self._get_connection(
-                host_info, pci_info, mdev_info, libvirt_version, qemu_version,
-                hostname,
+                host_info, pci_info, mdev_info, vdpa_info, libvirt_version,
+                qemu_version, hostname,
             )
             # This is fun. Firstly we need to do a global'ish mock so we can
             # actually start the service.
@@ -299,8 +301,8 @@ class LibvirtNeutronFixture(nova_fixtures.NeutronFixture):
                 'subnet_id': subnet_4['id']
             }
         ],
-        'binding:vif_type': 'hw_veb',
         'binding:vif_details': {'vlan': 42},
+        'binding:vif_type': 'hw_veb',
         'binding:vnic_type': 'direct',
     }
 

nova/tests/functional/libvirt/test_pci_sriov_servers.py

@@ -23,6 +23,7 @@
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
+from oslo_utils.fixture import uuidsentinel as uuids
 from oslo_utils import units
 
 import nova
@@ -275,9 +276,7 @@ def fake_create(cls, xml, host):
         self.assertNotIn('binding:profile', port)
 
         # create a server using the VF via neutron
-        flavor_id = self._create_flavor()
         self._create_server(
-            flavor_id=flavor_id,
             networks=[
                 {'port': base.LibvirtNeutronFixture.network_4_port_1['id']},
             ],
@@ -548,9 +547,7 @@ def test_create_server_after_change_in_nonsriov_pf_to_sriov_pf(self):
         self.neutron.create_port({'port': self.neutron.network_4_port_1})
 
         # create a server using the VF via neutron
-        flavor_id = self._create_flavor()
         self._create_server(
-            flavor_id=flavor_id,
             networks=[
                 {'port': base.LibvirtNeutronFixture.network_4_port_1['id']},
             ],
@@ -672,6 +669,222 @@ def test_detach_direct_physical(self):
             self.neutron.sriov_pf_port2['id'])
 
 
+class VDPAServersTest(_PCIServersTestBase):
+
+    # this is needed for os_compute_api:os-migrate-server:migrate policy
+    ADMIN_API = True
+    microversion = 'latest'
+
+    # Whitelist both the PF and VF; in reality, you probably wouldn't do this
+    # but we want to make sure that the PF is correctly taken off the table
+    # once any VF is used
+    PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps(x) for x in (
+        {
+            'vendor_id': '15b3',
+            'product_id': '101d',
+            'physical_network': 'physnet4',
+        },
+        {
+            'vendor_id': '15b3',
+            'product_id': '101e',
+            'physical_network': 'physnet4',
+        },
+    )]
+    # No need for aliases as these test will request SRIOV via neutron
+    PCI_ALIAS = []
+
+    NUM_PFS = 1
+    NUM_VFS = 4
+
+    FAKE_LIBVIRT_VERSION = 6_009_000  # 6.9.0
+    FAKE_QEMU_VERSION = 5_001_000  # 5.1.0
+
+    def setUp(self):
+        super().setUp()
+
+        # The ultimate base class _IntegratedTestBase uses NeutronFixture but
+        # we need a bit more intelligent neutron for these tests. Applying the
+        # new fixture here means that we re-stub what the previous neutron
+        # fixture already stubbed.
+        self.neutron = self.useFixture(base.LibvirtNeutronFixture(self))
+
+    def start_compute(self):
+        vf_ratio = self.NUM_VFS // self.NUM_PFS
+
+        pci_info = fakelibvirt.HostPCIDevicesInfo(
+            num_pci=0, num_pfs=0, num_vfs=0)
+        vdpa_info = fakelibvirt.HostVDPADevicesInfo()
+
+        pci_info.add_device(
+            dev_type='PF',
+            bus=0x6,
+            slot=0x0,
+            function=0,
+            iommu_group=40,  # totally arbitrary number
+            numa_node=0,
+            vf_ratio=vf_ratio,
+            vend_id='15b3',
+            vend_name='Mellanox Technologies',
+            prod_id='101d',
+            prod_name='MT2892 Family [ConnectX-6 Dx]',
+            driver_name='mlx5_core')
+
+        for idx in range(self.NUM_VFS):
+            vf = pci_info.add_device(
+                dev_type='VF',
+                bus=0x6,
+                slot=0x0,
+                function=idx + 1,
+                iommu_group=idx + 41,  # totally arbitrary number + offset
+                numa_node=0,
+                vf_ratio=vf_ratio,
+                parent=(0x6, 0x0, 0),
+                vend_id='15b3',
+                vend_name='Mellanox Technologies',
+                prod_id='101e',
+                prod_name='ConnectX Family mlx5Gen Virtual Function',
+                driver_name='mlx5_core')
+            vdpa_info.add_device(f'vdpa_vdpa{idx}', idx, vf)
+
+        return super().start_compute(
+            pci_info=pci_info, vdpa_info=vdpa_info,
+            libvirt_version=self.FAKE_LIBVIRT_VERSION,
+            qemu_version=self.FAKE_QEMU_VERSION)
+
+    def create_vdpa_port(self):
+        vdpa_port = {
+            'id': uuids.vdpa_port,
+            'network_id': self.neutron.network_4['id'],
+            'status': 'ACTIVE',
+            'mac_address': 'b5:bc:2e:e7:51:ee',
+            'fixed_ips': [
+                {
+                    'ip_address': '192.168.4.6',
+                    'subnet_id': self.neutron.subnet_4['id']
+                }
+            ],
+            'binding:vif_details': {},
+            'binding:vif_type': 'ovs',
+            'binding:vnic_type': 'vdpa',
+        }
+
+        # create the port
+        self.neutron.create_port({'port': vdpa_port})
+        return vdpa_port
+
+    def test_create_server(self):
+        """Create an instance using a neutron-provisioned vDPA VIF."""
+
+        orig_create = nova.virt.libvirt.guest.Guest.create
+
+        def fake_create(cls, xml, host):
+            tree = etree.fromstring(xml)
+            elem = tree.find('./devices/interface/[@type="vdpa"]')
+
+            # compare source device
+            # the MAC address is derived from the neutron port, while the
+            # source dev path assumes we attach vDPA devs in order
+            expected = """
+                <interface type="vdpa">
+                  <mac address="b5:bc:2e:e7:51:ee"/>
+                  <model type="virtio"/>
+                  <source dev="/dev/vhost-vdpa-3"/>
+                </interface>"""
+            actual = etree.tostring(elem, encoding='unicode')
+
+            self.assertXmlEqual(expected, actual)
+
+            return orig_create(xml, host)
+
+        self.stub_out(
+            'nova.virt.libvirt.guest.Guest.create',
+            fake_create,
+        )
+
+        hostname = self.start_compute()
+        num_pci = self.NUM_PFS + self.NUM_VFS
+
+        # both the PF and VF with vDPA capabilities (dev_type=vdpa) should have
+        # been counted
+        self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci)
+
+        # create the port
+        vdpa_port = self.create_vdpa_port()
+
+        # ensure the binding details are currently unset
+        port = self.neutron.show_port(vdpa_port['id'])['port']
+        self.assertNotIn('binding:profile', port)
+
+        # create a server using the vDPA device via neutron
+        self._create_server(networks=[{'port': vdpa_port['id']}])
+
+        # ensure there is one less VF available and that the PF is no longer
+        # usable
+        self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2)
+
+        # ensure the binding details sent to "neutron" were correct
+        port = self.neutron.show_port(vdpa_port['id'])['port']
+        self.assertIn('binding:profile', port)
+        self.assertEqual(
+            {
+                'pci_vendor_info': '15b3:101e',
+                'pci_slot': '0000:06:00.4',
+                'physical_network': 'physnet4',
+            },
+            port['binding:profile'],
+        )
+
+    def _test_common(self, op, *args, **kwargs):
+        self.start_compute()
+
+        # create the port and a server, with the port attached to the server
+        vdpa_port = self.create_vdpa_port()
+        server = self._create_server(networks=[{'port': vdpa_port['id']}])
+
+        # attempt the unsupported action and ensure it fails
+        ex = self.assertRaises(
+            client.OpenStackApiException,
+            op, server, *args, **kwargs)
+        self.assertIn(
+            'not supported for instance with vDPA ports',
+            ex.response.text)
+
+    def test_attach_interface(self):
+        self.start_compute()
+
+        # create the port and a server, but don't attach the port to the server
+        # yet
+        vdpa_port = self.create_vdpa_port()
+        server = self._create_server(networks='none')
+
+        # attempt to attach the port to the server
+        ex = self.assertRaises(
+            client.OpenStackApiException,
+            self._attach_interface, server, vdpa_port['id'])
+        self.assertIn(
+            'not supported for instance with vDPA ports',
+            ex.response.text)
+
+    def test_detach_interface(self):
+        self._test_common(self._detach_interface, uuids.vdpa_port)
+
+    def test_shelve(self):
+        self._test_common(self._shelve_server)
+
+    def test_suspend(self):
+        self._test_common(self._suspend_server)
+
+    def test_evacute(self):
+        self._test_common(self._evacuate_server)
+
+    def test_resize(self):
+        flavor_id = self._create_flavor()
+        self._test_common(self._resize_server, flavor_id)
+
+    def test_cold_migrate(self):
+        self._test_common(self._migrate_server)
+
+
 class PCIServersTest(_PCIServersTestBase):
 
     ADMIN_API = True