Fix migration with remote-managed ports & add FT

`binding:profile` updates are handled differently for migration from
instance creation which was not taken into account previously. Relevant
fields (card_serial_number, pf_mac_address, vf_num) are now added to the
`binding:profile` after a new remote-managed PCI device is determined at
the destination node.

Likewise, there is special handling for the unshelve operation which is
fixed too.

Func testing:

* Allow the generated device XML to contain the PCI VPD capability;
* Add test cases for basic operations on instances with remote-managed
  ports (tunnel or physical);
* Add a live migration test case similar to how it is done for
  non-remote-managed SR-IOV ports but taking remote-managed port related
  specifics into account;
* Add evacuate, shelve/unshelve, cold migration test cases.

Change-Id: I9a1532e9a98f89db69b9ae3b41b06318a43519b3

Author: Dmitrii Shcherbakov

nova/compute/manager.py

@@ -84,6 +84,7 @@
 from nova.objects import instance as obj_instance
 from nova.objects import migrate_data as migrate_data_obj
 from nova.pci import request as pci_req_module
+from nova.pci import utils as pci_utils
 from nova.pci import whitelist
 from nova import safe_utils
 from nova.scheduler.client import query
@@ -10885,6 +10886,17 @@ def _update_migrate_vifs_profile_with_pci(self,
             profile['pci_slot'] = pci_dev.address
             profile['pci_vendor_info'] = ':'.join([pci_dev.vendor_id,
                                                    pci_dev.product_id])
+            if profile.get('card_serial_number'):
+                # Assume it is there since Nova makes sure that PCI devices
+                # tagged as remote-managed have a serial in PCI VPD.
+                profile['card_serial_number'] = pci_dev.card_serial_number
+            if profile.get('pf_mac_address'):
+                profile['pf_mac_address'] = pci_utils.get_mac_by_pci_address(
+                    pci_dev.parent_addr)
+            if profile.get('vf_num'):
+                profile['vf_num'] = pci_utils.get_vf_num_by_pci_address(
+                    pci_dev.address)
+
             mig_vif.profile = profile
             LOG.debug("Updating migrate VIF profile for port %(port_id)s:"
                       "%(profile)s", {'port_id': port_id,

nova/compute/resource_tracker.py

@@ -328,7 +328,12 @@ def _move_claim(
             migration_id=migration.id,
             old_numa_topology=instance.numa_topology,
             new_numa_topology=claim.claimed_numa_topology,
-            old_pci_devices=instance.pci_devices,
+            # NOTE(gibi): the _update_usage_from_migration call below appends
+            # the newly claimed pci devices to the instance.pci_devices list
+            # to keep the migration context independent we need to make a copy
+            # that list here. We need a deep copy as we need to duplicate
+            # the instance.pci_devices.objects list
+            old_pci_devices=copy.deepcopy(instance.pci_devices),
             new_pci_devices=claimed_pci_devices,
             old_pci_requests=instance.pci_requests,
             new_pci_requests=new_pci_requests,

nova/network/neutron.py

@@ -3663,15 +3663,15 @@ def _get_pci_mapping_for_migration(self, instance, migration):
                   migration.get('status') == 'reverted')
         return instance.migration_context.get_pci_mapping_for_migration(revert)
 
-    def _get_port_pci_slot(self, context, instance, port):
-        """Find the PCI address of the device corresponding to the port.
+    def _get_port_pci_dev(self, context, instance, port):
+        """Find the PCI device corresponding to the port.
         Assumes the port is an SRIOV one.
 
         :param context: The request context.
         :param instance: The instance to which the port is attached.
         :param port: The Neutron port, as obtained from the Neutron API
             JSON form.
-        :return: The PCI address as a string, or None if unable to find.
+        :return: The PciDevice object, or None if unable to find.
         """
         # Find the port's PCIRequest, or return None
         for r in instance.pci_requests.requests:
@@ -3691,8 +3691,26 @@ def _get_port_pci_slot(self, context, instance, port):
             LOG.debug('No PCI device found for request %s',
                       request.request_id, instance=instance)
             return None
-        # Return the device's PCI address
-        return device.address
+        return device
+
+    def _update_port_pci_binding_profile(self, pci_dev, binding_profile):
+        """Update the binding profile dict with new PCI device data.
+
+        :param pci_dev: The PciDevice object to update the profile with.
+        :param binding_profile: The dict to update.
+        """
+        binding_profile.update({'pci_slot': pci_dev.address})
+        if binding_profile.get('card_serial_number'):
+            binding_profile.update({
+                'card_serial_number': pci_dev.card_serial_number})
+        if binding_profile.get('pf_mac_address'):
+            binding_profile.update({
+                'pf_mac_address': pci_utils.get_mac_by_pci_address(
+                    pci_dev.parent_addr)})
+        if binding_profile.get('vf_num'):
+            binding_profile.update({
+                'vf_num': pci_utils.get_vf_num_by_pci_address(
+                    pci_dev.address)})
 
     def _update_port_binding_for_instance(
             self, context, instance, host, migration=None,
@@ -3760,9 +3778,10 @@ def _update_port_binding_for_instance(
                 # need to figure out the pci_slot from the InstancePCIRequest
                 # and PciDevice objects.
                 else:
-                    pci_slot = self._get_port_pci_slot(context, instance, p)
-                    if pci_slot:
-                        binding_profile.update({'pci_slot': pci_slot})
+                    pci_dev = self._get_port_pci_dev(context, instance, p)
+                    if pci_dev:
+                        self._update_port_pci_binding_profile(pci_dev,
+                                                              binding_profile)
                         updates[constants.BINDING_PROFILE] = binding_profile
 
             # NOTE(gibi): during live migration the conductor already sets the

nova/tests/fixtures/libvirt.py

@@ -275,6 +275,7 @@ class FakePCIDevice(object):
             <product id='0x%(prod_id)s'>%(prod_name)s</product>
             <vendor id='0x%(vend_id)s'>%(vend_name)s</vendor>
         %(capability)s
+        %(vpd_capability)s
             <iommuGroup number='%(iommu_group)d'>
               <address domain='0x0000' bus='%(bus)#02x' slot='%(slot)#02x' function='0x%(function)d'/>
             </iommuGroup>
@@ -293,13 +294,22 @@ class FakePCIDevice(object):
         <availableInstances>%(instances)s</availableInstances>
         </type>""".strip())  # noqa
 
+    vpd_cap_templ = textwrap.dedent("""
+        <capability type='vpd'>
+        <name>%(name)s</name>
+        %(fields)s
+        </capability>""".strip())
+    vpd_fields_templ = textwrap.dedent("""
+        <fields access='%(access)s'>%(section_fields)s</fields>""".strip())
+    vpd_field_templ = """<%(field_name)s>%(field_value)s</%(field_name)s>"""
+
     is_capable_of_mdevs = False
 
     def __init__(
         self, dev_type, bus, slot, function, iommu_group, numa_node, *,
         vf_ratio=None, multiple_gpu_types=False, generic_types=False,
         parent=None, vend_id=None, vend_name=None, prod_id=None,
-        prod_name=None, driver_name=None,
+        prod_name=None, driver_name=None, vpd_fields=None
     ):
         """Populate pci devices
 
@@ -340,6 +350,8 @@ def __init__(
         self.prod_name = prod_name
         self.driver_name = driver_name
 
+        self.vpd_fields = vpd_fields
+
         self.generate_xml()
 
     def generate_xml(self, skip_capability=False):
@@ -447,6 +459,7 @@ def generate_xml(self, skip_capability=False):
             'prod_name': prod_name,
             'driver': driver,
             'capability': capability,
+            'vpd_capability': self.format_vpd_cap(),
             'iommu_group': self.iommu_group,
             'numa_node': self.numa_node,
             'parent': parent,
@@ -457,6 +470,30 @@ def generate_xml(self, skip_capability=False):
         if self.numa_node == -1:
             self.pci_device = self.pci_device.replace("<numa node='-1'/>", "")
 
+    def format_vpd_cap(self):
+        if not self.vpd_fields:
+            return ''
+        fields = []
+        for access_type in ('readonly', 'readwrite'):
+            section_fields = []
+            for field_name, field_value in self.vpd_fields.get(
+                    access_type, {}).items():
+                section_fields.append(self.vpd_field_templ % {
+                    'field_name': field_name,
+                    'field_value': field_value,
+                })
+            if section_fields:
+                fields.append(
+                    self.vpd_fields_templ % {
+                        'access': access_type,
+                        'section_fields': '\n'.join(section_fields),
+                    }
+                )
+        return self.vpd_cap_templ % {
+            'name': self.vpd_fields.get('name', ''),
+            'fields': '\n'.join(fields)
+        }
+
     def XMLDesc(self, flags):
         return self.pci_device
 
@@ -572,7 +609,7 @@ def add_device(
         self, dev_type, bus, slot, function, iommu_group, numa_node,
         vf_ratio=None, multiple_gpu_types=False, generic_types=False,
         parent=None, vend_id=None, vend_name=None, prod_id=None,
-        prod_name=None, driver_name=None,
+        prod_name=None, driver_name=None, vpd_fields=None,
     ):
         pci_dev_name = _get_libvirt_nodedev_name(bus, slot, function)
 
@@ -593,7 +630,9 @@ def add_device(
             vend_name=vend_name,
             prod_id=prod_id,
             prod_name=prod_name,
-            driver_name=driver_name)
+            driver_name=driver_name,
+            vpd_fields=vpd_fields,
+        )
         self.devices[pci_dev_name] = dev
         return dev
 

nova/tests/functional/libvirt/base.py

@@ -316,6 +316,21 @@ class LibvirtNeutronFixture(nova_fixtures.NeutronFixture):
         'binding:vif_type': 'ovs',
         'binding:vnic_type': 'normal',
     }
+    network_3_port_2 = {
+        'id': '132c3875-b175-4b20-8a57-7a76219a13ae',
+        'network_id': network_3['id'],
+        'status': 'ACTIVE',
+        'mac_address': 'd2:0b:fd:99:89:8b',
+        'fixed_ips': [
+            {
+                'ip_address': '192.168.2.7',
+                'subnet_id': subnet_3['id']
+            }
+        ],
+        'binding:vif_details': {},
+        'binding:vif_type': 'ovs',
+        'binding:vnic_type': 'remote-managed',
+    }
     network_4_port_1 = {
         'id': 'b4cd0b93-2ac8-40a7-9fa4-2cd680ccdf3e',
         'network_id': network_4['id'],
@@ -361,6 +376,21 @@ class LibvirtNeutronFixture(nova_fixtures.NeutronFixture):
         'binding:vif_type': 'hw_veb',
         'binding:vnic_type': 'direct',
     }
+    network_4_port_4 = {
+        'id': 'a31e381d-41ec-41e4-b5a5-ec4ef705fafa',
+        'network_id': network_1['id'],
+        'status': 'ACTIVE',
+        'mac_address': '71:ce:c7:2b:cd:dd',
+        'fixed_ips': [
+            {
+                'ip_address': '192.168.4.9',
+                'subnet_id': subnet_4['id']
+            }
+        ],
+        'binding:vif_details': {},
+        'binding:vif_type': 'ovs',
+        'binding:vnic_type': 'remote-managed',
+    }
 
     def __init__(self, test):
         super(LibvirtNeutronFixture, self).__init__(test)