diff --git a/tests/common.nix b/tests/common.nix index 7346554..d622ef8 100644 --- a/tests/common.nix +++ b/tests/common.nix @@ -25,6 +25,10 @@ let hugepages ? false, prefault ? false, serial ? "pty", + # Whether all device will be assigned a static BDF through the XML or only some + all_static_bdf ? false, + # Whether we add a function ID to specific BDFs or not + use_bdf_function ? false, }: '' @@ -115,15 +119,43 @@ let destroy cloud-hypervisor + ${ + # Add the implicitly created RNG device explicitly + if all_static_bdf then + '' + + /dev/urandom + +
+ + '' + else + "" + } + ${ + # Assign a fixed BDF that would normally be acquired by the implicit RNG device + if all_static_bdf then + if use_bdf_function then + '' +
+ '' + else + '' +
+ '' + else + "" + } +
${ if serial == "pty" then @@ -157,14 +189,26 @@ let ''; # Please keep in sync with documentation in networks.md! - new_interface = '' - - - - - - - ''; + new_interface = + { + explicit_bdf ? false, + }: + '' + + + + + + ${ + if explicit_bdf then + '' +
+ '' + else + "" + } + + ''; # Please keep in sync with documentation in networks.md! new_interface_type_network = '' @@ -526,9 +570,31 @@ in })}"; }; }; + "/etc/domain-chv-static-bdf.xml" = { + "C+" = { + argument = "${pkgs.writeText "domain-chv-static-bdf.xml" (virsh_ch_xml { + all_static_bdf = true; + })}"; + }; + }; + "/etc/domain-chv-static-bdf-with-function.xml" = { + "C+" = { + argument = "${pkgs.writeText "domain-chv-static-bdf-with-function.xml" (virsh_ch_xml { + all_static_bdf = true; + use_bdf_function = true; + })}"; + }; + }; "/etc/new_interface.xml" = { "C+" = { - argument = "${pkgs.writeText "new_interface.xml" new_interface}"; + argument = "${pkgs.writeText "new_interface.xml" (new_interface { })}"; + }; + }; + "/etc/new_interface_explicit_bdf.xml" = { + "C+" = { + argument = "${pkgs.writeText "new_interface_explicit_bdf.xml" (new_interface { + explicit_bdf = true; + })}"; }; }; "/etc/new_interface_type_network.xml" = { diff --git a/tests/testscript.py b/tests/testscript.py index 3ec6089..e62e4ad 100644 --- a/tests/testscript.py +++ b/tests/testscript.py @@ -14,6 +14,11 @@ from nixos_test_stubs import start_all, computeVM, controllerVM # type: ignore +VIRTIO_NETWORK_DEVICE = "1af4:1041" +VIRTIO_BLOCK_DEVICE = "1af4:1042" +VIRTIO_ENTROPY_SOURCE = "1af4:1044" + + class SaveLogsOnErrorTestCase(unittest.TestCase): """ Custom TestCase class that saves interesting logs in error case. @@ -1600,10 +1605,297 @@ def check_certificates(machine): check_certificates(computeVM) + def test_bdf_implicit_assignment(self): + """ + Test if all BDFs are correctly assigned in a scenario where some + are fixed in the XML and some are assigned by libvirt. + + The domain XML we use here leaves a slot ID 0x03 free, but + allocates IDs 0x01, 0x02 and 0x04. 0x01 and 0x02 are dynamically + assigned by libvirt and not given in the domain XML. As 0x04 is + the first free ID, we expect this to be selected for the first + device we add to show that libvirt uses gaps. We add another + disk to show that all succeeding BDFs would be allocated + dynamically. Moreover, we show that all BDF assignments + survive live migration. + """ + controllerVM.succeed("virsh define /etc/domain-chv.xml") + controllerVM.succeed("virsh start testvm") + assert wait_for_ssh(controllerVM) + + controllerVM.succeed( + "virsh attach-device testvm /etc/new_interface_explicit_bdf.xml" + ) + # Add a disks that receive the first free BDFs + controllerVM.succeed( + "qemu-img create -f raw /var/lib/libvirt/storage-pools/nfs-share/vdb.img 5M" + ) + controllerVM.succeed( + "virsh attach-disk --domain testvm --target vdb --source /var/lib/libvirt/storage-pools/nfs-share/vdb.img" + ) + controllerVM.succeed( + "qemu-img create -f raw /var/lib/libvirt/storage-pools/nfs-share/vdc.img 5M" + ) + controllerVM.succeed( + "virsh attach-disk --domain testvm --target vdc --source /var/lib/libvirt/storage-pools/nfs-share/vdc.img" + ) + devices = pci_devices_by_bdf(controllerVM) + # Implicitly added fixed to 0x01 + assert devices["00:01.0"] == VIRTIO_ENTROPY_SOURCE + # Added by XML; dynamic BDF + assert devices["00:02.0"] == VIRTIO_NETWORK_DEVICE + # Add through XML + assert devices["00:03.0"] == VIRTIO_BLOCK_DEVICE + # Defined fixed BDF in XML; Hotplugged + assert devices["00:04.0"] == VIRTIO_NETWORK_DEVICE + # Hotplugged by this test (vdb) + assert devices["00:05.0"] == VIRTIO_BLOCK_DEVICE + # Hotplugged by this test (vdc) + assert devices["00:06.0"] == VIRTIO_BLOCK_DEVICE + + # Check that we can reuse the same non-statically allocated BDF + controllerVM.succeed("virsh detach-disk --domain testvm --target vdb") + assert pci_devices_by_bdf(controllerVM).get("00:05.0") is None + controllerVM.succeed( + "virsh attach-disk --domain testvm --target vdb --source /var/lib/libvirt/storage-pools/nfs-share/vdb.img" + ) + assert pci_devices_by_bdf(controllerVM).get("00:05.0") == VIRTIO_BLOCK_DEVICE + + # We free slot 4 and 5 ... + controllerVM.succeed( + "virsh detach-device testvm /etc/new_interface_explicit_bdf.xml" + ) + controllerVM.succeed("virsh detach-disk --domain testvm --target vdb") + assert pci_devices_by_bdf(controllerVM).get("00:04.0") is None + assert pci_devices_by_bdf(controllerVM).get("00:05.0") is None + # ...and expect the same disk that was formerly attached non-statically to slot 5 now to pop up in slot 4 + # through implicit BDF allocation. + controllerVM.succeed( + "virsh attach-disk --domain testvm --target vdb --source /var/lib/libvirt/storage-pools/nfs-share/vdb.img" + ) + assert pci_devices_by_bdf(controllerVM).get("00:04.0") == VIRTIO_BLOCK_DEVICE + + # Check that BDFs stay the same after migration + devices_before_livemig = pci_devices_by_bdf(controllerVM) + controllerVM.succeed( + "virsh migrate --domain testvm --desturi ch+tcp://computeVM/session --live --p2p" + ) + assert wait_for_ssh(computeVM) + devices_after_livemig = pci_devices_by_bdf(computeVM) + assert devices_before_livemig == devices_after_livemig + + def test_bdf_explicit_assignment(self): + """ + Test if all BDFs are correctly assigned when binding them + explicitly to BDFs. + + This test also shows that we can freely define the BDF that is + given to the RNG device. Moreover, we show that all BDF + assignments survive live migration, that allocating the same + BDF twice fails and that we can reuse BDFs if the respective + device was detached. + + Developer Note: This test resets the NixOS image. + """ + controllerVM.succeed("virsh define /etc/domain-chv-static-bdf.xml") + # Reset the image to purge any information about boot devices when terminating + with CommandGuard(reset_system_image, controllerVM) as _: + controllerVM.succeed("virsh start testvm") + assert wait_for_ssh(controllerVM) + controllerVM.succeed( + "virsh attach-device testvm /etc/new_interface_explicit_bdf.xml" + ) + controllerVM.succeed( + "virsh attach-disk --domain testvm --target vdb --source /var/lib/libvirt/storage-pools/nfs-share/cirros.img --address pci:0.0.17.0 " + ) + + devices = pci_devices_by_bdf(controllerVM) + assert devices["00:01.0"] == VIRTIO_BLOCK_DEVICE + assert devices["00:02.0"] == VIRTIO_NETWORK_DEVICE + assert devices.get("00:03.0") is None + assert devices["00:04.0"] == VIRTIO_NETWORK_DEVICE + assert devices["00:05.0"] == VIRTIO_ENTROPY_SOURCE + assert devices.get("00:06.0") is None + assert devices["00:17.0"] == VIRTIO_BLOCK_DEVICE + + # Check that BDF is freed and can be reallocated when de-/attaching a (entirely different) device + controllerVM.succeed( + "virsh detach-device testvm /etc/new_interface_explicit_bdf.xml" + ) + controllerVM.succeed("virsh detach-disk --domain testvm --target vdb") + assert pci_devices_by_bdf(controllerVM).get("00:04.0") is None + assert pci_devices_by_bdf(controllerVM).get("00:17.0") is None + controllerVM.succeed( + "virsh attach-disk --domain testvm --target vdb --source /var/lib/libvirt/storage-pools/nfs-share/cirros.img --address pci:0.0.04.0" + ) + devices_before_livemig = pci_devices_by_bdf(controllerVM) + assert devices_before_livemig.get("00:04.0") == VIRTIO_BLOCK_DEVICE + + # Adding to the same bdf twice fails + controllerVM.fail( + "virsh attach-device testvm /etc/new_interface_explicit_bdf.xml" + ) + + # Check that BDFs stay the same after migration + controllerVM.succeed( + "virsh migrate --domain testvm --desturi ch+tcp://computeVM/session --live --p2p" + ) + assert wait_for_ssh(computeVM) + devices_after_livemig = pci_devices_by_bdf(computeVM) + assert devices_before_livemig == devices_after_livemig + + def test_bdfs_implicitly_assigned_same_after_recreate(self): + """ + Test that BDFs stay consistent after a recreate when hotplugging + a transient and then a persistent device. + + The persistent config needs to adopt the assigned BDF correctly + to recreate the same device at the same address after recreate. + """ + # Using define + start creates a "persistent" domain rather than a transient + controllerVM.succeed("virsh define /etc/domain-chv.xml") + controllerVM.succeed("virsh start testvm") + + assert wait_for_ssh(controllerVM) + + # Add a persistent network device, i.e. the device should re-appear + # when the VM is destroyed and recreated. + controllerVM.succeed( + "qemu-img create -f raw /var/lib/libvirt/storage-pools/nfs-share/vdb.img 5M" + ) + # Attach to implicit BDF 0:04.0, transient + controllerVM.succeed( + "virsh attach-disk --domain testvm --target vdb --source /var/lib/libvirt/storage-pools/nfs-share/vdb.img" + ) + # Attach to implicit BDF 0:05.0, persistent + controllerVM.succeed( + "virsh attach-device testvm /etc/new_interface.xml --persistent " + ) + # The net device was attached persistently, so we expect the device to be there after a recreate, but not the + # disk. We indeed expect it to be not there anymore and leave a hole in the assigned BDFs + devices_before = pci_devices_by_bdf(controllerVM) + del devices_before["00:04.0"] + + # Transiently detach the devices. Net should re-appear when the VM is recreated. + controllerVM.succeed("virsh detach-device testvm /etc/new_interface.xml") + controllerVM.succeed("virsh detach-disk --domain testvm --target vdb") + + controllerVM.succeed("virsh destroy testvm") + + controllerVM.succeed("virsh start testvm") + assert wait_for_ssh(controllerVM) + + devices_after = pci_devices_by_bdf(controllerVM) + assert devices_after == devices_before + + def test_bdfs_dont_conflict_after_transient_unplug(self): + """ + Test that BDFs that are handed out persistently are not freed by + transient unplugs. + + The persistent config needs to adopt the assigned BDF correctly + and when unplugging a device, the transient config has to + respect BDFs that are already reserved in the persistent config. + In other words, we test that BDFs are correctly synced between + persistent and transient config whenever both are affected and + that weird hot/-unplugging doesn't make both configs go out of + sync. + """ + with CommandGuard(reset_system_image, controllerVM) as _: + # Using define + start creates a "persistent" domain rather than a transient + controllerVM.succeed("virsh define /etc/domain-chv.xml") + controllerVM.succeed("virsh start testvm") + assert wait_for_ssh(controllerVM) + + # Add a persistent disk. + controllerVM.succeed( + "qemu-img create -f raw /var/lib/libvirt/storage-pools/nfs-share/vdb.img 5M" + ) + controllerVM.succeed( + "virsh attach-disk --domain testvm --target vdb --source /var/lib/libvirt/storage-pools/nfs-share/vdb.img --persistent" + ) + # Remove transient. The device is removed from the transient config but not from the persistent one. The + # transient config has to mark the BDF as still in use nevertheless. + controllerVM.succeed("virsh detach-disk --domain testvm --target vdb") + # Attach another device persistently. If we did not respect in the transient config that the disk we + # detached before is still present in persistent config, then we now try to assign BDF 4 twice in the + # persistent config. In other words: Persistent and transient config's BDF management are out of sync if + # this command fails. + controllerVM.succeed( + "virsh attach-device testvm /etc/new_interface.xml --persistent " + ) + + def test_bdf_invalid_device_id(self): + """ + Test that a BDF with invalid device ID generates an error in libvirt. + + We test the case that a device id higher than 31 is used by a device. + """ + # Create a VM + controllerVM.succeed("virsh define /etc/domain-chv.xml") + controllerVM.succeed("virsh start testvm") + assert wait_for_ssh(controllerVM) + + # We need to check that no devices are added, so let's save how + # many devices are present in the VM after creating it. + num_before_expected_failure = number_of_devices(controllerVM) + # Add a persistent disk. + controllerVM.succeed( + "qemu-img create -f raw /var/lib/libvirt/storage-pools/nfs-share/vdb.img 5M" + ) + # Now we create a disk that we hotplug to a BDF with a device + # ID 32. This should fail. + controllerVM.fail( + "virsh attach-disk --domain testvm --target vdb --source /var/lib/libvirt/storage-pools/nfs-share/vdb.img --persistent --address pci:0.0.20.0" + ) + assert number_of_devices(controllerVM) == num_before_expected_failure + + def test_bdf_valid_device_id_with_function_id(self): + """ + Test that a BDFs containing a function ID leads to errors. + + CHV currently doesn't support multi function devices. So we need + to checks that libvirt does not allow to attach such devices. We + check that instantiating a domain with function ID doesn't work. + Then we test that we cannot hotplug a device with a function ID + in its BDF definition. + """ + # We don't support multi function devices currently. The config + # below defines a device with function ID, so instantiating it + # should fail. + controllerVM.fail("virsh define /etc/domain-chv-static-bdf-with-function.xml") + + # Now create a VM from a definition that does not contain any + # function IDs in it device definition. + controllerVM.succeed("virsh define /etc/domain-chv.xml") + controllerVM.succeed("virsh start testvm") + assert wait_for_ssh(controllerVM) + + # We need to check that no devices are added, so let's save how + # many devices are present in the VM after creating it. + num_before_expected_failure = number_of_devices(controllerVM) + # Now we create a disk that we hotplug to a BDF with a function + # ID. This should fail. + controllerVM.succeed( + "qemu-img create -f raw /var/lib/libvirt/storage-pools/nfs-share/vdb.img 5M" + ) + controllerVM.fail( + "virsh attach-disk --domain testvm --target vdb --source /var/lib/libvirt/storage-pools/nfs-share/vdb.img --persistent --address pci:0.0.1f.5" + ) + # Even though we only land here if the command above failed, we + # should still ensure that no new devices magically appeared. + assert number_of_devices(controllerVM) == num_before_expected_failure + def suite(): # Test cases in alphabetical order testcases = [ + LibvirtTests.test_bdf_explicit_assignment, + LibvirtTests.test_bdf_implicit_assignment, + LibvirtTests.test_bdf_invalid_device_id, + LibvirtTests.test_bdf_valid_device_id_with_function_id, + LibvirtTests.test_bdfs_dont_conflict_after_transient_unplug, + LibvirtTests.test_bdfs_implicitly_assigned_same_after_recreate, LibvirtTests.test_disk_is_locked, LibvirtTests.test_disk_resize_qcow2, LibvirtTests.test_disk_resize_raw, @@ -1748,6 +2040,29 @@ def reset_system_image(machine): ) +def pci_devices_by_bdf(machine): + """ + Creates a dict of all PCI devices addressable by their BDF in the VM. + + BDFs are keys, while the combination of vendor and device IDs form the + associated value. + + :param machine: Host machine of the nested VM + :return: BDF mapped to devices, example: {'00:00.0': '8086:0d57'} + :rtype: dict[str, str] + """ + status, lines = ssh( + machine, + "lspci -n | awk '/^[0-9a-f]{2}:[0-9a-f]{2}\\.[0-9]/{bdf=$1}{class=$3} {print bdf \",\" class}'", + ) + assert status == 0 + out = {} + for line in lines.splitlines(): + bdf, device_class = line.split(",") + out[bdf] = device_class + return out + + runner = unittest.TextTestRunner() if not runner.run(suite()).wasSuccessful(): raise Exception("Test Run unsuccessful")