refactor: prepare ResourceAllocator for PCIe devices

PCIe devices need some times to relocate themselves in memory. To do so,
they need to keep an (atomic) reference to a type that implements
`DeviceRelocation` trait. The logic for relocation involves removing the
device from the bus it has been registered to, allocate a new address
range for it and reinsert it.

Instead of creating a new type for it, reuse `ResourceAllocator`. This
means that we need to move the buses from the `DeviceManager` inside
`ResourceAllocator`.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

Cargo.lock

@@ -44,7 +44,7 @@ userfaultfd = "0.8.1"
 utils = { path = "../utils" }
 uuid = "1.16.0"
 vhost = { version = "0.13.0", features = ["vhost-user-frontend"] }
-vm-allocator = "0.1.0"
+vm-allocator = { path = "../../../rust-vmm/vm-allocator" }
 vm-device = { path = "../vm-device" }
 vm-memory = { version = "0.16.1", features = [
   "backend-mmap",

src/vmm/Cargo.toml

@@ -53,7 +53,7 @@ impl AcpiTableWriter<'_> {
     /// buffer. It returns the address in which it wrote the table.
     fn write_acpi_table<S>(
         &mut self,
-        resource_allocator: &mut ResourceAllocator,
+        resource_allocator: &ResourceAllocator,
         table: &mut S,
     ) -> Result<u64, AcpiError>
     where
@@ -94,15 +94,15 @@ impl AcpiTableWriter<'_> {
         setup_arch_dsdt(&mut dsdt_data)?;
 
         let mut dsdt = Dsdt::new(OEM_ID, *b"FCVMDSDT", OEM_REVISION, dsdt_data);
-        self.write_acpi_table(&mut device_manager.resource_allocator, &mut dsdt)
+        self.write_acpi_table(&device_manager.resource_allocator, &mut dsdt)
     }
 
     /// Build the FADT table for the guest
     ///
     /// This includes a pointer with the location of the DSDT in guest memory
     fn build_fadt(
         &mut self,
-        resource_allocator: &mut ResourceAllocator,
+        resource_allocator: &ResourceAllocator,
         dsdt_addr: u64,
     ) -> Result<u64, AcpiError> {
         let mut fadt = Fadt::new(OEM_ID, *b"FCVMFADT", OEM_REVISION);
@@ -120,7 +120,7 @@ impl AcpiTableWriter<'_> {
     /// This includes information about the interrupt controllers supported in the platform
     fn build_madt(
         &mut self,
-        resource_allocator: &mut ResourceAllocator,
+        resource_allocator: &ResourceAllocator,
         nr_vcpus: u8,
     ) -> Result<u64, AcpiError> {
         let mut madt = Madt::new(
@@ -138,7 +138,7 @@ impl AcpiTableWriter<'_> {
     /// Currently, we pass to the guest just FADT and MADT tables.
     fn build_xsdt(
         &mut self,
-        resource_allocator: &mut ResourceAllocator,
+        resource_allocator: &ResourceAllocator,
         fadt_addr: u64,
         madt_addr: u64,
     ) -> Result<u64, AcpiError> {
@@ -180,15 +180,14 @@ pub(crate) fn create_acpi_tables(
     vcpus: &[Vcpu],
 ) -> Result<(), AcpiError> {
     let mut writer = AcpiTableWriter { mem };
-
     let dsdt_addr = writer.build_dsdt(device_manager)?;
-    let fadt_addr = writer.build_fadt(&mut device_manager.resource_allocator, dsdt_addr)?;
+
+    let fadt_addr = writer.build_fadt(&device_manager.resource_allocator, dsdt_addr)?;
     let madt_addr = writer.build_madt(
-        &mut device_manager.resource_allocator,
+        &device_manager.resource_allocator,
         vcpus.len().try_into().unwrap(),
     )?;
-    let xsdt_addr =
-        writer.build_xsdt(&mut device_manager.resource_allocator, fadt_addr, madt_addr)?;
+    let xsdt_addr = writer.build_xsdt(&device_manager.resource_allocator, fadt_addr, madt_addr)?;
     writer.build_rsdp(xsdt_addr)
 }
 
@@ -227,22 +226,22 @@ mod tests {
     #[test]
     fn test_write_acpi_table_memory_allocation() {
         // A mocke Vmm object with 128MBs of memory
-        let mut vmm = default_vmm();
+        let vmm = default_vmm();
         let mut writer = AcpiTableWriter {
             mem: vmm.vm.guest_memory(),
         };
 
         // This should succeed
         let mut sdt = MockSdt(vec![0; 4096]);
         let addr = writer
-            .write_acpi_table(&mut vmm.device_manager.resource_allocator, &mut sdt)
+            .write_acpi_table(&vmm.device_manager.resource_allocator, &mut sdt)
             .unwrap();
         assert_eq!(addr, SYSTEM_MEM_START);
 
         // Let's try to write two 4K pages plus one byte
         let mut sdt = MockSdt(vec![0; usize::try_from(SYSTEM_MEM_SIZE + 1).unwrap()]);
         let err = writer
-            .write_acpi_table(&mut vmm.device_manager.resource_allocator, &mut sdt)
+            .write_acpi_table(&vmm.device_manager.resource_allocator, &mut sdt)
             .unwrap_err();
         assert!(
             matches!(
@@ -257,27 +256,27 @@ mod tests {
         // succeed.
         let mut sdt = MockSdt(vec![0; 5]);
         let addr = writer
-            .write_acpi_table(&mut vmm.device_manager.resource_allocator, &mut sdt)
+            .write_acpi_table(&vmm.device_manager.resource_allocator, &mut sdt)
             .unwrap();
         assert_eq!(addr, SYSTEM_MEM_START + 4096);
         let mut sdt = MockSdt(vec![0; 2]);
         let addr = writer
-            .write_acpi_table(&mut vmm.device_manager.resource_allocator, &mut sdt)
+            .write_acpi_table(&vmm.device_manager.resource_allocator, &mut sdt)
             .unwrap();
         assert_eq!(addr, SYSTEM_MEM_START + 4101);
         let mut sdt = MockSdt(vec![0; 4]);
         let addr = writer
-            .write_acpi_table(&mut vmm.device_manager.resource_allocator, &mut sdt)
+            .write_acpi_table(&vmm.device_manager.resource_allocator, &mut sdt)
             .unwrap();
         assert_eq!(addr, SYSTEM_MEM_START + 4103);
         let mut sdt = MockSdt(vec![0; 8]);
         let addr = writer
-            .write_acpi_table(&mut vmm.device_manager.resource_allocator, &mut sdt)
+            .write_acpi_table(&vmm.device_manager.resource_allocator, &mut sdt)
             .unwrap();
         assert_eq!(addr, SYSTEM_MEM_START + 4107);
         let mut sdt = MockSdt(vec![0; 16]);
         let addr = writer
-            .write_acpi_table(&mut vmm.device_manager.resource_allocator, &mut sdt)
+            .write_acpi_table(&vmm.device_manager.resource_allocator, &mut sdt)
             .unwrap();
         assert_eq!(addr, SYSTEM_MEM_START + 4115);
     }
@@ -294,11 +293,11 @@ mod tests {
         let mut writer = AcpiTableWriter {
             mem: vm.guest_memory(),
         };
-        let mut resource_allocator = ResourceAllocator::new().unwrap();
+        let resource_allocator = ResourceAllocator::new().unwrap();
 
         let mut sdt = MockSdt(vec![0; usize::try_from(SYSTEM_MEM_SIZE).unwrap()]);
         let err = writer
-            .write_acpi_table(&mut resource_allocator, &mut sdt)
+            .write_acpi_table(&resource_allocator, &mut sdt)
             .unwrap_err();
         assert!(
             matches!(

src/vmm/src/acpi/mod.rs

@@ -477,7 +477,7 @@ mod tests {
             .register_virtio_test_device(
                 &vm,
                 mem.clone(),
-                &mut device_manager.resource_allocator,
+                &device_manager.resource_allocator,
                 dummy,
                 &mut cmdline,
                 "dummy",

src/vmm/src/arch/aarch64/fdt.rs

@@ -203,7 +203,7 @@ pub fn configure_system_for_boot(
     // Note that this puts the mptable at the last 1k of Linux's 640k base RAM
     mptable::setup_mptable(
         vmm.vm.guest_memory(),
-        &mut vmm.device_manager.resource_allocator,
+        &vmm.device_manager.resource_allocator,
         vcpu_config.vcpu_count,
     )
     .map_err(ConfigurationError::MpTableSetup)?;
@@ -582,8 +582,8 @@ mod tests {
     fn test_system_configuration() {
         let no_vcpus = 4;
         let gm = single_region_mem(0x10000);
-        let mut resource_allocator = ResourceAllocator::new().unwrap();
-        let err = mptable::setup_mptable(&gm, &mut resource_allocator, 1);
+        let resource_allocator = ResourceAllocator::new().unwrap();
+        let err = mptable::setup_mptable(&gm, &resource_allocator, 1);
         assert!(matches!(
             err.unwrap_err(),
             mptable::MptableError::NotEnoughMemory
@@ -592,24 +592,24 @@ mod tests {
         // Now assigning some memory that falls before the 32bit memory hole.
         let mem_size = mib_to_bytes(128);
         let gm = arch_mem(mem_size);
-        let mut resource_allocator = ResourceAllocator::new().unwrap();
-        mptable::setup_mptable(&gm, &mut resource_allocator, no_vcpus).unwrap();
+        let resource_allocator = ResourceAllocator::new().unwrap();
+        mptable::setup_mptable(&gm, &resource_allocator, no_vcpus).unwrap();
         configure_64bit_boot(&gm, GuestAddress(0), 0, &None).unwrap();
         configure_pvh(&gm, GuestAddress(0), &None).unwrap();
 
         // Now assigning some memory that is equal to the start of the 32bit memory hole.
         let mem_size = mib_to_bytes(3328);
         let gm = arch_mem(mem_size);
-        let mut resource_allocator = ResourceAllocator::new().unwrap();
-        mptable::setup_mptable(&gm, &mut resource_allocator, no_vcpus).unwrap();
+        let resource_allocator = ResourceAllocator::new().unwrap();
+        mptable::setup_mptable(&gm, &resource_allocator, no_vcpus).unwrap();
         configure_64bit_boot(&gm, GuestAddress(0), 0, &None).unwrap();
         configure_pvh(&gm, GuestAddress(0), &None).unwrap();
 
         // Now assigning some memory that falls after the 32bit memory hole.
         let mem_size = mib_to_bytes(3330);
         let gm = arch_mem(mem_size);
-        let mut resource_allocator = ResourceAllocator::new().unwrap();
-        mptable::setup_mptable(&gm, &mut resource_allocator, no_vcpus).unwrap();
+        let resource_allocator = ResourceAllocator::new().unwrap();
+        mptable::setup_mptable(&gm, &resource_allocator, no_vcpus).unwrap();
         configure_64bit_boot(&gm, GuestAddress(0), 0, &None).unwrap();
         configure_pvh(&gm, GuestAddress(0), &None).unwrap();
     }

src/vmm/src/arch/x86_64/mod.rs

@@ -116,7 +116,7 @@ fn compute_mp_size(num_cpus: u8) -> usize {
 /// Performs setup of the MP table for the given `num_cpus`.
 pub fn setup_mptable(
     mem: &GuestMemoryMmap,
-    resource_allocator: &mut ResourceAllocator,
+    resource_allocator: &ResourceAllocator,
     num_cpus: u8,
 ) -> Result<(), MptableError> {
     if num_cpus > MAX_SUPPORTED_CPUS {
@@ -334,27 +334,27 @@ mod tests {
     fn bounds_check() {
         let num_cpus = 4;
         let mem = single_region_mem_at(SYSTEM_MEM_START, compute_mp_size(num_cpus));
-        let mut resource_allocator = ResourceAllocator::new().unwrap();
+        let resource_allocator = ResourceAllocator::new().unwrap();
 
-        setup_mptable(&mem, &mut resource_allocator, num_cpus).unwrap();
+        setup_mptable(&mem, &resource_allocator, num_cpus).unwrap();
     }
 
     #[test]
     fn bounds_check_fails() {
         let num_cpus = 4;
         let mem = single_region_mem_at(SYSTEM_MEM_START, compute_mp_size(num_cpus) - 1);
-        let mut resource_allocator = ResourceAllocator::new().unwrap();
+        let resource_allocator = ResourceAllocator::new().unwrap();
 
-        setup_mptable(&mem, &mut resource_allocator, num_cpus).unwrap_err();
+        setup_mptable(&mem, &resource_allocator, num_cpus).unwrap_err();
     }
 
     #[test]
     fn mpf_intel_checksum() {
         let num_cpus = 1;
         let mem = single_region_mem_at(SYSTEM_MEM_START, compute_mp_size(num_cpus));
-        let mut resource_allocator = ResourceAllocator::new().unwrap();
+        let resource_allocator = ResourceAllocator::new().unwrap();
 
-        setup_mptable(&mem, &mut resource_allocator, num_cpus).unwrap();
+        setup_mptable(&mem, &resource_allocator, num_cpus).unwrap();
 
         let mpf_intel: mpspec::mpf_intel = mem.read_obj(GuestAddress(SYSTEM_MEM_START)).unwrap();
 
@@ -365,9 +365,9 @@ mod tests {
     fn mpc_table_checksum() {
         let num_cpus = 4;
         let mem = single_region_mem_at(SYSTEM_MEM_START, compute_mp_size(num_cpus));
-        let mut resource_allocator = ResourceAllocator::new().unwrap();
+        let resource_allocator = ResourceAllocator::new().unwrap();
 
-        setup_mptable(&mem, &mut resource_allocator, num_cpus).unwrap();
+        setup_mptable(&mem, &resource_allocator, num_cpus).unwrap();
 
         let mpf_intel: mpspec::mpf_intel = mem.read_obj(GuestAddress(SYSTEM_MEM_START)).unwrap();
         let mpc_offset = GuestAddress(u64::from(mpf_intel.physptr));
@@ -388,9 +388,9 @@ mod tests {
     fn mpc_entry_count() {
         let num_cpus = 1;
         let mem = single_region_mem_at(SYSTEM_MEM_START, compute_mp_size(num_cpus));
-        let mut resource_allocator = ResourceAllocator::new().unwrap();
+        let resource_allocator = ResourceAllocator::new().unwrap();
 
-        setup_mptable(&mem, &mut resource_allocator, num_cpus).unwrap();
+        setup_mptable(&mem, &resource_allocator, num_cpus).unwrap();
 
         let mpf_intel: mpspec::mpf_intel = mem.read_obj(GuestAddress(SYSTEM_MEM_START)).unwrap();
         let mpc_offset = GuestAddress(u64::from(mpf_intel.physptr));
@@ -419,8 +419,8 @@ mod tests {
         let mem = single_region_mem_at(SYSTEM_MEM_START, compute_mp_size(MAX_SUPPORTED_CPUS));
 
         for i in 0..MAX_SUPPORTED_CPUS {
-            let mut resource_allocator = ResourceAllocator::new().unwrap();
-            setup_mptable(&mem, &mut resource_allocator, i).unwrap();
+            let resource_allocator = ResourceAllocator::new().unwrap();
+            setup_mptable(&mem, &resource_allocator, i).unwrap();
 
             let mpf_intel: mpspec::mpf_intel =
                 mem.read_obj(GuestAddress(SYSTEM_MEM_START)).unwrap();
@@ -450,9 +450,9 @@ mod tests {
     fn cpu_entry_count_max() {
         let cpus = MAX_SUPPORTED_CPUS + 1;
         let mem = single_region_mem_at(SYSTEM_MEM_START, compute_mp_size(cpus));
-        let mut resource_allocator = ResourceAllocator::new().unwrap();
+        let resource_allocator = ResourceAllocator::new().unwrap();
 
-        let result = setup_mptable(&mem, &mut resource_allocator, cpus).unwrap_err();
+        let result = setup_mptable(&mem, &resource_allocator, cpus).unwrap_err();
         assert_eq!(result, MptableError::TooManyCpus);
     }
 }