refactor(vm): move Bus objects to Vm

We had previously added MMIO and Port IO buses inside ResourceAllocator
so that we could implement DeviceRelocation for the type. Now, we will
delegate device relocation responsibilities to ArchVm instead. That is
because device relocation requires access to the Vm file descriptor as
well.

As a result, we can move buses to the Vm object itself. Add MMIO bus to
VmCommon as both architectures use it. Add PortIO bus for x86
architecture only.

Not that we don't still support DeviceRelocation. VirtIO devices should
not request us to relocate them. Also, for adding such support we would
need to also support VirtIO reset. We will look into adding this
functionaliyt later on.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

src/vmm/src/arch/x86_64/vm.rs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::fmt;
+use std::sync::Arc;
 
 use kvm_bindings::{
     KVM_CLOCK_TSC_STABLE, KVM_IRQCHIP_IOAPIC, KVM_IRQCHIP_PIC_MASTER, KVM_IRQCHIP_PIC_SLAVE,
@@ -58,6 +59,8 @@ pub struct ArchVm {
     ///
     /// `None` if `KVM_CAP_XSAVE2` not supported.
     xsave2_size: Option<usize>,
+    /// Port IO bus
+    pub pio_bus: Arc<vm_device::Bus>,
 }
 
 impl ArchVm {
@@ -92,10 +95,13 @@ impl ArchVm {
             .set_tss_address(u64_to_usize(crate::arch::x86_64::layout::KVM_TSS_ADDRESS))
             .map_err(ArchVmError::SetTssAddress)?;
 
+        let pio_bus = Arc::new(vm_device::Bus::new());
+
         Ok(ArchVm {
             common,
             msrs_to_save,
             xsave2_size,
+            pio_bus,
         })
     }
 

src/vmm/src/device_manager/legacy.rs

@@ -119,7 +119,7 @@ impl PortIODeviceManager {
             input: None,
         }));
 
-        let io_bus = &vm.common.resource_allocator.pio_bus;
+        let io_bus = &vm.pio_bus;
         io_bus.insert(
             self.stdio_serial.clone(),
             Self::SERIAL_PORT_ADDRESSES[0],

src/vmm/src/device_manager/mmio.rs

@@ -201,7 +201,7 @@ impl MMIODeviceManager {
                 .map_err(MmioError::RegisterIrqFd)?;
         }
 
-        vm.common.resource_allocator.mmio_bus.insert(
+        vm.common.mmio_bus.insert(
             device.inner.clone(),
             device.resources.addr,
             device.resources.len,
@@ -295,7 +295,7 @@ impl MMIODeviceManager {
             inner: serial,
         };
 
-        vm.common.resource_allocator.mmio_bus.insert(
+        vm.common.mmio_bus.insert(
             device.inner.clone(),
             device.resources.addr,
             device.resources.len,
@@ -326,7 +326,7 @@ impl MMIODeviceManager {
     /// given as parameter, otherwise allocate a new MMIO resources for it.
     pub fn register_mmio_rtc(
         &mut self,
-        resource_allocator: &ResourceAllocator,
+        vm: &Vm,
         rtc: Arc<Mutex<RTCDevice>>,
         device_info_opt: Option<MMIODeviceInfo>,
     ) -> Result<(), MmioError> {
@@ -335,7 +335,7 @@ impl MMIODeviceManager {
         let device_info = if let Some(device_info) = device_info_opt {
             device_info
         } else {
-            let gsi = resource_allocator.allocate_gsi(1)?;
+            let gsi = vm.common.resource_allocator.allocate_gsi(1)?;
             MMIODeviceInfo {
                 addr: RTC_MEM_START,
                 len: MMIO_LEN,
@@ -348,7 +348,7 @@ impl MMIODeviceManager {
             inner: rtc,
         };
 
-        resource_allocator.mmio_bus.insert(
+        vm.common.mmio_bus.insert(
             device.inner.clone(),
             device.resources.addr,
             device.resources.len,

src/vmm/src/device_manager/mod.rs

@@ -217,7 +217,7 @@ impl DeviceManager {
         let boot_timer = Arc::new(Mutex::new(BootTimer::new(request_ts)));
 
         self.mmio_devices
-            .register_mmio_boot_timer(&vm.common.resource_allocator.mmio_bus, boot_timer)?;
+            .register_mmio_boot_timer(&vm.common.mmio_bus, boot_timer)?;
 
         Ok(())
     }
@@ -256,8 +256,7 @@ impl DeviceManager {
         }
 
         let rtc = Arc::new(Mutex::new(RTCDevice::new()));
-        self.mmio_devices
-            .register_mmio_rtc(&vm.common.resource_allocator, rtc, None)?;
+        self.mmio_devices.register_mmio_rtc(vm, rtc, None)?;
         Ok(())
     }
 

src/vmm/src/device_manager/pci_mngr.rs

@@ -33,7 +33,6 @@ use crate::devices::virtio::{TYPE_BALLOON, TYPE_BLOCK, TYPE_NET, TYPE_RNG};
 use crate::resources::VmResources;
 use crate::snapshot::Persist;
 use crate::vstate::memory::GuestMemoryMmap;
-use crate::vstate::resources::ResourceAllocator;
 use crate::vstate::vm::{InterruptError, MsiVectorGroup};
 use crate::{EventManager, Vm};
 
@@ -75,14 +74,14 @@ impl PciDevices {
 
         // Currently we don't assign any IRQs to PCI devices. We will be using MSI-X interrupts
         // only.
-        let pci_segment = PciSegment::new(0, &vm.common.resource_allocator, &[0u8; 32])?;
+        let pci_segment = PciSegment::new(0, vm, &[0u8; 32])?;
         self.pci_segment = Some(pci_segment);
 
         Ok(())
     }
 
     fn register_bars_with_bus(
-        resource_allocator: &ResourceAllocator,
+        vm: &Vm,
         virtio_device: &Arc<Mutex<VirtioPciDevice>>,
     ) -> Result<(), PciManagerError> {
         for bar in &virtio_device.lock().expect("Poisoned lock").bar_regions {
@@ -94,11 +93,8 @@ impl PciDevices {
                         bar.size()
                     );
                     #[cfg(target_arch = "x86_64")]
-                    resource_allocator.pio_bus.insert(
-                        virtio_device.clone(),
-                        bar.addr(),
-                        bar.size(),
-                    )?;
+                    vm.pio_bus
+                        .insert(virtio_device.clone(), bar.addr(), bar.size())?;
                     #[cfg(target_arch = "aarch64")]
                     log::error!("pci: We do not support I/O region allocation")
                 }
@@ -108,11 +104,9 @@ impl PciDevices {
                         bar.addr(),
                         bar.size()
                     );
-                    resource_allocator.mmio_bus.insert(
-                        virtio_device.clone(),
-                        bar.addr(),
-                        bar.size(),
-                    )?;
+                    vm.common
+                        .mmio_bus
+                        .insert(virtio_device.clone(), bar.addr(), bar.size())?;
                 }
             }
         }
@@ -168,7 +162,7 @@ impl PciDevices {
         self.virtio_devices
             .insert((device_type, id.clone()), virtio_device.clone());
 
-        Self::register_bars_with_bus(resource_allocator, &virtio_device)?;
+        Self::register_bars_with_bus(vm, &virtio_device)?;
         virtio_device
             .lock()
             .expect("Poisoned lock")
@@ -213,7 +207,7 @@ impl PciDevices {
         self.virtio_devices
             .insert((device_type, device_id.to_string()), virtio_device.clone());
 
-        Self::register_bars_with_bus(&vm.common.resource_allocator, &virtio_device)?;
+        Self::register_bars_with_bus(vm, &virtio_device)?;
         virtio_device
             .lock()
             .expect("Poisoned lock")

src/vmm/src/device_manager/persist.rs

@@ -389,7 +389,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                 if state.type_ == DeviceType::Rtc {
                     let rtc = Arc::new(Mutex::new(RTCDevice::new()));
                     dev_manager.register_mmio_rtc(
-                        &constructor_args.vm.common.resource_allocator,
+                        constructor_args.vm,
                         rtc,
                         Some(state.device_info),
                     )?;

src/vmm/src/devices/pci/pci_segment.rs

@@ -21,7 +21,7 @@ use uuid::Uuid;
 use vm_allocator::AddressAllocator;
 use vm_device::{BusDeviceSync, BusError};
 
-use crate::arch::{PCI_MMCONFIG_START, PCI_MMIO_CONFIG_SIZE_PER_SEGMENT};
+use crate::arch::{ArchVm as Vm, PCI_MMCONFIG_START, PCI_MMIO_CONFIG_SIZE_PER_SEGMENT};
 use crate::vstate::resources::ResourceAllocator;
 
 pub struct PciSegment {
@@ -67,28 +67,21 @@ impl std::fmt::Debug for PciSegment {
 }
 
 impl PciSegment {
-    fn build(
-        id: u16,
-        resource_allocator: &Arc<ResourceAllocator>,
-        pci_irq_slots: &[u8; 32],
-    ) -> Result<PciSegment, BusError> {
+    fn build(id: u16, vm: &Arc<Vm>, pci_irq_slots: &[u8; 32]) -> Result<PciSegment, BusError> {
         let pci_root = PciRoot::new(None);
-        let pci_bus = Arc::new(Mutex::new(PciBus::new(
-            pci_root,
-            resource_allocator.clone(),
-        )));
+        let pci_bus = Arc::new(Mutex::new(PciBus::new(pci_root, vm.clone())));
 
         let pci_config_mmio = Arc::new(Mutex::new(PciConfigMmio::new(Arc::clone(&pci_bus))));
         let mmio_config_address = PCI_MMCONFIG_START + PCI_MMIO_CONFIG_SIZE_PER_SEGMENT * id as u64;
 
-        resource_allocator.mmio_bus.insert(
+        vm.common.mmio_bus.insert(
             Arc::clone(&pci_config_mmio) as Arc<dyn BusDeviceSync>,
             mmio_config_address,
             PCI_MMIO_CONFIG_SIZE_PER_SEGMENT,
         )?;
 
-        let mem32_allocator = resource_allocator.mmio32_memory.clone();
-        let mem64_allocator = resource_allocator.mmio64_memory.clone();
+        let mem32_allocator = vm.common.resource_allocator.mmio32_memory.clone();
+        let mem64_allocator = vm.common.resource_allocator.mmio64_memory.clone();
 
         let start_of_mem32_area = mem32_allocator.lock().unwrap().base();
         let end_of_mem32_area = mem32_allocator.lock().unwrap().end();
@@ -119,13 +112,15 @@ impl PciSegment {
     #[cfg(target_arch = "x86_64")]
     pub(crate) fn new(
         id: u16,
-        resource_allocator: &Arc<ResourceAllocator>,
+        vm: &Arc<Vm>,
         pci_irq_slots: &[u8; 32],
     ) -> Result<PciSegment, BusError> {
-        let mut segment = Self::build(id, resource_allocator, pci_irq_slots)?;
+        use crate::Vm;
+
+        let mut segment = Self::build(id, vm, pci_irq_slots)?;
         let pci_config_io = Arc::new(Mutex::new(PciConfigIo::new(Arc::clone(&segment.pci_bus))));
 
-        resource_allocator.pio_bus.insert(
+        vm.pio_bus.insert(
             pci_config_io.clone(),
             PCI_CONFIG_IO_PORT,
             PCI_CONFIG_IO_PORT_SIZE,
@@ -151,10 +146,10 @@ impl PciSegment {
     #[cfg(target_arch = "aarch64")]
     pub(crate) fn new(
         id: u16,
-        resource_allocator: &Arc<ResourceAllocator>,
+        vm: &Arc<Vm>,
         pci_irq_slots: &[u8; 32],
     ) -> Result<PciSegment, BusError> {
-        let segment = Self::build(id, resource_allocator, pci_irq_slots)?;
+        let segment = Self::build(id, vm, pci_irq_slots)?;
         info!(
             "pci: adding PCI segment: id={:#x}, PCI MMIO config address: {:#x}, mem32 area: \
              [{:#x}-{:#x}], mem64 area: [{:#x}-{:#x}]",
@@ -468,13 +463,14 @@ mod tests {
 
     use super::*;
     use crate::arch;
+    use crate::builder::tests::default_vmm;
     use crate::utils::u64_to_usize;
 
     #[test]
     fn test_pci_segment_build() {
-        let resource_allocator = Arc::new(ResourceAllocator::new().unwrap());
+        let vmm = default_vmm();
         let pci_irq_slots = &[0u8; 32];
-        let pci_segment = PciSegment::new(0, &resource_allocator, pci_irq_slots).unwrap();
+        let pci_segment = PciSegment::new(0, &vmm.vm, pci_irq_slots).unwrap();
 
         assert_eq!(pci_segment.id, 0);
         assert_eq!(
@@ -503,35 +499,34 @@ mod tests {
     #[cfg(target_arch = "x86_64")]
     #[test]
     fn test_io_bus() {
-        let resource_allocator = Arc::new(ResourceAllocator::new().unwrap());
+        let vmm = default_vmm();
         let pci_irq_slots = &[0u8; 32];
-        let pci_segment = PciSegment::new(0, &resource_allocator, pci_irq_slots).unwrap();
+        let pci_segment = PciSegment::new(0, &vmm.vm, pci_irq_slots).unwrap();
 
         let mut data = [0u8; u64_to_usize(PCI_CONFIG_IO_PORT_SIZE)];
-        resource_allocator
-            .pio_bus
-            .read(PCI_CONFIG_IO_PORT, &mut data)
-            .unwrap();
+        vmm.vm.pio_bus.read(PCI_CONFIG_IO_PORT, &mut data).unwrap();
 
-        resource_allocator
+        vmm.vm
             .pio_bus
             .read(PCI_CONFIG_IO_PORT + PCI_CONFIG_IO_PORT_SIZE, &mut data)
             .unwrap_err();
     }
 
     #[test]
     fn test_mmio_bus() {
-        let resource_allocator = Arc::new(ResourceAllocator::new().unwrap());
+        let vmm = default_vmm();
         let pci_irq_slots = &[0u8; 32];
-        let pci_segment = PciSegment::new(0, &resource_allocator, pci_irq_slots).unwrap();
+        let pci_segment = PciSegment::new(0, &vmm.vm, pci_irq_slots).unwrap();
 
         let mut data = [0u8; u64_to_usize(PCI_MMIO_CONFIG_SIZE_PER_SEGMENT)];
 
-        resource_allocator
+        vmm.vm
+            .common
             .mmio_bus
             .read(pci_segment.mmio_config_address, &mut data)
             .unwrap();
-        resource_allocator
+        vmm.vm
+            .common
             .mmio_bus
             .read(
                 pci_segment.mmio_config_address + PCI_MMIO_CONFIG_SIZE_PER_SEGMENT,
@@ -542,9 +537,9 @@ mod tests {
 
     #[test]
     fn test_next_device_bdf() {
-        let resource_allocator = Arc::new(ResourceAllocator::new().unwrap());
+        let vmm = default_vmm();
         let pci_irq_slots = &[0u8; 32];
-        let pci_segment = PciSegment::new(0, &resource_allocator, pci_irq_slots).unwrap();
+        let pci_segment = PciSegment::new(0, &vmm.vm, pci_irq_slots).unwrap();
 
         // Start checking from device id 1, since 0 is allocated to the Root port.
         for dev_id in 1..32 {

src/vmm/src/lib.rs

@@ -373,10 +373,9 @@ impl Vmm {
         self.vcpus_handles.reserve(vcpu_count);
 
         for mut vcpu in vcpus.drain(..) {
-            vcpu.set_mmio_bus(self.vm.common.resource_allocator.mmio_bus.clone());
+            vcpu.set_mmio_bus(self.vm.common.mmio_bus.clone());
             #[cfg(target_arch = "x86_64")]
-            vcpu.kvm_vcpu
-                .set_pio_bus(self.vm.common.resource_allocator.pio_bus.clone());
+            vcpu.kvm_vcpu.set_pio_bus(self.vm.pio_bus.clone());
 
             self.vcpus_handles
                 .push(vcpu.start_threaded(vcpu_seccomp_filter.clone(), barrier.clone())?);