vm: track device interrupts within Vm object

Add logic to track the device interrupts used by the microVM. This is
not strictly needed right now, but we will need it when adding support
for MSI-X interrupts. MSI-X interrupts are configured at runtime and we
need to interact with KVM to set the interruput routes. To do it, we
need to keep track all of the interrupts the VM is using.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

src/vmm/src/arch/aarch64/fdt.rs

@@ -499,17 +499,16 @@ mod tests {
     use std::ffi::CString;
     use std::sync::{Arc, Mutex};
 
-    use kvm_ioctls::Kvm;
     use linux_loader::cmdline as kernel_cmdline;
 
     use super::*;
-    use crate::EventManager;
     use crate::arch::aarch64::gic::create_gic;
     use crate::arch::aarch64::layout;
     use crate::device_manager::mmio::tests::DummyDevice;
     use crate::device_manager::tests::default_device_manager;
     use crate::test_utils::arch_mem;
     use crate::vstate::memory::GuestAddress;
+    use crate::{EventManager, Kvm, Vm};
 
     // The `load` function from the `device_tree` will mistakenly check the actual size
     // of the buffer with the allocated size. This works around that.
@@ -525,9 +524,9 @@ mod tests {
         let mem = arch_mem(layout::FDT_MAX_SIZE + 0x1000);
         let mut event_manager = EventManager::new().unwrap();
         let mut device_manager = default_device_manager();
-        let kvm = Kvm::new().unwrap();
-        let vm = kvm.create_vm().unwrap();
-        let gic = create_gic(&vm, 1, None).unwrap();
+        let kvm = Kvm::new(vec![]).unwrap();
+        let vm = Vm::new(&kvm).unwrap();
+        let gic = create_gic(vm.fd(), 1, None).unwrap();
         let mut cmdline = kernel_cmdline::Cmdline::new(4096).unwrap();
         cmdline.insert("console", "/dev/tty0").unwrap();
 
@@ -562,9 +561,9 @@ mod tests {
     fn test_create_fdt_with_vmgenid() {
         let mem = arch_mem(layout::FDT_MAX_SIZE + 0x1000);
         let mut device_manager = default_device_manager();
-        let kvm = Kvm::new().unwrap();
-        let vm = kvm.create_vm().unwrap();
-        let gic = create_gic(&vm, 1, None).unwrap();
+        let kvm = Kvm::new(vec![]).unwrap();
+        let vm = Vm::new(&kvm).unwrap();
+        let gic = create_gic(vm.fd(), 1, None).unwrap();
         let mut cmdline = kernel_cmdline::Cmdline::new(4096).unwrap();
         cmdline.insert("console", "/dev/tty0").unwrap();
 
@@ -585,9 +584,9 @@ mod tests {
     fn test_create_fdt() {
         let mem = arch_mem(layout::FDT_MAX_SIZE + 0x1000);
         let device_manager = default_device_manager();
-        let kvm = Kvm::new().unwrap();
-        let vm = kvm.create_vm().unwrap();
-        let gic = create_gic(&vm, 1, None).unwrap();
+        let kvm = Kvm::new(vec![]).unwrap();
+        let vm = Vm::new(&kvm).unwrap();
+        let gic = create_gic(vm.fd(), 1, None).unwrap();
 
         let saved_dtb_bytes = match gic.fdt_compatibility() {
             "arm,gic-v3" => include_bytes!("output_GICv3.dtb"),
@@ -642,9 +641,9 @@ mod tests {
     fn test_create_fdt_with_initrd() {
         let mem = arch_mem(layout::FDT_MAX_SIZE + 0x1000);
         let device_manager = default_device_manager();
-        let kvm = Kvm::new().unwrap();
-        let vm = kvm.create_vm().unwrap();
-        let gic = create_gic(&vm, 1, None).unwrap();
+        let kvm = Kvm::new(vec![]).unwrap();
+        let vm = Vm::new(&kvm).unwrap();
+        let gic = create_gic(vm.fd(), 1, None).unwrap();
 
         let saved_dtb_bytes = match gic.fdt_compatibility() {
             "arm,gic-v3" => include_bytes!("output_initrd_GICv3.dtb"),

src/vmm/src/builder.rs

@@ -165,23 +165,14 @@ pub fn build_microvm_for_boot(
         .cpu_template
         .get_cpu_template()?;
 
-    /*
-        let (mut vmm, mut vcpus) = create_vmm_and_vcpus(
-            instance_info,
-            event_manager,
-            vm_resources.machine_config.vcpu_count,
-            cpu_template.kvm_capabilities.clone(),
-        )?;
-    */
-
     let kvm = Kvm::new(cpu_template.kvm_capabilities.clone())?;
     // Set up Kvm Vm and register memory regions.
     // Build custom CPU config if a custom template is provided.
     let mut vm = Vm::new(&kvm)?;
     let (mut vcpus, vcpus_exit_evt) = vm.create_vcpus(vm_resources.machine_config.vcpu_count)?;
     vm.register_memory_regions(guest_memory)?;
 
-    let mut device_manager = DeviceManager::new(event_manager, &vcpus_exit_evt, vm.fd())?;
+    let mut device_manager = DeviceManager::new(event_manager, &vcpus_exit_evt, &vm)?;
 
     let entry_point = load_kernel(&boot_config.kernel_file, vm.guest_memory())?;
     let initrd = InitrdConfig::from_config(boot_config, vm.guest_memory())?;
@@ -257,9 +248,9 @@ pub fn build_microvm_for_boot(
     }
 
     #[cfg(target_arch = "aarch64")]
-    device_manager.attach_legacy_devices_aarch64(vm.fd(), event_manager, &mut boot_cmdline)?;
+    device_manager.attach_legacy_devices_aarch64(&vm, event_manager, &mut boot_cmdline)?;
 
-    device_manager.attach_vmgenid_device(vm.guest_memory(), vm.fd())?;
+    device_manager.attach_vmgenid_device(vm.guest_memory(), &vm)?;
 
     #[cfg(target_arch = "aarch64")]
     if vcpus[0].kvm_vcpu.supports_pvtime() {
@@ -423,7 +414,7 @@ pub fn build_microvm_from_snapshot(
         .create_vcpus(vm_resources.machine_config.vcpu_count)
         .map_err(StartMicrovmError::Vm)?;
 
-    let mut device_manager = DeviceManager::new(event_manager, &vcpus_exit_evt, vm.fd()).unwrap();
+    let mut device_manager = DeviceManager::new(event_manager, &vcpus_exit_evt, &vm).unwrap();
 
     vm.register_memory_regions(guest_memory)
         .map_err(StartMicrovmError::Vm)?;
@@ -477,7 +468,7 @@ pub fn build_microvm_from_snapshot(
     // Restore devices states.
     let device_ctor_args = DeviceRestoreArgs {
         mem: vm.guest_memory(),
-        vm: vm.fd(),
+        vm: &vm,
         event_manager,
         vm_resources,
         instance_id: &instance_info.id,
@@ -580,14 +571,7 @@ fn attach_entropy_device(
         .to_string();
 
     event_manager.add_subscriber(entropy_device.clone());
-    device_manager.attach_virtio_device(
-        vm.guest_memory(),
-        vm.fd(),
-        id,
-        entropy_device.clone(),
-        cmdline,
-        false,
-    )
+    device_manager.attach_virtio_device(vm, id, entropy_device.clone(), cmdline, false)
 }
 
 fn attach_block_devices<'a, I: Iterator<Item = &'a Arc<Mutex<Block>>> + Debug>(
@@ -614,14 +598,7 @@ fn attach_block_devices<'a, I: Iterator<Item = &'a Arc<Mutex<Block>>> + Debug>(
         };
         // The device mutex mustn't be locked here otherwise it will deadlock.
         event_manager.add_subscriber(block.clone());
-        device_manager.attach_virtio_device(
-            vm.guest_memory(),
-            vm.fd(),
-            id,
-            block.clone(),
-            cmdline,
-            is_vhost_user,
-        )?;
+        device_manager.attach_virtio_device(vm, id, block.clone(), cmdline, is_vhost_user)?;
     }
     Ok(())
 }
@@ -637,14 +614,7 @@ fn attach_net_devices<'a, I: Iterator<Item = &'a Arc<Mutex<Net>>> + Debug>(
         let id = net_device.lock().expect("Poisoned lock").id().clone();
         event_manager.add_subscriber(net_device.clone());
         // The device mutex mustn't be locked here otherwise it will deadlock.
-        device_manager.attach_virtio_device(
-            vm.guest_memory(),
-            vm.fd(),
-            id,
-            net_device.clone(),
-            cmdline,
-            false,
-        )?;
+        device_manager.attach_virtio_device(vm, id, net_device.clone(), cmdline, false)?;
     }
     Ok(())
 }
@@ -659,14 +629,7 @@ fn attach_unixsock_vsock_device(
     let id = String::from(unix_vsock.lock().expect("Poisoned lock").id());
     event_manager.add_subscriber(unix_vsock.clone());
     // The device mutex mustn't be locked here otherwise it will deadlock.
-    device_manager.attach_virtio_device(
-        vm.guest_memory(),
-        vm.fd(),
-        id,
-        unix_vsock.clone(),
-        cmdline,
-        false,
-    )
+    device_manager.attach_virtio_device(vm, id, unix_vsock.clone(), cmdline, false)
 }
 
 fn attach_balloon_device(
@@ -679,14 +642,7 @@ fn attach_balloon_device(
     let id = String::from(balloon.lock().expect("Poisoned lock").id());
     event_manager.add_subscriber(balloon.clone());
     // The device mutex mustn't be locked here otherwise it will deadlock.
-    device_manager.attach_virtio_device(
-        vm.guest_memory(),
-        vm.fd(),
-        id,
-        balloon.clone(),
-        cmdline,
-        false,
-    )
+    device_manager.attach_virtio_device(vm, id, balloon.clone(), cmdline, false)
 }
 
 #[cfg(test)]
@@ -933,7 +889,7 @@ pub(crate) mod tests {
     #[cfg(target_arch = "x86_64")]
     pub(crate) fn insert_vmgenid_device(vmm: &mut Vmm) {
         vmm.device_manager
-            .attach_vmgenid_device(vmm.vm.guest_memory(), vmm.vm.fd())
+            .attach_vmgenid_device(vmm.vm.guest_memory(), &vmm.vm)
             .unwrap();
         assert!(vmm.device_manager.acpi_devices.vmgenid.is_some());
     }

src/vmm/src/device_manager/acpi.rs

@@ -2,8 +2,8 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use acpi_tables::{Aml, aml};
-use kvm_ioctls::VmFd;
 
+use crate::Vm;
 use crate::devices::acpi::vmgenid::VmGenId;
 
 #[derive(Debug, Default)]
@@ -21,12 +21,8 @@ impl ACPIDeviceManager {
     /// Attach a new VMGenID device to the microVM
     ///
     /// This will register the device's interrupt with KVM
-    pub fn attach_vmgenid(
-        &mut self,
-        vmgenid: VmGenId,
-        vm_fd: &VmFd,
-    ) -> Result<(), kvm_ioctls::Error> {
-        vm_fd.register_irqfd(&vmgenid.interrupt_evt, vmgenid.gsi)?;
+    pub fn attach_vmgenid(&mut self, vmgenid: VmGenId, vm: &Vm) -> Result<(), kvm_ioctls::Error> {
+        vm.register_irq(&vmgenid.interrupt_evt, vmgenid.gsi)?;
         self.vmgenid = Some(vmgenid);
         Ok(())
     }

src/vmm/src/device_manager/legacy.rs

@@ -11,11 +11,11 @@ use std::sync::{Arc, Mutex};
 
 use acpi_tables::aml::AmlError;
 use acpi_tables::{Aml, aml};
-use kvm_ioctls::VmFd;
 use libc::EFD_NONBLOCK;
 use vm_superio::Serial;
 use vmm_sys_util::eventfd::EventFd;
 
+use crate::Vm;
 use crate::devices::legacy::serial::SerialOut;
 use crate::devices::legacy::{EventFdTrigger, I8042Device, SerialDevice, SerialEventsWrapper};
 
@@ -100,7 +100,7 @@ impl PortIODeviceManager {
     pub fn register_devices(
         &mut self,
         io_bus: &vm_device::Bus,
-        vm_fd: &VmFd,
+        vm: &Vm,
     ) -> Result<(), LegacyDeviceError> {
         let serial_2_4 = Arc::new(Mutex::new(SerialDevice {
             serial: Serial::with_events(
@@ -148,18 +148,15 @@ impl PortIODeviceManager {
             Self::I8042_KDB_DATA_REGISTER_SIZE,
         )?;
 
-        vm_fd
-            .register_irqfd(&self.com_evt_1_3, Self::COM_EVT_1_3_GSI)
+        vm.register_irq(&self.com_evt_1_3, Self::COM_EVT_1_3_GSI)
             .map_err(|e| {
                 LegacyDeviceError::EventFd(std::io::Error::from_raw_os_error(e.errno()))
             })?;
-        vm_fd
-            .register_irqfd(&self.com_evt_2_4, Self::COM_EVT_2_4_GSI)
+        vm.register_irq(&self.com_evt_2_4, Self::COM_EVT_2_4_GSI)
             .map_err(|e| {
                 LegacyDeviceError::EventFd(std::io::Error::from_raw_os_error(e.errno()))
             })?;
-        vm_fd
-            .register_irqfd(&self.kbd_evt, Self::KBD_EVT_GSI)
+        vm.register_irq(&self.kbd_evt, Self::KBD_EVT_GSI)
             .map_err(|e| {
                 LegacyDeviceError::EventFd(std::io::Error::from_raw_os_error(e.errno()))
             })?;
@@ -264,6 +261,6 @@ mod tests {
             )),
         )
         .unwrap();
-        ldm.register_devices(&io_bus, vm.fd()).unwrap();
+        ldm.register_devices(&io_bus, &vm).unwrap();
     }
 }