refactor: use VirtioInterrupt in VirtIO devices

VirtIO devices assume they're operating under an MMIO transport and as a
consequence they use IrqTrigger as interrupts. Switch that to using
VirtioInterrupt for all VirtIO device objects. Only assume a
VirtioInterrupt is an IrqTrigger in MMIO specific code.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

src/vmm/src/device_manager/mmio.rs

@@ -505,7 +505,7 @@ impl MMIODeviceManager {
                             .unwrap();
                         if vsock.is_activated() {
                             info!("kick vsock {id}.");
-                            vsock.signal_used_queue().unwrap();
+                            vsock.signal_used_queue(0).unwrap();
                         }
                     }
                     TYPE_RNG => {
@@ -525,6 +525,7 @@ impl MMIODeviceManager {
 #[cfg(test)]
 mod tests {
 
+    use std::ops::Deref;
     use std::sync::Arc;
 
     use vmm_sys_util::eventfd::EventFd;
@@ -534,6 +535,7 @@ mod tests {
     use crate::devices::virtio::ActivateError;
     use crate::devices::virtio::device::VirtioDevice;
     use crate::devices::virtio::queue::Queue;
+    use crate::devices::virtio::transport::VirtioInterrupt;
     use crate::devices::virtio::transport::mmio::IrqTrigger;
     use crate::test_utils::multi_region_mem_raw;
     use crate::vstate::kvm::Kvm;
@@ -620,11 +622,8 @@ mod tests {
             &self.queue_evts
         }
 
-        fn interrupt_trigger(&self) -> Arc<IrqTrigger> {
-            self.interrupt_trigger
-                .as_ref()
-                .expect("Device is not activated")
-                .clone()
+        fn interrupt_trigger(&self) -> &dyn VirtioInterrupt {
+            self.interrupt_trigger.as_ref().unwrap().deref()
         }
 
         fn ack_features_by_page(&mut self, page: u32, value: u32) {
@@ -645,7 +644,7 @@ mod tests {
         fn activate(
             &mut self,
             _: GuestMemoryMmap,
-            _: Arc<IrqTrigger>,
+            _: Arc<dyn VirtioInterrupt>,
         ) -> Result<(), ActivateError> {
             Ok(())
         }

src/vmm/src/devices/virtio/balloon/device.rs

@@ -1,6 +1,7 @@
 // Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+use std::ops::Deref;
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -24,8 +25,9 @@ use super::{
     VIRTIO_BALLOON_S_SWAP_OUT,
 };
 use crate::devices::virtio::balloon::BalloonError;
+use crate::devices::virtio::device::ActiveState;
 use crate::devices::virtio::generated::virtio_config::VIRTIO_F_VERSION_1;
-use crate::devices::virtio::transport::mmio::{IrqTrigger, IrqType};
+use crate::devices::virtio::transport::{VirtioInterrupt, VirtioInterruptType};
 use crate::logger::IncMetric;
 use crate::utils::u64_to_usize;
 use crate::vstate::memory::{Address, ByteValued, Bytes, GuestAddress, GuestMemoryMmap};
@@ -258,7 +260,7 @@ impl Balloon {
 
     pub(crate) fn process_inflate(&mut self) -> Result<(), BalloonError> {
         // This is safe since we checked in the event handler that the device is activated.
-        let (mem, _) = self.device_state.active_state().unwrap();
+        let mem = &self.device_state.active_state().unwrap().mem;
         METRICS.inflate_count.inc();
 
         let queue = &mut self.queues[INFLATE_INDEX];
@@ -339,7 +341,7 @@ impl Balloon {
         }
 
         if needs_interrupt {
-            self.signal_used_queue()?;
+            self.signal_used_queue(INFLATE_INDEX)?;
         }
 
         Ok(())
@@ -357,15 +359,15 @@ impl Balloon {
         }
 
         if needs_interrupt {
-            self.signal_used_queue()
+            self.signal_used_queue(DEFLATE_INDEX)
         } else {
             Ok(())
         }
     }
 
     pub(crate) fn process_stats_queue(&mut self) -> Result<(), BalloonError> {
         // This is safe since we checked in the event handler that the device is activated.
-        let (mem, _) = self.device_state.active_state().unwrap();
+        let mem = &self.device_state.active_state().unwrap().mem;
         METRICS.stats_updates_count.inc();
 
         while let Some(head) = self.queues[STATS_INDEX].pop() {
@@ -401,9 +403,12 @@ impl Balloon {
         Ok(())
     }
 
-    pub(crate) fn signal_used_queue(&self) -> Result<(), BalloonError> {
+    pub(crate) fn signal_used_queue(&self, qidx: usize) -> Result<(), BalloonError> {
         self.interrupt_trigger()
-            .trigger_irq(IrqType::Vring)
+            .trigger(VirtioInterruptType::Queue(
+                qidx.try_into()
+                    .unwrap_or_else(|_| panic!("balloon: invalid queue id: {qidx}")),
+            ))
             .map_err(|err| {
                 METRICS.event_fails.inc();
                 BalloonError::InterruptError(err)
@@ -428,7 +433,7 @@ impl Balloon {
             self.queues[STATS_INDEX]
                 .add_used(index, 0)
                 .map_err(BalloonError::Queue)?;
-            self.signal_used_queue()
+            self.signal_used_queue(STATS_INDEX)
         } else {
             error!("Failed to update balloon stats, missing descriptor.");
             Ok(())
@@ -440,7 +445,7 @@ impl Balloon {
         if self.is_activated() {
             self.config_space.num_pages = mib_to_pages(amount_mib)?;
             self.interrupt_trigger()
-                .trigger_irq(IrqType::Config)
+                .trigger(VirtioInterruptType::Config)
                 .map_err(BalloonError::InterruptError)
         } else {
             Err(BalloonError::DeviceNotActive)
@@ -551,11 +556,12 @@ impl VirtioDevice for Balloon {
         &self.queue_evts
     }
 
-    fn interrupt_trigger(&self) -> Arc<IrqTrigger> {
+    fn interrupt_trigger(&self) -> &dyn VirtioInterrupt {
         self.device_state
             .active_state()
             .expect("Device is not activated")
-            .1
+            .interrupt
+            .deref()
     }
 
     fn read_config(&self, offset: u64, data: &mut [u8]) {
@@ -585,14 +591,14 @@ impl VirtioDevice for Balloon {
     fn activate(
         &mut self,
         mem: GuestMemoryMmap,
-        interrupt: Arc<IrqTrigger>,
+        interrupt: Arc<dyn VirtioInterrupt>,
     ) -> Result<(), ActivateError> {
         for q in self.queues.iter_mut() {
             q.initialize(&mem)
                 .map_err(ActivateError::QueueMemoryError)?;
         }
 
-        self.device_state = DeviceState::Activated((mem, interrupt));
+        self.device_state = DeviceState::Activated(ActiveState { mem, interrupt });
         if self.activate_evt.write(1).is_err() {
             METRICS.activate_fails.inc();
             self.device_state = DeviceState::Inactive;
@@ -1057,7 +1063,9 @@ pub(crate) mod tests {
                 assert!(balloon.stats_desc_index.is_some());
                 balloon.process_stats_timer_event().unwrap();
                 assert!(balloon.stats_desc_index.is_none());
-                assert!(balloon.interrupt_trigger().has_pending_irq(IrqType::Vring));
+                assert!(balloon.interrupt_trigger().has_pending_interrupt(
+                    VirtioInterruptType::Queue(STATS_INDEX.try_into().unwrap())
+                ));
             });
         }
     }
@@ -1112,8 +1120,10 @@ pub(crate) mod tests {
     fn test_num_pages() {
         let mut balloon = Balloon::new(0, true, 0, false).unwrap();
         // Switch the state to active.
-        balloon.device_state =
-            DeviceState::Activated((single_region_mem(0x1), default_interrupt()));
+        balloon.device_state = DeviceState::Activated(ActiveState {
+            mem: single_region_mem(0x1),
+            interrupt: default_interrupt(),
+        });
 
         assert_eq!(balloon.num_pages(), 0);
         assert_eq!(balloon.actual_pages(), 0);

src/vmm/src/devices/virtio/balloon/persist.rs

@@ -12,10 +12,10 @@ use timerfd::{SetTimeFlags, TimerState};
 use super::*;
 use crate::devices::virtio::TYPE_BALLOON;
 use crate::devices::virtio::balloon::device::{BalloonStats, ConfigSpace};
-use crate::devices::virtio::device::DeviceState;
+use crate::devices::virtio::device::{ActiveState, DeviceState};
 use crate::devices::virtio::persist::VirtioDeviceState;
 use crate::devices::virtio::queue::FIRECRACKER_MAX_QUEUE_SIZE;
-use crate::devices::virtio::transport::mmio::IrqTrigger;
+use crate::devices::virtio::transport::VirtioInterrupt;
 use crate::snapshot::Persist;
 use crate::vstate::memory::GuestMemoryMmap;
 
@@ -96,7 +96,7 @@ pub struct BalloonConstructorArgs {
     /// Pointer to guest memory.
     pub mem: GuestMemoryMmap,
     /// Interrupt used from the device.
-    pub interrupt: Arc<IrqTrigger>,
+    pub interrupt: Arc<dyn VirtioInterrupt>,
     pub restored_from_file: bool,
 }
 
@@ -155,8 +155,10 @@ impl Persist<'_> for Balloon {
         };
 
         if state.virtio_state.activated {
-            balloon.device_state =
-                DeviceState::Activated((constructor_args.mem, constructor_args.interrupt));
+            balloon.device_state = DeviceState::Activated(ActiveState {
+                mem: constructor_args.mem,
+                interrupt: constructor_args.interrupt,
+            });
 
             if balloon.stats_enabled() {
                 // Restore the stats descriptor.

src/vmm/src/devices/virtio/balloon/test_utils.rs

@@ -3,14 +3,16 @@
 
 #![doc(hidden)]
 
+#[cfg(test)]
+use crate::devices::virtio::device::VirtioDevice;
 use crate::devices::virtio::test_utils::VirtQueue;
 #[cfg(test)]
 use crate::devices::virtio::{balloon::BALLOON_NUM_QUEUES, balloon::Balloon};
 
 #[cfg(test)]
 pub fn invoke_handler_for_queue_event(b: &mut Balloon, queue_index: usize) {
     use crate::devices::virtio::balloon::{DEFLATE_INDEX, INFLATE_INDEX, STATS_INDEX};
-    use crate::devices::virtio::transport::mmio::IrqType;
+    use crate::devices::virtio::transport::VirtioInterruptType;
 
     assert!(queue_index < BALLOON_NUM_QUEUES);
     // Trigger the queue event.
@@ -23,8 +25,11 @@ pub fn invoke_handler_for_queue_event(b: &mut Balloon, queue_index: usize) {
         _ => unreachable!(),
     };
     // Validate the queue operation finished successfully.
-    let (_, interrupt) = b.device_state.active_state().unwrap();
-    assert!(interrupt.has_pending_irq(IrqType::Vring));
+    let interrupt = b.interrupt_trigger();
+    assert!(
+        interrupt
+            .has_pending_interrupt(VirtioInterruptType::Queue(queue_index.try_into().unwrap()))
+    );
 }
 
 pub fn set_request(queue: &VirtQueue, idx: u16, addr: u64, len: u32, flags: u16) {

src/vmm/src/devices/virtio/block/device.rs

@@ -12,7 +12,7 @@ use super::vhost_user::device::{VhostUserBlock, VhostUserBlockConfig};
 use super::virtio::device::{VirtioBlock, VirtioBlockConfig};
 use crate::devices::virtio::device::VirtioDevice;
 use crate::devices::virtio::queue::Queue;
-use crate::devices::virtio::transport::mmio::IrqTrigger;
+use crate::devices::virtio::transport::VirtioInterrupt;
 use crate::devices::virtio::{ActivateError, TYPE_BLOCK};
 use crate::rate_limiter::BucketUpdate;
 use crate::snapshot::Persist;
@@ -176,20 +176,10 @@ impl VirtioDevice for Block {
         }
     }
 
-    fn interrupt_trigger(&self) -> Arc<IrqTrigger> {
+    fn interrupt_trigger(&self) -> &dyn VirtioInterrupt {
         match self {
-            Self::Virtio(b) => {
-                b.device_state
-                    .active_state()
-                    .expect("Device is not initialized")
-                    .1
-            }
-            Self::VhostUser(b) => {
-                b.device_state
-                    .active_state()
-                    .expect("Device is not initialized")
-                    .1
-            }
+            Self::Virtio(b) => b.interrupt_trigger(),
+            Self::VhostUser(b) => b.interrupt_trigger(),
         }
     }
 
@@ -210,7 +200,7 @@ impl VirtioDevice for Block {
     fn activate(
         &mut self,
         mem: GuestMemoryMmap,
-        interrupt: Arc<IrqTrigger>,
+        interrupt: Arc<dyn VirtioInterrupt>,
     ) -> Result<(), ActivateError> {
         match self {
             Self::Virtio(b) => b.activate(mem, interrupt),

src/vmm/src/devices/virtio/block/persist.rs

@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
 
 use super::vhost_user::persist::VhostUserBlockState;
 use super::virtio::persist::VirtioBlockState;
-use crate::devices::virtio::transport::mmio::IrqTrigger;
+use crate::devices::virtio::transport::VirtioInterrupt;
 use crate::vstate::memory::GuestMemoryMmap;
 
 /// Block device state.
@@ -21,5 +21,5 @@ pub enum BlockState {
 #[derive(Debug)]
 pub struct BlockConstructorArgs {
     pub mem: GuestMemoryMmap,
-    pub interrupt: Arc<IrqTrigger>,
+    pub interrupt: Arc<dyn VirtioInterrupt>,
 }