TBS

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

src/vmm/src/device_manager/persist.rs

@@ -473,22 +473,10 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                                   as_subscriber: Arc<Mutex<dyn MutEventSubscriber>>,
                                   id: &String,
                                   state: &MmioTransportState,
+                                  interrupt: Arc<IrqTrigger>,
                                   device_info: &MMIODeviceInfo,
                                   event_manager: &mut EventManager|
          -> Result<(), Self::Error> {
-            // If the device was already activated, an interrupt was created and assigned to it. In
-            // that case grab a reference to it and pass it to the transport state. Otherwise,
-            // create a fresh one. Like this both the transport and the device will hold a
-            // reference to the same interrupt object, at all times.
-            let interrupt = {
-                let locked_dev = device.lock().expect("Poisoned lock");
-                if locked_dev.is_activated() {
-                    locked_dev.interrupt_trigger()
-                } else {
-                    Arc::new(IrqTrigger::new().expect("Could not create IRQ for MMIO device"))
-                }
-            };
-
             let restore_args = MmioTransportConstructorArgs {
                 mem: mem.clone(),
                 interrupt,
@@ -526,9 +514,12 @@ impl<'a> Persist<'a> for MMIODeviceManager {
         };
 
         if let Some(balloon_state) = &state.balloon_device {
+            let interrupt =
+                Arc::new(IrqTrigger::new().expect("Could not create interrupt for MMIO device"));
             let device = Arc::new(Mutex::new(Balloon::restore(
                 BalloonConstructorArgs {
                     mem: mem.clone(),
+                    interrupt: interrupt.clone(),
                     restored_from_file: constructor_args.restored_from_file,
                 },
                 &balloon_state.device_state,
@@ -544,14 +535,20 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                 device,
                 &balloon_state.device_id,
                 &balloon_state.transport_state,
+                interrupt,
                 &balloon_state.device_info,
                 constructor_args.event_manager,
             )?;
         }
 
         for block_state in &state.block_devices {
+            let interrupt =
+                Arc::new(IrqTrigger::new().expect("Could not create interrupt for MMIO device"));
             let device = Arc::new(Mutex::new(Block::restore(
-                BlockConstructorArgs { mem: mem.clone() },
+                BlockConstructorArgs {
+                    mem: mem.clone(),
+                    interrupt: interrupt.clone(),
+                },
                 &block_state.device_state,
             )?));
 
@@ -565,6 +562,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                 device,
                 &block_state.device_id,
                 &block_state.transport_state,
+                interrupt,
                 &block_state.device_info,
                 constructor_args.event_manager,
             )?;
@@ -587,9 +585,12 @@ impl<'a> Persist<'a> for MMIODeviceManager {
         }
 
         for net_state in &state.net_devices {
+            let interrupt =
+                Arc::new(IrqTrigger::new().expect("Could not create interrupt for MMIO device"));
             let device = Arc::new(Mutex::new(Net::restore(
                 NetConstructorArgs {
                     mem: mem.clone(),
+                    interrupt: interrupt.clone(),
                     mmds: constructor_args
                         .vm_resources
                         .mmds
@@ -610,6 +611,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                 device,
                 &net_state.device_id,
                 &net_state.transport_state,
+                interrupt,
                 &net_state.device_info,
                 constructor_args.event_manager,
             )?;
@@ -620,9 +622,12 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                 cid: vsock_state.device_state.frontend.cid,
             };
             let backend = VsockUnixBackend::restore(ctor_args, &vsock_state.device_state.backend)?;
+            let interrupt =
+                Arc::new(IrqTrigger::new().expect("Could not create interrupt for MMIO device"));
             let device = Arc::new(Mutex::new(Vsock::restore(
                 VsockConstructorArgs {
                     mem: mem.clone(),
+                    interrupt: interrupt.clone(),
                     backend,
                 },
                 &vsock_state.device_state.frontend,
@@ -638,13 +643,16 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                 device,
                 &vsock_state.device_id,
                 &vsock_state.transport_state,
+                interrupt,
                 &vsock_state.device_info,
                 constructor_args.event_manager,
             )?;
         }
 
         if let Some(entropy_state) = &state.entropy_device {
-            let ctor_args = EntropyConstructorArgs::new(mem.clone());
+            let interrupt =
+                Arc::new(IrqTrigger::new().expect("Could not create interrupt for MMIO device"));
+            let ctor_args = EntropyConstructorArgs::new(mem.clone(), interrupt.clone());
 
             let device = Arc::new(Mutex::new(Entropy::restore(
                 ctor_args,
@@ -661,6 +669,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
                 device,
                 &entropy_state.device_id,
                 &entropy_state.transport_state,
+                interrupt,
                 &entropy_state.device_info,
                 constructor_args.event_manager,
             )?;

src/vmm/src/devices/virtio/balloon/persist.rs

@@ -4,7 +4,6 @@
 //! Defines the structures needed for saving/restoring balloon devices.
 
 use std::sync::Arc;
-use std::sync::atomic::AtomicU32;
 use std::time::Duration;
 
 use serde::{Deserialize, Serialize};
@@ -96,6 +95,8 @@ pub struct BalloonState {
 pub struct BalloonConstructorArgs {
     /// Pointer to guest memory.
     pub mem: GuestMemoryMmap,
+    /// Interrupt used from the device.
+    pub interrupt: Arc<IrqTrigger>,
     pub restored_from_file: bool,
 }
 
@@ -154,10 +155,8 @@ impl Persist<'_> for Balloon {
         };
 
         if state.virtio_state.activated {
-            let mut interrupt = IrqTrigger::new().expect("Could not create IRQ for VirtIO device");
-            interrupt.irq_status = Arc::new(AtomicU32::new(state.virtio_state.interrupt_status));
             balloon.device_state =
-                DeviceState::Activated((constructor_args.mem, Arc::new(interrupt)));
+                DeviceState::Activated((constructor_args.mem, constructor_args.interrupt));
 
             if balloon.stats_enabled() {
                 // Restore the stats descriptor.
@@ -184,7 +183,7 @@ mod tests {
     use super::*;
     use crate::devices::virtio::TYPE_BALLOON;
     use crate::devices::virtio::device::VirtioDevice;
-    use crate::devices::virtio::test_utils::default_mem;
+    use crate::devices::virtio::test_utils::{default_interrupt, default_mem};
     use crate::snapshot::Snapshot;
 
     #[test]
@@ -201,6 +200,7 @@ mod tests {
         let restored_balloon = Balloon::restore(
             BalloonConstructorArgs {
                 mem: guest_mem,
+                interrupt: default_interrupt(),
                 restored_from_file: true,
             },
             &Snapshot::deserialize(&mut mem.as_slice()).unwrap(),

src/vmm/src/devices/virtio/block/persist.rs

@@ -1,10 +1,13 @@
 // Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+use std::sync::Arc;
+
 use serde::{Deserialize, Serialize};
 
 use super::vhost_user::persist::VhostUserBlockState;
 use super::virtio::persist::VirtioBlockState;
+use crate::devices::virtio::transport::mmio::IrqTrigger;
 use crate::vstate::memory::GuestMemoryMmap;
 
 /// Block device state.
@@ -18,4 +21,5 @@ pub enum BlockState {
 #[derive(Debug)]
 pub struct BlockConstructorArgs {
     pub mem: GuestMemoryMmap,
+    pub interrupt: Arc<IrqTrigger>,
 }

src/vmm/src/devices/virtio/block/virtio/persist.rs

@@ -3,9 +3,6 @@
 
 //! Defines the structures needed for saving/restoring block devices.
 
-use std::sync::Arc;
-use std::sync::atomic::AtomicU32;
-
 use device::ConfigSpace;
 use serde::{Deserialize, Serialize};
 use vmm_sys_util::eventfd::EventFd;
@@ -19,7 +16,6 @@ use crate::devices::virtio::block::virtio::metrics::BlockMetricsPerDevice;
 use crate::devices::virtio::device::DeviceState;
 use crate::devices::virtio::generated::virtio_blk::VIRTIO_BLK_F_RO;
 use crate::devices::virtio::persist::VirtioDeviceState;
-use crate::devices::virtio::transport::mmio::IrqTrigger;
 use crate::rate_limiter::RateLimiter;
 use crate::rate_limiter::persist::RateLimiterState;
 use crate::snapshot::Persist;
@@ -112,16 +108,11 @@ impl Persist<'_> for VirtioBlock {
             )
             .map_err(VirtioBlockError::Persist)?;
 
-        let mut irq_trigger = IrqTrigger::new().map_err(VirtioBlockError::IrqTrigger)?;
-        irq_trigger.irq_status = Arc::new(AtomicU32::new(state.virtio_state.interrupt_status));
-
         let avail_features = state.virtio_state.avail_features;
         let acked_features = state.virtio_state.acked_features;
 
         let device_state = if state.virtio_state.activated {
-            let mut interrupt = IrqTrigger::new().expect("Could not create IRQ for VirtIO device");
-            interrupt.irq_status = Arc::new(AtomicU32::new(state.virtio_state.interrupt_status));
-            DeviceState::Activated((constructor_args.mem, Arc::new(interrupt)))
+            DeviceState::Activated((constructor_args.mem, constructor_args.interrupt))
         } else {
             DeviceState::Inactive
         };
@@ -161,7 +152,7 @@ mod tests {
     use super::*;
     use crate::devices::virtio::block::virtio::device::VirtioBlockConfig;
     use crate::devices::virtio::device::VirtioDevice;
-    use crate::devices::virtio::test_utils::default_mem;
+    use crate::devices::virtio::test_utils::{default_interrupt, default_mem};
     use crate::snapshot::Snapshot;
 
     #[test]
@@ -233,7 +224,10 @@ mod tests {
 
         // Restore the block device.
         let restored_block = VirtioBlock::restore(
-            BlockConstructorArgs { mem: guest_mem },
+            BlockConstructorArgs {
+                mem: guest_mem,
+                interrupt: default_interrupt(),
+            },
             &Snapshot::deserialize(&mut mem.as_slice()).unwrap(),
         )
         .unwrap();

src/vmm/src/devices/virtio/net/persist.rs

@@ -4,7 +4,6 @@
 //! Defines the structures needed for saving/restoring net devices.
 
 use std::io;
-use std::sync::atomic::AtomicU32;
 use std::sync::{Arc, Mutex};
 
 use serde::{Deserialize, Serialize};
@@ -72,6 +71,8 @@ pub struct NetState {
 pub struct NetConstructorArgs {
     /// Pointer to guest memory.
     pub mem: GuestMemoryMmap,
+    /// Interrupt for the device.
+    pub interrupt: Arc<IrqTrigger>,
     /// Pointer to the MMDS data store.
     pub mmds: Option<Arc<Mutex<Mmds>>>,
 }
@@ -158,9 +159,8 @@ impl Persist<'_> for Net {
                 .set_offload(supported_flags)
                 .map_err(NetPersistError::TapSetOffload)?;
 
-            let mut interrupt = IrqTrigger::new().expect("Could not create IRQ for VirtIO device");
-            interrupt.irq_status = Arc::new(AtomicU32::new(state.virtio_state.interrupt_status));
-            net.device_state = DeviceState::Activated((constructor_args.mem, Arc::new(interrupt)));
+            net.device_state =
+                DeviceState::Activated((constructor_args.mem, constructor_args.interrupt));
 
             // Recreate `Net::rx_buffer`. We do it by re-parsing the RX queue. We're temporarily
             // rolling back `next_avail` in the RX queue and call `parse_rx_descriptors`.
@@ -180,7 +180,7 @@ mod tests {
     use super::*;
     use crate::devices::virtio::device::VirtioDevice;
     use crate::devices::virtio::net::test_utils::{default_net, default_net_no_mmds};
-    use crate::devices::virtio::test_utils::default_mem;
+    use crate::devices::virtio::test_utils::{default_interrupt, default_mem};
     use crate::snapshot::Snapshot;
 
     fn validate_save_and_restore(net: Net, mmds_ds: Option<Arc<Mutex<Mmds>>>) {
@@ -213,6 +213,7 @@ mod tests {
             match Net::restore(
                 NetConstructorArgs {
                     mem: guest_mem,
+                    interrupt: default_interrupt(),
                     mmds: mmds_ds,
                 },
                 &Snapshot::deserialize(&mut mem.as_slice()).unwrap(),

src/vmm/src/devices/virtio/persist.rs

@@ -122,27 +122,18 @@ pub struct VirtioDeviceState {
     pub acked_features: u64,
     /// List of queues.
     pub queues: Vec<QueueState>,
-    /// The MMIO interrupt status.
-    pub interrupt_status: u32,
     /// Flag for activated status.
     pub activated: bool,
 }
 
 impl VirtioDeviceState {
     /// Construct the virtio state of a device.
     pub fn from_device(device: &dyn VirtioDevice) -> Self {
-        let interrupt_status = if device.is_activated() {
-            device.interrupt_status().load(Ordering::Relaxed)
-        } else {
-            0
-        };
-
         VirtioDeviceState {
             device_type: device.device_type(),
             avail_features: device.avail_features(),
             acked_features: device.acked_features(),
             queues: device.queues().iter().map(Persist::save).collect(),
-            interrupt_status,
             activated: device.is_activated(),
         }
     }
@@ -214,6 +205,7 @@ pub struct MmioTransportState {
     queue_select: u32,
     device_status: u32,
     config_generation: u32,
+    interrupt_status: u32,
 }
 
 /// Auxiliary structure for initializing the transport when resuming from a snapshot.
@@ -241,6 +233,7 @@ impl Persist<'_> for MmioTransport {
             queue_select: self.queue_select,
             device_status: self.device_status,
             config_generation: self.config_generation,
+            interrupt_status: self.interrupt.irq_status.load(Ordering::SeqCst),
         }
     }
 
@@ -259,6 +252,10 @@ impl Persist<'_> for MmioTransport {
         transport.queue_select = state.queue_select;
         transport.device_status = state.device_status;
         transport.config_generation = state.config_generation;
+        transport
+            .interrupt
+            .irq_status
+            .store(state.interrupt_status, Ordering::SeqCst);
         Ok(transport)
     }
 }