feat(virtio/mmio): MmioTransport knows about vhost_user

Now `MmioTransport` stores info if the device uses
vhost_user implementation. Based on this info the
interrupt status reported to the guest is either
an updatde interrupt status (for usual virtio devices)
or `VIRTIO_MMIO_INT_VRING`(0x1) for vhost_user devices.
This is done because vhost_user devices can not update
interrupt status variable in the FC.

Signed-off-by: Egor Lazarchuk <[email protected]>

Author: ShadowCurse

src/vmm/src/builder.rs

@@ -835,13 +835,14 @@ fn attach_virtio_device<T: 'static + VirtioDevice + MutEventSubscriber + Debug>(
     id: String,
     device: Arc<Mutex<T>>,
     cmdline: &mut LoaderKernelCmdline,
+    is_vhost_user: bool,
 ) -> Result<(), StartMicrovmError> {
     use self::StartMicrovmError::*;
 
     event_manager.add_subscriber(device.clone());
 
     // The device mutex mustn't be locked here otherwise it will deadlock.
-    let device = MmioTransport::new(vmm.guest_memory().clone(), device);
+    let device = MmioTransport::new(vmm.guest_memory().clone(), device, is_vhost_user);
     vmm.mmio_device_manager
         .register_mmio_virtio_for_boot(vmm.vm.fd(), id, device, cmdline)
         .map_err(RegisterMmioDevice)
@@ -875,7 +876,14 @@ fn attach_entropy_device(
         .id()
         .to_string();
 
-    attach_virtio_device(event_manager, vmm, id, entropy_device.clone(), cmdline)
+    attach_virtio_device(
+        event_manager,
+        vmm,
+        id,
+        entropy_device.clone(),
+        cmdline,
+        false,
+    )
 }
 
 fn attach_block_devices<'a, I: Iterator<Item = &'a BlockDeviceType> + Debug>(
@@ -885,7 +893,7 @@ fn attach_block_devices<'a, I: Iterator<Item = &'a BlockDeviceType> + Debug>(
     event_manager: &mut EventManager,
 ) -> Result<(), StartMicrovmError> {
     macro_rules! attach_block {
-        ($block:ident) => {
+        ($block:ident, $is_vhost_user:ident) => {
             let id = {
                 let locked = $block.lock().expect("Poisoned lock");
                 if locked.root_device {
@@ -903,17 +911,24 @@ fn attach_block_devices<'a, I: Iterator<Item = &'a BlockDeviceType> + Debug>(
                 locked.id.clone()
             };
             // The device mutex mustn't be locked here otherwise it will deadlock.
-            attach_virtio_device(event_manager, vmm, id, $block.clone(), cmdline)?;
+            attach_virtio_device(
+                event_manager,
+                vmm,
+                id,
+                $block.clone(),
+                cmdline,
+                $is_vhost_user,
+            )?;
         };
     }
 
     for block in blocks {
         match block {
             BlockDeviceType::VirtioBlock(block) => {
-                attach_block!(block);
+                attach_block!(block, false);
             }
             BlockDeviceType::VhostUserBlock(block) => {
-                attach_block!(block);
+                attach_block!(block, true);
             }
         };
     }
@@ -929,7 +944,7 @@ fn attach_net_devices<'a, I: Iterator<Item = &'a Arc<Mutex<Net>>> + Debug>(
     for net_device in net_devices {
         let id = net_device.lock().expect("Poisoned lock").id().clone();
         // The device mutex mustn't be locked here otherwise it will deadlock.
-        attach_virtio_device(event_manager, vmm, id, net_device.clone(), cmdline)?;
+        attach_virtio_device(event_manager, vmm, id, net_device.clone(), cmdline, false)?;
     }
     Ok(())
 }
@@ -942,7 +957,7 @@ fn attach_unixsock_vsock_device(
 ) -> Result<(), StartMicrovmError> {
     let id = String::from(unix_vsock.lock().expect("Poisoned lock").id());
     // The device mutex mustn't be locked here otherwise it will deadlock.
-    attach_virtio_device(event_manager, vmm, id, unix_vsock.clone(), cmdline)
+    attach_virtio_device(event_manager, vmm, id, unix_vsock.clone(), cmdline, false)
 }
 
 fn attach_balloon_device(
@@ -953,7 +968,7 @@ fn attach_balloon_device(
 ) -> Result<(), StartMicrovmError> {
     let id = String::from(balloon.lock().expect("Poisoned lock").id());
     // The device mutex mustn't be locked here otherwise it will deadlock.
-    attach_virtio_device(event_manager, vmm, id, balloon.clone(), cmdline)
+    attach_virtio_device(event_manager, vmm, id, balloon.clone(), cmdline, false)
 }
 
 // Adds `O_NONBLOCK` to the stdout flags.

src/vmm/src/device_manager/mmio.rs

@@ -483,7 +483,7 @@ mod tests {
             cmdline: &mut kernel_cmdline::Cmdline,
             dev_id: &str,
         ) -> Result<u64, MmioError> {
-            let mmio_device = MmioTransport::new(guest_mem, device);
+            let mmio_device = MmioTransport::new(guest_mem, device, false);
             let device_info =
                 self.register_mmio_virtio_for_boot(vm, dev_id.to_string(), mmio_device, cmdline)?;
             Ok(device_info.addr)

src/vmm/src/device_manager/persist.rs

@@ -446,6 +446,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
         }
 
         let mut restore_helper = |device: Arc<Mutex<dyn VirtioDevice>>,
+                                  is_vhost_user: bool,
                                   as_subscriber: Arc<Mutex<dyn MutEventSubscriber>>,
                                   id: &String,
                                   state: &MmioTransportState,
@@ -455,6 +456,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
             let restore_args = MmioTransportConstructorArgs {
                 mem: mem.clone(),
                 device,
+                is_vhost_user,
             };
             let mmio_transport = MmioTransport::restore(restore_args, state)
                 .map_err(|()| DevicePersistError::MmioTransport)?;
@@ -499,6 +501,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
 
             restore_helper(
                 device.clone(),
+                false,
                 device,
                 &balloon_state.device_id,
                 &balloon_state.transport_state,
@@ -520,6 +523,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
 
             restore_helper(
                 device.clone(),
+                false,
                 device,
                 &virtio_block_state.device_id,
                 &virtio_block_state.transport_state,
@@ -541,6 +545,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
 
             restore_helper(
                 device.clone(),
+                true,
                 device,
                 &vhost_user_block_state.device_id,
                 &vhost_user_block_state.transport_state,
@@ -586,6 +591,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
 
             restore_helper(
                 device.clone(),
+                false,
                 device,
                 &net_state.device_id,
                 &net_state.transport_state,
@@ -614,6 +620,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
 
             restore_helper(
                 device.clone(),
+                false,
                 device,
                 &vsock_state.device_id,
                 &vsock_state.transport_state,
@@ -637,6 +644,7 @@ impl<'a> Persist<'a> for MMIODeviceManager {
 
             restore_helper(
                 device.clone(),
+                false,
                 device,
                 &entropy_state.device_id,
                 &entropy_state.transport_state,

src/vmm/src/devices/virtio/mmio.rs

@@ -57,11 +57,16 @@ pub struct MmioTransport {
     pub(crate) config_generation: u32,
     mem: GuestMemoryMmap,
     pub(crate) interrupt_status: Arc<AtomicU32>,
+    pub is_vhost_user: bool,
 }
 
 impl MmioTransport {
     /// Constructs a new MMIO transport for the given virtio device.
-    pub fn new(mem: GuestMemoryMmap, device: Arc<Mutex<dyn VirtioDevice>>) -> MmioTransport {
+    pub fn new(
+        mem: GuestMemoryMmap,
+        device: Arc<Mutex<dyn VirtioDevice>>,
+        is_vhost_user: bool,
+    ) -> MmioTransport {
         let interrupt_status = device.lock().expect("Poisoned lock").interrupt_status();
 
         MmioTransport {
@@ -73,6 +78,7 @@ impl MmioTransport {
             config_generation: 0,
             mem,
             interrupt_status,
+            is_vhost_user,
         }
     }
 
@@ -239,7 +245,19 @@ impl MmioTransport {
                     }
                     0x34 => self.with_queue(0, |q| u32::from(q.get_max_size())),
                     0x44 => self.with_queue(0, |q| u32::from(q.ready)),
-                    0x60 => self.interrupt_status.load(Ordering::SeqCst),
+                    0x60 => {
+                        // For vhost-user backed devices we can only report
+                        // `VIRTIO_MMIO_INT_VRING` (bit 0 set) value, as any
+                        // changes to the interrupt status value cannot be propagated
+                        // back to FC from vhost-user-backend. This also means that backed should
+                        // not change device configuration as it also requires update to the
+                        // interrupt status (bit 1 set).
+                        if !self.is_vhost_user {
+                            self.interrupt_status.load(Ordering::SeqCst)
+                        } else {
+                            VIRTIO_MMIO_INT_VRING
+                        }
+                    }
                     0x70 => self.device_status,
                     0xfc => self.config_generation,
                     _ => {
@@ -440,7 +458,7 @@ pub(crate) mod tests {
         let mut dummy = DummyDevice::new();
         // Validate reset is no-op.
         assert!(dummy.reset().is_none());
-        let mut d = MmioTransport::new(m, Arc::new(Mutex::new(dummy)));
+        let mut d = MmioTransport::new(m, Arc::new(Mutex::new(dummy)), false);
 
         // We just make sure here that the implementation of a mmio device behaves as we expect,
         // given a known virtio device implementation (the dummy device).
@@ -469,7 +487,7 @@ pub(crate) mod tests {
     #[test]
     fn test_bus_device_read() {
         let m = GuestMemoryMmap::from_raw_regions(&[(GuestAddress(0), 0x1000)], false).unwrap();
-        let mut d = MmioTransport::new(m, Arc::new(Mutex::new(DummyDevice::new())));
+        let mut d = MmioTransport::new(m, Arc::new(Mutex::new(DummyDevice::new())), false);
 
         let mut buf = vec![0xff, 0, 0xfe, 0];
         let buf_copy = buf.to_vec();
@@ -520,6 +538,11 @@ pub(crate) mod tests {
         d.bus_read(0x60, &mut buf[..]);
         assert_eq!(read_le_u32(&buf[..]), 111);
 
+        d.is_vhost_user = true;
+        d.interrupt_status.store(222, Ordering::SeqCst);
+        d.bus_read(0x60, &mut buf[..]);
+        assert_eq!(read_le_u32(&buf[..]), VIRTIO_MMIO_INT_VRING);
+
         d.bus_read(0x70, &mut buf[..]);
         assert_eq!(read_le_u32(&buf[..]), 0);
 
@@ -548,7 +571,7 @@ pub(crate) mod tests {
     fn test_bus_device_write() {
         let m = GuestMemoryMmap::from_raw_regions(&[(GuestAddress(0), 0x1000)], false).unwrap();
         let dummy_dev = Arc::new(Mutex::new(DummyDevice::new()));
-        let mut d = MmioTransport::new(m, dummy_dev.clone());
+        let mut d = MmioTransport::new(m, dummy_dev.clone(), false);
         let mut buf = vec![0; 5];
         write_le_u32(&mut buf[..4], 1);
 
@@ -706,7 +729,7 @@ pub(crate) mod tests {
     #[test]
     fn test_bus_device_activate() {
         let m = GuestMemoryMmap::from_raw_regions(&[(GuestAddress(0), 0x1000)], false).unwrap();
-        let mut d = MmioTransport::new(m, Arc::new(Mutex::new(DummyDevice::new())));
+        let mut d = MmioTransport::new(m, Arc::new(Mutex::new(DummyDevice::new())), false);
 
         assert!(!d.are_queues_valid());
         assert!(!d.locked_device().is_activated());
@@ -824,7 +847,7 @@ pub(crate) mod tests {
     #[test]
     fn test_bus_device_reset() {
         let m = GuestMemoryMmap::from_raw_regions(&[(GuestAddress(0), 0x1000)], false).unwrap();
-        let mut d = MmioTransport::new(m, Arc::new(Mutex::new(DummyDevice::new())));
+        let mut d = MmioTransport::new(m, Arc::new(Mutex::new(DummyDevice::new())), false);
         let mut buf = [0; 4];
 
         assert!(!d.are_queues_valid());

src/vmm/src/devices/virtio/persist.rs

@@ -205,6 +205,8 @@ pub struct MmioTransportConstructorArgs {
     pub mem: GuestMemoryMmap,
     /// Device associated with the current MMIO state.
     pub device: Arc<Mutex<dyn VirtioDevice>>,
+    /// Is device backed by vhost-user.
+    pub is_vhost_user: bool,
 }
 
 impl Persist<'_> for MmioTransport {
@@ -226,7 +228,11 @@ impl Persist<'_> for MmioTransport {
         constructor_args: Self::ConstructorArgs,
         state: &Self::State,
     ) -> Result<Self, Self::Error> {
-        let mut transport = MmioTransport::new(constructor_args.mem, constructor_args.device);
+        let mut transport = MmioTransport::new(
+            constructor_args.mem,
+            constructor_args.device,
+            constructor_args.is_vhost_user,
+        );
         transport.features_select = state.features_select;
         transport.acked_features_select = state.acked_features_select;
         transport.queue_select = state.queue_select;
@@ -391,7 +397,11 @@ mod tests {
             .serialize(&mut buf.as_mut_slice(), &version_map, 1)
             .unwrap();
 
-        let restore_args = MmioTransportConstructorArgs { mem, device };
+        let restore_args = MmioTransportConstructorArgs {
+            mem,
+            device,
+            is_vhost_user: false,
+        };
         let restored_mmio_transport = MmioTransport::restore(
             restore_args,
             &MmioTransportState::deserialize(&mut buf.as_slice(), &version_map, 1).unwrap(),
@@ -412,15 +422,15 @@ mod tests {
             FileEngineType::default(),
         );
         let block = Arc::new(Mutex::new(block));
-        let mmio_transport = MmioTransport::new(mem.clone(), block.clone());
+        let mmio_transport = MmioTransport::new(mem.clone(), block.clone(), false);
 
         (mmio_transport, mem, block)
     }
 
     fn create_default_net() -> (MmioTransport, GuestMemoryMmap, Arc<Mutex<Net>>) {
         let mem = default_mem();
         let net = Arc::new(Mutex::new(default_net()));
-        let mmio_transport = MmioTransport::new(mem.clone(), net.clone());
+        let mmio_transport = MmioTransport::new(mem.clone(), net.clone(), false);
 
         (mmio_transport, mem, net)
     }
@@ -440,7 +450,7 @@ mod tests {
         let backend = VsockUnixBackend::new(guest_cid, uds_path).unwrap();
         let vsock = Vsock::new(guest_cid, backend).unwrap();
         let vsock = Arc::new(Mutex::new(vsock));
-        let mmio_transport = MmioTransport::new(mem.clone(), vsock.clone());
+        let mmio_transport = MmioTransport::new(mem.clone(), vsock.clone(), false);
 
         (mmio_transport, mem, vsock)
     }