implement userspace bounce buffering support

Add support to our virtio devices to allow userspace bounce buffering of
virtio buffers. This is an alternative to swiotlb.

Don't implement it for vhost-user-blk and for virtio-block with async
engine, because I have no idea how that would even work.

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

src/vmm/src/device_manager/mmio.rs

@@ -604,6 +604,14 @@ mod tests {
             unimplemented!()
         }
 
+        fn force_userspace_bounce_buffers(&mut self) {
+            todo!()
+        }
+
+        fn userspace_bounce_buffers(&self) -> bool {
+            todo!()
+        }
+
         fn device_type(&self) -> u32 {
             0
         }

src/vmm/src/devices/virtio/balloon/device.rs

@@ -563,6 +563,14 @@ impl VirtioDevice for Balloon {
         self.avail_features |= 1 << VIRTIO_F_ACCESS_PLATFORM;
     }
 
+    fn force_userspace_bounce_buffers(&mut self) {
+        // balloon device doesn't have a need for bounce buffers
+    }
+
+    fn userspace_bounce_buffers(&self) -> bool {
+        false
+    }
+
     fn device_type(&self) -> u32 {
         TYPE_BALLOON
     }

src/vmm/src/devices/virtio/block/device.rs

@@ -155,6 +155,20 @@ impl VirtioDevice for Block {
         }
     }
 
+    fn force_userspace_bounce_buffers(&mut self) {
+        match self {
+            Block::Virtio(b) => b.force_userspace_bounce_buffers(),
+            Block::VhostUser(b) => b.force_userspace_bounce_buffers(),
+        }
+    }
+
+    fn userspace_bounce_buffers(&self) -> bool {
+        match self {
+            Block::Virtio(b) => b.userspace_bounce_buffers(),
+            Block::VhostUser(b) => b.userspace_bounce_buffers(),
+        }
+    }
+
     fn device_type(&self) -> u32 {
         TYPE_BLOCK
     }

src/vmm/src/devices/virtio/block/vhost_user/device.rs

@@ -300,6 +300,14 @@ impl<T: VhostUserHandleBackend + Send + 'static> VirtioDevice for VhostUserBlock
         self.avail_features |= 1 << VIRTIO_F_ACCESS_PLATFORM;
     }
 
+    fn force_userspace_bounce_buffers(&mut self) {
+        // Nothing Firecracker can do about this, the backend would need to do the bouncing
+    }
+
+    fn userspace_bounce_buffers(&self) -> bool {
+        false
+    }
+
     fn device_type(&self) -> u32 {
         TYPE_BLOCK
     }

src/vmm/src/devices/virtio/block/virtio/device.rs

@@ -584,6 +584,20 @@ impl VirtioDevice for VirtioBlock {
         self.avail_features |= 1 << VIRTIO_F_ACCESS_PLATFORM;
     }
 
+    fn force_userspace_bounce_buffers(&mut self) {
+        match self.disk.file_engine {
+            FileEngine::Async(_) => panic!("No idea how this is supposed to work for io_uring"),
+            FileEngine::Sync(ref mut engine) => engine.start_bouncing(),
+        }
+    }
+
+    fn userspace_bounce_buffers(&self) -> bool {
+        match self.disk.file_engine {
+            FileEngine::Async(_) => false,
+            FileEngine::Sync(ref engine) => engine.is_bouncing(),
+        }
+    }
+
     fn device_type(&self) -> u32 {
         TYPE_BLOCK
     }

src/vmm/src/devices/virtio/block/virtio/io/sync_io.rs

@@ -6,7 +6,7 @@ use std::io::{Seek, SeekFrom, Write};
 
 use vm_memory::{GuestMemoryError, ReadVolatile, WriteVolatile};
 
-use crate::vstate::memory::{GuestAddress, GuestMemory, GuestMemoryMmap};
+use crate::vstate::memory::{GuestAddress, GuestMemory, GuestMemoryMmap, MaybeBounce};
 
 #[derive(Debug, thiserror::Error, displaydoc::Display)]
 pub enum SyncIoError {
@@ -22,25 +22,36 @@ pub enum SyncIoError {
 
 #[derive(Debug)]
 pub struct SyncFileEngine {
-    file: File,
+    // 65536 is the largest buffer a linux guest will give us, empirically
+    file: MaybeBounce<File, { u16::MAX as usize + 1 }>,
 }
 
 // SAFETY: `File` is send and ultimately a POD.
 unsafe impl Send for SyncFileEngine {}
 
 impl SyncFileEngine {
     pub fn from_file(file: File) -> SyncFileEngine {
-        SyncFileEngine { file }
+        SyncFileEngine {
+            file: MaybeBounce::new_persistent(file, false),
+        }
     }
 
     #[cfg(test)]
     pub fn file(&self) -> &File {
-        &self.file
+        &self.file.target
+    }
+
+    pub fn start_bouncing(&mut self) {
+        self.file.should_bounce = true
+    }
+
+    pub fn is_bouncing(&self) -> bool {
+        self.file.should_bounce
     }
 
     /// Update the backing file of the engine
     pub fn update_file(&mut self, file: File) {
-        self.file = file
+        self.file.target = file
     }
 
     pub fn read(
@@ -77,8 +88,8 @@ impl SyncFileEngine {
 
     pub fn flush(&mut self) -> Result<(), SyncIoError> {
         // flush() first to force any cached data out of rust buffers.
-        self.file.flush().map_err(SyncIoError::Flush)?;
+        self.file.target.flush().map_err(SyncIoError::Flush)?;
         // Sync data out to physical media on host.
-        self.file.sync_all().map_err(SyncIoError::SyncAll)
+        self.file.target.sync_all().map_err(SyncIoError::SyncAll)
     }
 }

src/vmm/src/devices/virtio/block/virtio/persist.rs

@@ -16,7 +16,7 @@ use crate::devices::virtio::TYPE_BLOCK;
 use crate::devices::virtio::block::persist::BlockConstructorArgs;
 use crate::devices::virtio::block::virtio::device::FileEngineType;
 use crate::devices::virtio::block::virtio::metrics::BlockMetricsPerDevice;
-use crate::devices::virtio::device::{DeviceState, IrqTrigger};
+use crate::devices::virtio::device::{DeviceState, IrqTrigger, VirtioDevice};
 use crate::devices::virtio::generated::virtio_blk::VIRTIO_BLK_F_RO;
 use crate::devices::virtio::persist::VirtioDeviceState;
 use crate::rate_limiter::RateLimiter;
@@ -127,7 +127,7 @@ impl Persist<'_> for VirtioBlock {
             capacity: disk_properties.nsectors.to_le(),
         };
 
-        Ok(VirtioBlock {
+        let mut dev = VirtioBlock {
             avail_features,
             acked_features,
             config_space,
@@ -148,7 +148,13 @@ impl Persist<'_> for VirtioBlock {
             rate_limiter,
             is_io_engine_throttled: false,
             metrics: BlockMetricsPerDevice::alloc(state.id.clone()),
-        })
+        };
+
+        if state.virtio_state.bounce_in_userspace {
+            dev.force_userspace_bounce_buffers()
+        }
+
+        Ok(dev)
     }
 }
 

src/vmm/src/devices/virtio/device.rs

@@ -105,6 +105,12 @@ pub trait VirtioDevice: AsAny + Send {
     /// Make the virtio device offer the VIRTIO_F_ACCESS_PLATFORM feature
     fn force_swiotlb(&mut self);
 
+    /// Make the virtio device user userspace bounce buffers
+    fn force_userspace_bounce_buffers(&mut self);
+
+    /// Whether this device is using userspace bounce buffers
+    fn userspace_bounce_buffers(&self) -> bool;
+
     /// Check if virtio device has negotiated given feature.
     fn has_feature(&self, feature: u64) -> bool {
         (self.acked_features() & (1 << feature)) != 0
@@ -266,6 +272,14 @@ pub(crate) mod tests {
             unimplemented!()
         }
 
+        fn force_userspace_bounce_buffers(&mut self) {
+            todo!()
+        }
+
+        fn userspace_bounce_buffers(&self) -> bool {
+            todo!()
+        }
+
         fn device_type(&self) -> u32 {
             todo!()
         }

src/vmm/src/devices/virtio/mmio.rs

@@ -427,6 +427,14 @@ pub(crate) mod tests {
             unimplemented!()
         }
 
+        fn force_userspace_bounce_buffers(&mut self) {
+            todo!()
+        }
+
+        fn userspace_bounce_buffers(&self) -> bool {
+            todo!()
+        }
+
         fn device_type(&self) -> u32 {
             123
         }