feat(iovec): add support for VIRTQ_DESC_F_INDIRECT to IoVecBuffer

Now IoVecBuffer/Mut can be built from descriptor chains
utilizing VIRTQ_DESC_F_INDIRECT flag.
The way indirect descriptors work is:
- Descriptors from descriptor table instead of pointing
to the buffers where data needs to be written to/read from
now can point to buffers that contain other descriptor table.
That 'indirect' descriptor table contains descriptor which
point to actual buffers for data.
- All descriptor in the 'indirect' descriptor table are
processed sequentially.
- The `VIRTQ_DESC_F_WRITE` flag is ignored for the main descriptor
(the one from original descriptor table)

Signed-off-by: Egor Lazarchuk <[email protected]>

Author: ShadowCurse

src/vmm/src/devices/virtio/iovec.rs

@@ -6,11 +6,12 @@ use std::io::ErrorKind;
 use libc::{c_void, iovec, size_t};
 use smallvec::SmallVec;
 use vm_memory::{
-    GuestMemoryError, ReadVolatile, VolatileMemoryError, VolatileSlice, WriteVolatile,
+    GuestAddress, GuestMemoryError, ReadVolatile, VolatileMemoryError, VolatileSlice, WriteVolatile,
 };
 
+use super::queue::Descriptor;
 use crate::devices::virtio::queue::DescriptorChain;
-use crate::vstate::memory::{Bitmap, GuestMemory};
+use crate::vstate::memory::{Bitmap, GuestMemory, GuestMemoryMmap};
 
 #[derive(Debug, thiserror::Error, displaydoc::Display)]
 pub enum IoVecError {
@@ -20,6 +21,8 @@ pub enum IoVecError {
     ReadOnlyDescriptor,
     /// Tried to create an `IoVec` or `IoVecMut` from a descriptor chain that was too large
     OverflowedDescriptor,
+    /// Nested indirect descriptor
+    NestedIndirectDescriptor,
     /// Guest memory error: {0}
     GuestMemory(#[from] GuestMemoryError),
 }
@@ -63,34 +66,76 @@ impl IoVecBuffer {
 
         let mut next_descriptor = Some(head);
         while let Some(desc) = next_descriptor {
-            if desc.is_write_only() {
-                return Err(IoVecError::WriteOnlyDescriptor);
-            }
+            if desc.is_indirect() {
+                // We use get_slice instead of `get_host_address` here in order to have the whole
+                // range of the descriptor chain checked, i.e. [addr, addr + len) is a valid memory
+                // region in the GuestMemoryMmap.
+                let indirect_desc_slice = desc
+                    .mem
+                    .get_slice(desc.addr, desc.len as usize)
+                    .map_err(IoVecError::GuestMemory)?;
+
+                // SAFETY:
+                // We checked the slice above. We just transform it into
+                // a slice of Descriptors.
+                let indirect_desc_slice: &[Descriptor] = unsafe {
+                    std::slice::from_raw_parts(
+                        indirect_desc_slice.ptr_guard().as_ptr().cast(),
+                        desc.len as usize / std::mem::size_of::<Descriptor>(),
+                    )
+                };
+
+                for d in indirect_desc_slice.iter() {
+                    if desc.is_write_only() {
+                        return Err(IoVecError::WriteOnlyDescriptor);
+                    }
+                    if d.is_indirect() {
+                        return Err(IoVecError::NestedIndirectDescriptor);
+                    }
+                    self.add_descriptor(desc.mem, GuestAddress(d.addr), d.len)?;
+                    if !d.has_next() {
+                        break;
+                    }
+                }
+            } else {
+                if desc.is_write_only() {
+                    return Err(IoVecError::WriteOnlyDescriptor);
+                }
 
-            // We use get_slice instead of `get_host_address` here in order to have the whole
-            // range of the descriptor chain checked, i.e. [addr, addr + len) is a valid memory
-            // region in the GuestMemoryMmap.
-            let iov_base = desc
-                .mem
-                .get_slice(desc.addr, desc.len as usize)?
-                .ptr_guard_mut()
-                .as_ptr()
-                .cast::<c_void>();
-            self.vecs.push(iovec {
-                iov_base,
-                iov_len: desc.len as size_t,
-            });
-            self.len = self
-                .len
-                .checked_add(desc.len)
-                .ok_or(IoVecError::OverflowedDescriptor)?;
+                self.add_descriptor(desc.mem, desc.addr, desc.len)?;
+            }
 
             next_descriptor = desc.next_descriptor();
         }
 
         Ok(())
     }
 
+    fn add_descriptor(
+        &mut self,
+        mem: &GuestMemoryMmap,
+        addr: GuestAddress,
+        len: u32,
+    ) -> Result<(), IoVecError> {
+        // We use get_slice instead of `get_host_address` here in order to have the whole
+        // range of the descriptor chain checked, i.e. [addr, addr + len) is a valid memory
+        // region in the GuestMemoryMmap.
+        let iov_base = mem
+            .get_slice(addr, len as usize)?
+            .ptr_guard_mut()
+            .as_ptr()
+            .cast::<c_void>();
+        self.vecs.push(iovec {
+            iov_base,
+            iov_len: len as size_t,
+        });
+        self.len = self
+            .len
+            .checked_add(len)
+            .ok_or(IoVecError::OverflowedDescriptor)?;
+        Ok(())
+    }
+
     /// Create an `IoVecBuffer` from a `DescriptorChain`
     ///
     /// # Safety
@@ -240,36 +285,79 @@ impl IoVecBufferMut {
 
         let mut next_descriptor = Some(head);
         while let Some(desc) = next_descriptor {
-            if !desc.is_write_only() {
-                return Err(IoVecError::ReadOnlyDescriptor);
-            }
+            if desc.is_indirect() {
+                // We use get_slice instead of `get_host_address` here in order to have the whole
+                // range of the descriptor chain checked, i.e. [addr, addr + len) is a valid memory
+                // region in the GuestMemoryMmap.
+                let indirect_desc_slice = desc
+                    .mem
+                    .get_slice(desc.addr, desc.len as usize)
+                    .map_err(IoVecError::GuestMemory)?;
+
+                // SAFETY:
+                // We checked the slice above. We just transform it into
+                // a slice of Descriptors.
+                let indirect_desc_slice: &[Descriptor] = unsafe {
+                    std::slice::from_raw_parts(
+                        indirect_desc_slice.ptr_guard().as_ptr().cast(),
+                        desc.len as usize / std::mem::size_of::<Descriptor>(),
+                    )
+                };
+
+                for d in indirect_desc_slice.iter() {
+                    if !desc.is_write_only() {
+                        return Err(IoVecError::ReadOnlyDescriptor);
+                    }
+                    if d.is_indirect() {
+                        return Err(IoVecError::NestedIndirectDescriptor);
+                    }
+                    self.add_descriptor(desc.mem, GuestAddress(d.addr), d.len)?;
+                    if !d.has_next() {
+                        break;
+                    }
+                }
+            } else {
+                if !desc.is_write_only() {
+                    return Err(IoVecError::ReadOnlyDescriptor);
+                }
 
-            // We use get_slice instead of `get_host_address` here in order to have the whole
-            // range of the descriptor chain checked, i.e. [addr, addr + len) is a valid memory
-            // region in the GuestMemoryMmap.
-            let slice = desc.mem.get_slice(desc.addr, desc.len as usize)?;
-
-            // We need to mark the area of guest memory that will be mutated through this
-            // IoVecBufferMut as dirty ahead of time, as we loose access to all
-            // vm-memory related information after converting down to iovecs.
-            slice.bitmap().mark_dirty(0, desc.len as usize);
-
-            let iov_base = slice.ptr_guard_mut().as_ptr().cast::<c_void>();
-            self.vecs.push(iovec {
-                iov_base,
-                iov_len: desc.len as size_t,
-            });
-            self.len = self
-                .len
-                .checked_add(desc.len)
-                .ok_or(IoVecError::OverflowedDescriptor)?;
+                self.add_descriptor(desc.mem, desc.addr, desc.len)?;
+            }
 
             next_descriptor = desc.next_descriptor();
         }
 
         Ok(())
     }
 
+    fn add_descriptor(
+        &mut self,
+        mem: &GuestMemoryMmap,
+        addr: GuestAddress,
+        len: u32,
+    ) -> Result<(), IoVecError> {
+        // We use get_slice instead of `get_host_address` here in order to have the whole
+        // range of the descriptor chain checked, i.e. [addr, addr + len) is a valid memory
+        // region in the GuestMemoryMmap.
+        let slice = mem.get_slice(addr, len as usize)?;
+
+        // We need to mark the area of guest memory that will be mutated through this
+        // IoVecBufferMut as dirty ahead of time, as we loose access to all
+        // vm-memory related information after converting down to iovecs.
+        slice.bitmap().mark_dirty(0, len as usize);
+
+        let iov_base = slice.ptr_guard_mut().as_ptr().cast::<c_void>();
+        self.vecs.push(iovec {
+            iov_base,
+            iov_len: len as size_t,
+        });
+        self.len = self
+            .len
+            .checked_add(len)
+            .ok_or(IoVecError::OverflowedDescriptor)?;
+        Ok(())
+    }
+
     /// Create an `IoVecBuffer` from a `DescriptorChain`
     ///
     /// # Safety

src/vmm/src/devices/virtio/queue.rs

@@ -18,6 +18,7 @@ use crate::vstate::memory::{
 
 pub(super) const VIRTQ_DESC_F_NEXT: u16 = 0x1;
 pub(super) const VIRTQ_DESC_F_WRITE: u16 = 0x2;
+pub(super) const VIRTQ_DESC_F_INDIRECT: u16 = 0x4;
 
 /// Max size of virtio queues offered by firecracker's virtio devices.
 pub(super) const FIRECRACKER_MAX_QUEUE_SIZE: u16 = 256;
@@ -43,12 +44,32 @@ pub enum QueueError {
 /// https://docs.oasis-open.org/virtio/virtio/v1.1/csprd01/virtio-v1.1-csprd01.html#x1-430008
 /// 2.6.5 The Virtqueue Descriptor Table
 #[repr(C)]
-#[derive(Default, Clone, Copy)]
-struct Descriptor {
-    addr: u64,
-    len: u32,
-    flags: u16,
-    next: u16,
+#[derive(Debug, Default, Clone, Copy)]
+pub struct Descriptor {
+    pub addr: u64,
+    pub len: u32,
+    pub flags: u16,
+    pub next: u16,
+}
+
+impl Descriptor {
+    /// Gets if this descriptor chain has another descriptor chain linked after it.
+    pub fn has_next(&self) -> bool {
+        self.flags & VIRTQ_DESC_F_NEXT != 0
+    }
+
+    /// If the driver designated this as a write only descriptor.
+    ///
+    /// If this is false, this descriptor is read only.
+    /// Write only means the emulated device can write and the driver can read.
+    pub fn is_write_only(&self) -> bool {
+        self.flags & VIRTQ_DESC_F_WRITE != 0
+    }
+
+    /// If the driver designated this as a indirect descriptor.
+    pub fn is_indirect(&self) -> bool {
+        self.flags & VIRTQ_DESC_F_INDIRECT != 0
+    }
 }
 
 // SAFETY: `Descriptor` is a POD and contains no padding.
@@ -159,6 +180,11 @@ impl<'a, M: GuestMemory> DescriptorChain<'a, M> {
         self.flags & VIRTQ_DESC_F_WRITE != 0
     }
 
+    /// If the driver designated this as a indirect descriptor.
+    pub fn is_indirect(&self) -> bool {
+        self.flags & VIRTQ_DESC_F_INDIRECT != 0
+    }
+
     /// Gets the next descriptor in this descriptor chain, if there is one.
     ///
     /// Note that this is distinct from the next descriptor chain returned by `AvailIter`, which is

src/vmm/src/devices/virtio/vsock/mod.rs

@@ -124,6 +124,8 @@ pub enum VsockError {
     DescChainTooShortForHeader(usize),
     /// The descriptor chain length was greater than the max ([u32::MAX])
     DescChainOverflow,
+    /// Nested indirect descriptor
+    NestedIndirectDescriptor,
     /// The vsock header `len` field holds an invalid value: {0}
     InvalidPktLen(u32),
     /// A data fetch was attempted when no data was available.
@@ -147,6 +149,7 @@ impl From<IoVecError> for VsockError {
             IoVecError::ReadOnlyDescriptor => VsockError::UnwritableDescriptor,
             IoVecError::GuestMemory(err) => VsockError::GuestMemoryMmap(err),
             IoVecError::OverflowedDescriptor => VsockError::DescChainOverflow,
+            IoVecError::NestedIndirectDescriptor => VsockError::NestedIndirectDescriptor,
         }
     }
 }