feat(net): use readv for reading frames from TAP device

Right now, we are performing two copies for writing a frame from the TAP
device into guest memory:
- read the frame into an internal array held by the Net device
- copy that array into a buffers of a DescriptorChain.

In order to avoid this double copy we can use the readv system call to
read directly from the TAP device into the buffers described by
DescriptorChain.

The main challenge with this is that DescriptorChain objects describe
memory that is at least 65562 bytes long when guest TSO4, TSO6 or UFO
are enabled or 1526 otherwise and parsing the chain includes overhead
which we pay even if the frame we are receiving is much smaller than
these sizes.

PR #4748 reduced
the overheads involved with parsing DescriptorChain objects. To further
avoid this overhead, move the parsing of DescriptorChain objects out of
the hot path of process_rx() where we are actually receiving a frame
into process_rx_queue_event() where we get the notification that the
guest added new buffers for network RX.

Co-authored-by: Babis Chalios <[email protected]>
Signed-off-by: Egor Lazarchuk <[email protected]>

Author: ShadowCurse

resources/seccomp/aarch64-unknown-linux-musl.json

@@ -32,6 +32,10 @@
                 "syscall": "writev",
                 "comment": "Used by the VirtIO net device to write to tap"
             },
+            {
+                "syscall": "readv",
+                "comment": "Used by the VirtIO net device to read from tap"
+            },
             {
                 "syscall": "fsync"
             },

resources/seccomp/x86_64-unknown-linux-musl.json

@@ -32,6 +32,10 @@
                 "syscall": "writev",
                 "comment": "Used by the VirtIO net device to write to tap"
             },
+            {
+                "syscall": "readv",
+                "comment": "Used by the VirtIO net device to read from tap"
+            },
             {
                 "syscall": "fsync"
             },

src/vmm/benches/queue.rs

@@ -10,50 +10,9 @@ use std::num::Wrapping;
 
 use criterion::{criterion_group, criterion_main, Criterion};
 use vm_memory::GuestAddress;
-use vmm::devices::virtio::queue::{VIRTQ_DESC_F_NEXT, VIRTQ_DESC_F_WRITE};
-use vmm::devices::virtio::test_utils::VirtQueue;
+use vmm::devices::virtio::test_utils::{set_dtable_many_chains, set_dtable_one_chain, VirtQueue};
 use vmm::test_utils::single_region_mem;
 
-/// Create one chain with n descriptors
-/// Descriptor buffers will leave at the offset of 2048 bytes
-/// to leave some room for queue objects.
-/// We don't really care about sizes of descriptors,
-/// so pick 1024.
-fn set_dtable_one_chain(rxq: &VirtQueue, n: usize) {
-    let desc_size = 1024;
-    for i in 0..n {
-        rxq.dtable[i].set(
-            (2048 + desc_size * i) as u64,
-            desc_size as u32,
-            VIRTQ_DESC_F_WRITE | VIRTQ_DESC_F_NEXT,
-            (i + 1) as u16,
-        );
-    }
-    rxq.dtable[n - 1].flags.set(VIRTQ_DESC_F_WRITE);
-    rxq.dtable[n - 1].next.set(0);
-    rxq.avail.ring[0].set(0);
-    rxq.avail.idx.set(n as u16);
-}
-
-/// Create n chains with 1 descriptors each
-/// Descriptor buffers will leave at the offset of 2048 bytes
-/// to leave some room for queue objects.
-/// We don't really care about sizes of descriptors,
-/// so pick 1024.
-fn set_dtable_many_chains(rxq: &VirtQueue, n: usize) {
-    let desc_size = 1024;
-    for i in 0..n {
-        rxq.dtable[i].set(
-            (2048 + desc_size * i) as u64,
-            desc_size as u32,
-            VIRTQ_DESC_F_WRITE,
-            0,
-        );
-        rxq.avail.ring[i].set(i as u16);
-    }
-    rxq.avail.idx.set(n as u16);
-}
-
 pub fn queue_benchmark(c: &mut Criterion) {
     let mem = single_region_mem(65562);
     let rxq = VirtQueue::new(GuestAddress(0), &mem, 256);