fix(vsock): restore previous performance with PCI disabled

We noticed that, compared to main, the vsock device is up to 30% slower
when PCI is disabled. This is due to the refactor in 087e185
("fix(vsock): pass correct index when triggering interrupts"), as we're
now sending interrupts as soon as we detect the event, rather than
deferring them and sending only one interrupt at the end.
While with MSI we need to send multiple interrupts, so there is little
difference (still, tests show up to 5% improvement with this change),
with legacy IRQ there's only one interrupt line so we end up sending
multiple back to back interrupts rather than a single one.
This patch reverts to the previous behaviour and uses the newly
introduced `trigger_queues` method to deduplicate interrupts in the case
of IrqTrigger.

Signed-off-by: Riccardo Mancini <[email protected]>

Author: Manciukic

src/vmm/src/devices/virtio/vsock/device.rs

@@ -148,6 +148,16 @@ where
             .map_err(DeviceError::FailedSignalingIrq)
     }
 
+    /// Signal the guest which queues are ready to be consumed
+    pub fn signal_used_queues(&self, used_queues: &[u16]) -> Result<(), DeviceError> {
+        self.device_state
+            .active_state()
+            .expect("Device is not initialized")
+            .interrupt
+            .trigger_queues(used_queues)
+            .map_err(DeviceError::FailedSignalingIrq)
+    }
+
     /// Walk the driver-provided RX queue buffers and attempt to fill them up with any data that we
     /// have pending. Return `true` if descriptors have been added to the used ring, and `false`
     /// otherwise.

src/vmm/src/devices/virtio/vsock/event_handler.rs

@@ -34,6 +34,7 @@ use super::VsockBackend;
 use super::device::{EVQ_INDEX, RXQ_INDEX, TXQ_INDEX, Vsock};
 use crate::devices::virtio::device::VirtioDevice;
 use crate::devices::virtio::queue::InvalidAvailIdx;
+use crate::devices::virtio::vsock::defs::VSOCK_NUM_QUEUES;
 use crate::devices::virtio::vsock::metrics::METRICS;
 use crate::logger::IncMetric;
 
@@ -47,49 +48,50 @@ where
     const PROCESS_EVQ: u32 = 3;
     const PROCESS_NOTIFY_BACKEND: u32 = 4;
 
-    pub fn handle_rxq_event(&mut self, evset: EventSet) {
+    pub fn handle_rxq_event(&mut self, evset: EventSet) -> Vec<u16> {
+        let mut used_queues = Vec::new();
         if evset != EventSet::IN {
             warn!("vsock: rxq unexpected event {:?}", evset);
             METRICS.rx_queue_event_fails.inc();
-            return;
+            return used_queues;
         }
 
         if let Err(err) = self.queue_events[RXQ_INDEX].read() {
             error!("Failed to get vsock rx queue event: {:?}", err);
             METRICS.rx_queue_event_fails.inc();
         } else if self.backend.has_pending_rx() {
             if self.process_rx().unwrap() {
-                self.signal_used_queue(RXQ_INDEX)
-                    .expect("vsock: Could not trigger device interrupt or RX queue");
+                used_queues.push(RXQ_INDEX.try_into().unwrap());
             }
             METRICS.rx_queue_event_count.inc();
         }
+        used_queues
     }
 
-    pub fn handle_txq_event(&mut self, evset: EventSet) {
+    pub fn handle_txq_event(&mut self, evset: EventSet) -> Vec<u16> {
+        let mut used_queues = Vec::new();
         if evset != EventSet::IN {
             warn!("vsock: txq unexpected event {:?}", evset);
             METRICS.tx_queue_event_fails.inc();
-            return;
+            return used_queues;
         }
 
         if let Err(err) = self.queue_events[TXQ_INDEX].read() {
             error!("Failed to get vsock tx queue event: {:?}", err);
             METRICS.tx_queue_event_fails.inc();
         } else {
             if self.process_tx().unwrap() {
-                self.signal_used_queue(TXQ_INDEX)
-                    .expect("vsock: Could not trigger device interrupt or TX queue");
+                used_queues.push(TXQ_INDEX.try_into().unwrap());
             }
             METRICS.tx_queue_event_count.inc();
             // The backend may have queued up responses to the packets we sent during
             // TX queue processing. If that happened, we need to fetch those responses
             // and place them into RX buffers.
             if self.backend.has_pending_rx() && self.process_rx().unwrap() {
-                self.signal_used_queue(RXQ_INDEX)
-                    .expect("vsock: Could not trigger device interrupt or RX queue");
+                used_queues.push(RXQ_INDEX.try_into().unwrap());
             }
         }
+        used_queues
     }
 
     pub fn handle_evq_event(&mut self, evset: EventSet) {
@@ -106,23 +108,22 @@ where
     }
 
     /// Notify backend of new events.
-    pub fn notify_backend(&mut self, evset: EventSet) -> Result<(), InvalidAvailIdx> {
+    pub fn notify_backend(&mut self, evset: EventSet) -> Result<Vec<u16>, InvalidAvailIdx> {
+        let mut used_queues = Vec::new();
         self.backend.notify(evset);
         // After the backend has been kicked, it might've freed up some resources, so we
         // can attempt to send it more data to process.
         // In particular, if `self.backend.send_pkt()` halted the TX queue processing (by
         // returning an error) at some point in the past, now is the time to try walking the
         // TX queue again.
         if self.process_tx()? {
-            self.signal_used_queue(TXQ_INDEX)
-                .expect("vsock: Could not trigger device interrupt or TX queue");
+            used_queues.push(TXQ_INDEX.try_into().unwrap());
         }
         if self.backend.has_pending_rx() && self.process_rx()? {
-            self.signal_used_queue(RXQ_INDEX)
-                .expect("vsock: Could not trigger device interrupt or RX queue");
+            used_queues.push(RXQ_INDEX.try_into().unwrap())
         }
 
-        Ok(())
+        Ok(used_queues)
     }
 
     fn register_runtime_events(&self, ops: &mut EventOps) {
@@ -190,14 +191,25 @@ where
         let evset = event.event_set();
 
         if self.is_activated() {
-            match source {
-                Self::PROCESS_ACTIVATE => self.handle_activate_event(ops),
+            let used_queues = match source {
+                Self::PROCESS_ACTIVATE => {
+                    self.handle_activate_event(ops);
+                    Vec::new()
+                }
                 Self::PROCESS_RXQ => self.handle_rxq_event(evset),
                 Self::PROCESS_TXQ => self.handle_txq_event(evset),
-                Self::PROCESS_EVQ => self.handle_evq_event(evset),
+                Self::PROCESS_EVQ => {
+                    self.handle_evq_event(evset);
+                    Vec::new()
+                }
                 Self::PROCESS_NOTIFY_BACKEND => self.notify_backend(evset).unwrap(),
-                _ => warn!("Unexpected vsock event received: {:?}", source),
+                _ => {
+                    warn!("Unexpected vsock event received: {:?}", source);
+                    Vec::new()
+                }
             };
+            self.signal_used_queues(&used_queues)
+                .expect("vsock: Could not trigger device interrupt");
         } else {
             warn!(
                 "Vsock: The device is not yet activated. Spurious event received: {:?}",