Generify worker threads on x86

Rather than spawning a new worker thread every time some functionality
needs it, which results in it's own Message type, create something
generic that can be used in one thread.

Signed-off-by: Jake Correnti <[email protected]>

Author: jakecorrenti

Cargo.lock

@@ -5,6 +5,7 @@ use kvm_bindings::{
 use kvm_ioctls::{Error, VmFd};
 
 use utils::eventfd::EventFd;
+use utils::worker_message::WorkerMessage;
 
 use crate::bus::BusDevice;
 use crate::legacy::irqchip::IrqChipT;
@@ -67,12 +68,6 @@ const IOAPIC_TRIGGER_EDGE: u64 = 0;
 /// 7:0 Interrupt Vector (INTVEC) (RW)
 type RedirectionTableEntry = u64;
 
-#[derive(Debug)]
-pub enum IrqWorkerMessage {
-    GsiRoute(Vec<kvm_irq_routing_entry>),
-    IrqLine(u32, bool),
-}
-
 #[derive(Debug, Default)]
 pub struct IoApicEntryInfo {
     masked: u8,
@@ -101,14 +96,14 @@ pub struct IoApic {
     version: u8,
     irq_eoi: [i32; IOAPIC_NUM_PINS],
     irq_routes: Vec<kvm_irq_routing_entry>,
-    irq_sender: crossbeam_channel::Sender<(IrqWorkerMessage, EventFd)>,
+    irq_sender: crossbeam_channel::Sender<(WorkerMessage, EventFd)>,
     event_fd: EventFd,
 }
 
 impl IoApic {
     pub fn new(
         vm: &VmFd,
-        _irq_sender: crossbeam_channel::Sender<(IrqWorkerMessage, EventFd)>,
+        _irq_sender: crossbeam_channel::Sender<(WorkerMessage, EventFd)>,
     ) -> Result<Self, Error> {
         let mut cap = kvm_enable_cap {
             cap: KVM_CAP_SPLIT_IRQCHIP,
@@ -180,7 +175,7 @@ impl IoApic {
         }
     }
 
-    fn send_irq_worker_message(&self, msg: IrqWorkerMessage) {
+    fn send_irq_worker_message(&self, msg: WorkerMessage) {
         self.irq_sender
             .send((msg, self.event_fd.try_clone().unwrap()))
             .unwrap();
@@ -254,7 +249,7 @@ impl IoApic {
             }
         }
 
-        self.send_irq_worker_message(IrqWorkerMessage::GsiRoute(self.irq_routes.clone()));
+        self.send_irq_worker_message(WorkerMessage::GsiRoute(self.irq_routes.clone()));
     }
 
     fn service(&mut self) {
@@ -279,10 +274,10 @@ impl IoApic {
                     }
 
                     if info.trig_mode as u64 == IOAPIC_TRIGGER_EDGE {
-                        self.send_irq_worker_message(IrqWorkerMessage::IrqLine(i as u32, true));
-                        self.send_irq_worker_message(IrqWorkerMessage::IrqLine(i as u32, false));
+                        self.send_irq_worker_message(WorkerMessage::IrqLine(i as u32, true));
+                        self.send_irq_worker_message(WorkerMessage::IrqLine(i as u32, false));
                     } else {
-                        self.send_irq_worker_message(IrqWorkerMessage::IrqLine(i as u32, true));
+                        self.send_irq_worker_message(WorkerMessage::IrqLine(i as u32, true));
                     }
                 }
             }

src/devices/src/legacy/ioapic.rs

@@ -42,7 +42,7 @@ pub use self::hvfgicv3::HvfGicV3;
 pub use self::i8042::Error as I8042DeviceError;
 pub use self::i8042::I8042Device;
 #[cfg(all(target_os = "linux", target_arch = "x86_64"))]
-pub use self::ioapic::{IoApic, IrqWorkerMessage};
+pub use self::ioapic::IoApic;
 pub use self::irqchip::{IrqChip, IrqChipDevice, IrqChipT};
 #[cfg(all(target_os = "linux", target_arch = "aarch64"))]
 pub use self::kvmgicv3::KvmGicV3;

src/devices/src/legacy/mod.rs

@@ -22,7 +22,6 @@ use std::sync::atomic::{AtomicI32, Ordering};
 use std::sync::LazyLock;
 use std::sync::Mutex;
 
-#[cfg(any(target_os = "macos", feature = "tee"))]
 use crossbeam_channel::unbounded;
 #[cfg(feature = "blk")]
 use devices::virtio::block::ImageType;
@@ -1510,29 +1509,29 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
     #[cfg(target_os = "macos")]
     let (sender, receiver) = unbounded();
 
-    #[cfg(target_arch = "x86_64")]
-    let (irq_sender, irq_receiver) = crossbeam_channel::unbounded();
     #[cfg(feature = "tee")]
     let (pm_sender, pm_receiver) = unbounded();
     #[cfg(feature = "tee")]
     let pm_efd =
         EventFd::new(EFD_SEMAPHORE).expect("unable to create TEE memory properties eventfd");
+    #[cfg(target_arch = "x86_64")]
+    let (sender, receiver) = unbounded();
 
     let _vmm = match vmm::builder::build_microvm(
         &ctx_cfg.vmr,
         &mut event_manager,
         ctx_cfg.shutdown_efd,
         #[cfg(target_os = "macos")]
         sender,
-        #[cfg(target_arch = "x86_64")]
-        irq_sender,
         #[cfg(feature = "tee")]
         (
             pm_sender,
             pm_efd
                 .try_clone()
                 .expect("unable to clone TEE memory properties eventfd"),
         ),
+        #[cfg(target_arch = "x86_64")]
+        sender,
     ) {
         Ok(vmm) => vmm,
         Err(e) => {
@@ -1544,9 +1543,6 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
     #[cfg(any(target_os = "macos", feature = "tee"))]
     let mapper_vmm = _vmm.clone();
 
-    #[cfg(target_arch = "x86_64")]
-    let irq_vmm = _vmm.clone();
-
     #[cfg(target_os = "macos")]
     if ctx_cfg.gpu_virgl_flags.is_some() {
         std::thread::Builder::new()
@@ -1569,50 +1565,7 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
 
     #[cfg(target_arch = "x86_64")]
     if ctx_cfg.vmr.split_irqchip {
-        std::thread::Builder::new()
-            .name("irq worker".into())
-            .spawn(move || loop {
-                match irq_receiver.recv() {
-                    Err(e) => error!("Error in receiver: {:?}", e),
-                    Ok((message, evt_fd)) => match message {
-                        devices::legacy::IrqWorkerMessage::GsiRoute(entries) => {
-                            let mut irq_routing = utils::sized_vec::vec_with_array_field::<
-                                kvm_bindings::kvm_irq_routing,
-                                kvm_bindings::kvm_irq_routing_entry,
-                            >(entries.len());
-                            irq_routing[0].nr = entries.len() as u32;
-                            irq_routing[0].flags = 0;
-
-                            unsafe {
-                                let entries_slice: &mut [kvm_bindings::kvm_irq_routing_entry] =
-                                    irq_routing[0].entries.as_mut_slice(entries.len());
-                                entries_slice.copy_from_slice(&entries);
-                            }
-
-                            irq_vmm
-                                .lock()
-                                .unwrap()
-                                .kvm_vm()
-                                .fd()
-                                .set_gsi_routing(&irq_routing[0])
-                                .unwrap();
-
-                            evt_fd.write(1).unwrap();
-                        }
-                        devices::legacy::IrqWorkerMessage::IrqLine(irq, active) => {
-                            irq_vmm
-                                .lock()
-                                .unwrap()
-                                .kvm_vm()
-                                .fd()
-                                .set_irq_line(irq, active)
-                                .unwrap();
-                            evt_fd.write(1).unwrap();
-                        }
-                    },
-                }
-            })
-            .unwrap();
+        vmm::worker::start_worker_thread(_vmm.clone(), receiver.clone()).unwrap();
     }
 
     #[cfg(feature = "tee")]

src/libkrun/src/lib.rs

@@ -10,3 +10,6 @@ env_logger = "0.9.0"
 libc = ">=0.2.85"
 log = "0.4.0"
 vmm-sys-util = "0.12.1"
+
+[target.'cfg(target_os = "linux")'.dependencies]
+kvm-bindings = { version = ">=0.10", features = ["fam-wrappers"] }

src/utils/Cargo.toml

@@ -23,3 +23,4 @@ pub mod sized_vec;
 pub mod sm;
 pub mod syscall;
 pub mod time;
+pub mod worker_message;

src/utils/src/lib.rs

@@ -0,0 +1,7 @@
+#[derive(Debug)]
+pub enum WorkerMessage {
+    #[cfg(target_arch = "x86_64")]
+    GsiRoute(Vec<kvm_bindings::kvm_irq_routing_entry>),
+    #[cfg(target_arch = "x86_64")]
+    IrqLine(u32, bool),
+}

src/utils/src/worker_message.rs

@@ -5,10 +5,7 @@
 
 #[cfg(target_os = "macos")]
 use crossbeam_channel::unbounded;
-
-#[cfg(any(target_os = "macos", feature = "tee"))]
 use crossbeam_channel::Sender;
-
 use kernel::cmdline::Cmdline;
 #[cfg(target_os = "macos")]
 use std::collections::HashMap;
@@ -82,6 +79,7 @@ use linux_loader::loader::{self, KernelLoader};
 use nix::unistd::isatty;
 use polly::event_manager::{Error as EventManagerError, EventManager};
 use utils::eventfd::EventFd;
+use utils::worker_message::WorkerMessage;
 #[cfg(all(target_arch = "x86_64", not(feature = "efi"), not(feature = "tee")))]
 use vm_memory::mmap::MmapRegion;
 #[cfg(not(feature = "tee"))]
@@ -516,11 +514,8 @@ pub fn build_microvm(
     event_manager: &mut EventManager,
     _shutdown_efd: Option<EventFd>,
     #[cfg(target_os = "macos")] _map_sender: Sender<MemoryMapping>,
-    #[cfg(target_arch = "x86_64")] irq_sender: crossbeam_channel::Sender<(
-        devices::legacy::IrqWorkerMessage,
-        EventFd,
-    )>,
     #[cfg(feature = "tee")] pm_sender: (Sender<MemoryProperties>, EventFd),
+    #[cfg(target_arch = "x86_64")] _sender: Sender<(WorkerMessage, EventFd)>,
 ) -> std::result::Result<Arc<Mutex<Vmm>>, StartMicrovmError> {
     let payload = choose_payload(vm_resources)?;
 
@@ -675,7 +670,7 @@ pub fn build_microvm(
     #[cfg(target_arch = "x86_64")]
     {
         let ioapic: Box<dyn IrqChipT> = if vm_resources.split_irqchip {
-            Box::new(IoApic::new(vm.fd(), irq_sender).map_err(StartMicrovmError::CreateKvmIrqChip)?)
+            Box::new(IoApic::new(vm.fd(), _sender).map_err(StartMicrovmError::CreateKvmIrqChip)?)
         } else {
             Box::new(KvmIoapic::new(vm.fd()).map_err(StartMicrovmError::CreateKvmIrqChip)?)
         };

src/vmm/src/builder.rs

@@ -31,6 +31,7 @@ use crate::linux::vstate;
 #[cfg(target_os = "macos")]
 mod macos;
 mod terminal;
+pub mod worker;
 
 #[cfg(target_os = "macos")]
 pub use hvf::MemoryMapping;

src/vmm/src/lib.rs

@@ -0,0 +1,58 @@
+use std::io;
+use std::sync::{Arc, Mutex};
+
+use utils::eventfd::EventFd;
+use utils::worker_message::WorkerMessage;
+
+use crossbeam_channel::Receiver;
+
+pub fn start_worker_thread(
+    vmm: Arc<Mutex<super::Vmm>>,
+    #[cfg(not(target_os = "macos"))] receiver: Receiver<(WorkerMessage, EventFd)>,
+) -> io::Result<()> {
+    std::thread::Builder::new()
+        .name("vmm worker".into())
+        .spawn(move || loop {
+            match receiver.recv() {
+                Err(e) => error!("error receiving message from vmm worker thread: {:?}", e),
+                #[cfg(target_os = "linux")]
+                Ok((message, evt_fd)) => vmm.lock().unwrap().match_worker_message(message, evt_fd),
+            }
+        })?;
+    Ok(())
+}
+
+impl super::Vmm {
+    fn match_worker_message(
+        &self,
+        msg: WorkerMessage,
+        #[cfg(target_os = "linux")] evt_fd: EventFd,
+    ) {
+        match msg {
+            #[cfg(target_arch = "x86_64")]
+            WorkerMessage::GsiRoute(entries) => {
+                let mut irq_routing = utils::sized_vec::vec_with_array_field::<
+                    kvm_bindings::kvm_irq_routing,
+                    kvm_bindings::kvm_irq_routing_entry,
+                >(entries.len());
+                irq_routing[0].nr = entries.len() as u32;
+                irq_routing[0].flags = 0;
+
+                unsafe {
+                    let entries_slice: &mut [kvm_bindings::kvm_irq_routing_entry] =
+                        irq_routing[0].entries.as_mut_slice(entries.len());
+                    entries_slice.copy_from_slice(&entries);
+                }
+
+                self.vm.fd().set_gsi_routing(&irq_routing[0]).unwrap();
+
+                evt_fd.write(1).unwrap();
+            }
+            #[cfg(target_arch = "x86_64")]
+            WorkerMessage::IrqLine(irq, active) => {
+                self.vm.fd().set_irq_line(irq, active).unwrap();
+                evt_fd.write(1).unwrap();
+            }
+        }
+    }
+}