MSI-X support

Cargo.toml

@@ -208,6 +208,7 @@ exclude = [
 x86_64 = { git = "https://github.com/rust-osdev/x86_64.git" }
 # FIXME: remove once merged: https://github.com/rcore-os/trapframe-rs/pull/16
 trapframe = { git = "https://github.com/hermit-os/trapframe-rs", branch = "global_asm" }
+pci_types = { git = "https://github.com/rust-osdev/pci_types.git" }
 safe-mmio = { git = "https://github.com/hermit-os/safe-mmio", branch = "be" }
 
 [profile.profiling]

src/arch/x86_64/kernel/apic.rs

@@ -675,6 +675,10 @@ fn calibrate_timer() {
 
 fn __set_oneshot_timer(wakeup_time: Option<u64>) {
 	if let Some(wt) = wakeup_time {
+		if wt < processor::get_timer_ticks() {
+			error!("Wakeup time is in the past.");
+			return;
+		}
 		if processor::supports_tsc_deadline() {
 			// wt is the absolute wakeup time in microseconds based on processor::get_timer_ticks.
 			// We can simply multiply it by the processor frequency to get the absolute Time-Stamp Counter deadline

src/drivers/net/virtio/mod.rs

@@ -18,6 +18,7 @@ use alloc::vec::Vec;
 use core::mem::{ManuallyDrop, MaybeUninit, transmute};
 use core::str::FromStr;
 
+use bit_field::BitField;
 use smallvec::SmallVec;
 use smoltcp::phy::{Checksum, ChecksumCapabilities, DeviceCapabilities};
 use smoltcp::wire::{ETHERNET_HEADER_LEN, EthernetFrame, Ipv4Packet, Ipv6Packet};
@@ -237,6 +238,8 @@ pub(crate) struct VirtioNetDriver<T = Init> {
 	pub(super) com_cfg: ComCfg,
 	pub(super) isr_stat: IsrStatus,
 	pub(super) notif_cfg: NotifCfg,
+	#[cfg(feature = "pci")]
+	pub(super) msix_table: Option<VolatileRef<'static, [[u32; 4]]>>,
 
 	pub(super) inner: T,
 
@@ -789,11 +792,31 @@ impl VirtioNetDriver<Uninit> {
 		}
 		debug!("{:?}", self.checksums);
 
+		#[cfg(feature = "pci")]
+		if let Some(msix_table) = self.msix_table.as_mut() {
+			unsafe {
+				msix_table
+					.as_mut_ptr()
+					.map(|table| table.get_unchecked_mut(0))
+					.update(|[mut addr_low, addr_high, mut data, mut control]| {
+						[
+							*addr_low.set_bits(20..32, 0xfee),
+							addr_high,
+							*data.set_bits(0..8, u32::from(constants::MSIX_VECTOR) + 32),
+							*control.set_bit(0, false),
+						]
+					});
+			}
+			self.com_cfg.select_vq(0).unwrap().set_msix_table_index(0);
+		};
+
 		Ok(VirtioNetDriver {
 			dev_cfg: self.dev_cfg,
 			com_cfg: self.com_cfg,
 			isr_stat: self.isr_stat,
 			notif_cfg: self.notif_cfg,
+			#[cfg(feature = "pci")]
+			msix_table: self.msix_table,
 			inner,
 			num_vqs: self.num_vqs,
 			irq: self.irq,
@@ -975,6 +998,7 @@ pub mod constants {
 	// Configuration constants
 	pub const MAX_NUM_VQ: u16 = 2;
 	pub(super) const BUFF_PER_PACKET: u16 = 2;
+	pub(crate) const MSIX_VECTOR: u8 = 112;
 }
 
 /// Error module of virtios network driver. Containing the (VirtioNetError)[VirtioNetError]

src/drivers/net/virtio/pci.rs

@@ -4,7 +4,7 @@ use volatile::VolatileRef;
 
 use super::{Init, Uninit};
 use crate::arch::pci::PciConfigRegion;
-use crate::drivers::net::virtio::{NetDevCfg, VirtioNetDriver};
+use crate::drivers::net::virtio::{NetDevCfg, VirtioNetDriver, constants};
 use crate::drivers::pci::PciDevice;
 use crate::drivers::virtio::error::{self, VirtioError};
 use crate::drivers::virtio::transport::pci;
@@ -36,6 +36,7 @@ impl VirtioNetDriver<Uninit> {
 			notif_cfg,
 			isr_cfg,
 			dev_cfg_list,
+			msix_table,
 			..
 		} = caps_coll;
 
@@ -49,9 +50,10 @@ impl VirtioNetDriver<Uninit> {
 			com_cfg,
 			isr_stat: isr_cfg,
 			notif_cfg,
+			msix_table,
 			inner: Uninit,
 			num_vqs: 0,
-			irq: device.get_irq().unwrap(),
+			irq: constants::MSIX_VECTOR,
 			checksums: ChecksumCapabilities::default(),
 		})
 	}

src/drivers/pci.rs

@@ -451,8 +451,11 @@ pub(crate) fn get_interrupt_handlers() -> HashMap<InterruptLine, InterruptHandle
 
 	#[cfg(any(feature = "rtl8139", feature = "virtio-net",))]
 	if let Some(device) = NETWORK_DEVICE.lock().as_ref() {
+		let irq = device.get_interrupt_number();
+		crate::arch::interrupts::add_irq_name(irq, "virtio");
+		info!("Virtio interrupt handler at line {irq}");
 		handlers
-			.entry(device.get_interrupt_number())
+			.entry(irq)
 			.or_default()
 			.push_back(crate::executor::network::network_handler);
 	}

src/drivers/virtio/transport/pci.rs

@@ -186,6 +186,7 @@ pub struct UniCapsColl {
 	pub(crate) isr_cfg: IsrStatus,
 	pub(crate) sh_mem_cfg_list: Vec<ShMemCfg>,
 	pub(crate) dev_cfg_list: Vec<PciCap>,
+	pub(crate) msix_table: Option<VolatileRef<'static, [[u32; 4]]>>,
 }
 /// Wraps a [`CommonCfg`] in order to preserve
 /// the original structure.
@@ -258,6 +259,14 @@ impl VqCfgHandler<'_> {
 			.write(addr.as_u64().into());
 	}
 
+	pub fn set_msix_table_index(&mut self, index: u16) {
+		self.select_queue();
+		self.raw
+			.as_mut_ptr()
+			.queue_msix_vector()
+			.write(index.into());
+	}
+
 	pub fn notif_off(&mut self) -> u16 {
 		self.select_queue();
 		self.raw.as_mut_ptr().queue_notify_off().read().to_ne()
@@ -668,43 +677,6 @@ impl PciBar {
 	}
 }
 
-/// Reads all PCI capabilities, starting at the capabilities list pointer from the
-/// PCI device.
-///
-/// Returns ONLY Virtio specific capabilities, which allow to locate the actual capability
-/// structures inside the memory areas, indicated by the BaseAddressRegisters (BAR's).
-fn read_caps(device: &PciDevice<PciConfigRegion>) -> Result<Vec<PciCap>, PciError> {
-	let device_id = device.device_id();
-
-	let capabilities = device
-		.capabilities()
-		.unwrap()
-		.filter_map(|capability| match capability {
-			PciCapability::Vendor(capability) => Some(capability),
-			_ => None,
-		})
-		.map(|addr| CapData::read(addr, device.access()).unwrap())
-		.filter(|cap| cap.cfg_type != CapCfgType::Pci)
-		.flat_map(|cap| {
-			let slot = cap.bar;
-			device
-				.memory_map_bar(slot, true)
-				.map(|(addr, size)| PciCap {
-					bar: VirtioPciBar::new(slot, addr.as_u64(), size.try_into().unwrap()),
-					dev_id: device_id,
-					cap,
-				})
-		})
-		.collect::<Vec<_>>();
-
-	if capabilities.is_empty() {
-		error!("No virtio capability found for device {device_id:x}");
-		Err(PciError::NoVirtioCaps(device_id))
-	} else {
-		Ok(capabilities)
-	}
-}
-
 pub(crate) fn map_caps(device: &PciDevice<PciConfigRegion>) -> Result<UniCapsColl, VirtioError> {
 	let device_id = device.device_id();
 
@@ -714,64 +686,96 @@ pub(crate) fn map_caps(device: &PciDevice<PciConfigRegion>) -> Result<UniCapsCol
 		return Err(VirtioError::FromPci(PciError::NoCapPtr(device_id)));
 	}
 
-	// Get list of PciCaps pointing to capabilities
-	let cap_list = match read_caps(device) {
-		Ok(list) => list,
-		Err(pci_error) => return Err(VirtioError::FromPci(pci_error)),
-	};
-
 	let mut com_cfg = None;
 	let mut notif_cfg = None;
 	let mut isr_cfg = None;
 	let mut sh_mem_cfg_list = Vec::new();
 	let mut dev_cfg_list = Vec::new();
-	// Map Caps in virtual memory
-	for pci_cap in cap_list {
-		match pci_cap.cap.cfg_type {
-			CapCfgType::Common => {
-				if com_cfg.is_none() {
-					match pci_cap.map_common_cfg() {
-						Some(cap) => com_cfg = Some(ComCfg::new(cap)),
-						None => error!(
-							"Common config capability of device {device_id:x} could not be mapped!"
-						),
-					}
+	let mut msix_table = None;
+
+	// Reads all PCI capabilities, starting at the capabilities list pointer from the
+	// PCI device.
+	//
+	// Maps ONLY Virtio specific capabilities and the MSI-X capability , which allow to locate the actual capability
+	// structures inside the memory areas, indicated by the BaseAddressRegisters (BAR's).
+	for capability in device.capabilities().unwrap() {
+		match capability {
+			PciCapability::Vendor(addr) => {
+				let cap = CapData::read(addr, device.access()).unwrap();
+				if cap.cfg_type == CapCfgType::Pci {
+					continue;
 				}
-			}
-			CapCfgType::Notify => {
-				if notif_cfg.is_none() {
-					match NotifCfg::new(&pci_cap) {
-						Some(notif) => notif_cfg = Some(notif),
-						None => error!(
-							"Notification config capability of device {device_id:x} could not be used!"
-						),
+				let slot = cap.bar;
+				let Some((addr, size)) = device.memory_map_bar(slot, true) else {
+					continue;
+				};
+				let pci_cap = PciCap {
+					bar: VirtioPciBar::new(slot, addr.as_u64(), size.try_into().unwrap()),
+					dev_id: device_id,
+					cap,
+				};
+				match pci_cap.cap.cfg_type {
+					CapCfgType::Common => {
+						if com_cfg.is_none() {
+							match pci_cap.map_common_cfg() {
+								Some(cap) => com_cfg = Some(ComCfg::new(cap)),
+								None => error!(
+									"Common config capability of device {device_id:x} could not be mapped!"
+								),
+							}
+						}
 					}
-				}
-			}
-			CapCfgType::Isr => {
-				if isr_cfg.is_none() {
-					match pci_cap.map_isr_status() {
-						Some(isr_stat) => isr_cfg = Some(IsrStatus::new(isr_stat)),
-						None => error!(
-							"ISR status config capability of device {device_id:x} could not be used!"
-						),
+					CapCfgType::Notify => {
+						if notif_cfg.is_none() {
+							match NotifCfg::new(&pci_cap) {
+								Some(notif) => notif_cfg = Some(notif),
+								None => error!(
+									"Notification config capability of device {device_id:x} could not be used!"
+								),
+							}
+						}
+					}
+					CapCfgType::Isr => {
+						if isr_cfg.is_none() {
+							match pci_cap.map_isr_status() {
+								Some(isr_stat) => isr_cfg = Some(IsrStatus::new(isr_stat)),
+								None => error!(
+									"ISR status config capability of device {device_id:x} could not be used!"
+								),
+							}
+						}
 					}
+					CapCfgType::SharedMemory => match ShMemCfg::new(&pci_cap) {
+						Some(sh_mem) => sh_mem_cfg_list.push(sh_mem),
+						None => {
+							let cap_id = pci_cap.cap.id;
+							error!(
+								"Shared Memory config capability with id {cap_id} of device {device_id:x} could not be used!"
+							);
+						}
+					},
+					CapCfgType::Device => dev_cfg_list.push(pci_cap),
+					_ => continue,
 				}
 			}
-			CapCfgType::SharedMemory => match ShMemCfg::new(&pci_cap) {
-				Some(sh_mem) => sh_mem_cfg_list.push(sh_mem),
-				None => {
-					let cap_id = pci_cap.cap.id;
-					error!(
-						"Shared Memory config capability with id {cap_id} of device {device_id:x} could not be used!"
-					);
-				}
-			},
-			CapCfgType::Device => dev_cfg_list.push(pci_cap),
-
-			// PCI's configuration space is allowed to hold other structures, which are not virtio specific and are therefore ignored
-			// in the following
-			_ => continue,
+			PciCapability::MsiX(mut msix_capability) => {
+				msix_capability.set_enabled(true, device.access());
+				let (base_addr, _) = device
+					.memory_map_bar(msix_capability.table_bar(), true)
+					.unwrap();
+
+				let table_ptr = ptr::with_exposed_provenance_mut(
+					base_addr.as_usize() + usize::try_from(msix_capability.table_offset()).unwrap(),
+				);
+				msix_table = Some(VolatileRef::from_mut_ref(unsafe {
+					&mut *(ptr::slice_from_raw_parts_mut(
+						table_ptr,
+						msix_capability.table_size().into(),
+					))
+				}));
+			}
+			// PCI's configuration space is allowed to hold other structures, which are not useful for us and are therefore ignored.
+			_ => {}
 		}
 	}
 
@@ -781,6 +785,7 @@ pub(crate) fn map_caps(device: &PciDevice<PciConfigRegion>) -> Result<UniCapsCol
 		isr_cfg: isr_cfg.ok_or(VirtioError::NoIsrCfg(device_id))?,
 		sh_mem_cfg_list,
 		dev_cfg_list,
+		msix_table,
 	})
 }
 
@@ -814,11 +819,6 @@ pub(crate) fn init_device(
 		virtio::Id::Net => match VirtioNetDriver::init(device) {
 			Ok(virt_net_drv) => {
 				info!("Virtio network driver initialized.");
-
-				let irq = device.get_irq().unwrap();
-				crate::arch::interrupts::add_irq_name(irq, "virtio");
-				info!("Virtio interrupt handler at line {irq}");
-
 				Ok(VirtioDriver::Network(virt_net_drv))
 			}
 			Err(virtio_error) => {