pci: add support for PCIe segment

Add a PCIe segment which includes a single PCIe root port and a bus.
At the moment, we always enabled it.

things left to do:
* enable it only if `--pci` is passed as a command line argument
* add a test that checks the presence of the root port in the guest
  (lspci)
* ensure it works for Aarch64.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

Cargo.lock

@@ -449,7 +449,7 @@ fn main_exec() -> Result<(), MainError> {
 /// the default the jailer would set).
 ///
 /// We do this resizing because the kernel default is 64, with a reallocation happening whenever
-/// the tabel fills up. This was happening for some larger microVMs, and reallocating the
+/// the table fills up. This was happening for some larger microVMs, and reallocating the
 /// fdtable while a lot of file descriptors are active (due to being eventfds/timerfds registered
 /// to epoll) incurs a penalty of 30ms-70ms on the snapshot restore path.
 fn resize_fdtable() -> Result<(), ResizeFdTableError> {

src/firecracker/src/main.rs

@@ -14,7 +14,7 @@ libc = "0.2.172"
 log = "0.4.27"
 serde = { version = "1.0.219", features = ["derive"] }
 thiserror = "2.0.12"
-vm-allocator = "0.1.1"
+vm-allocator = { git = "https://github.com/rust-vmm/vm-allocator" }
 vm-device = { path = "../vm-device" }
 vm-memory = { version = "0.16.1", features = [
   "backend-mmap",

src/pci/Cargo.toml

@@ -44,7 +44,7 @@ userfaultfd = "0.8.1"
 utils = { path = "../utils" }
 uuid = "1.16.0"
 vhost = { version = "0.13.0", features = ["vhost-user-frontend"] }
-vm-allocator = { path = "../../../rust-vmm/vm-allocator" }
+vm-allocator = { git = "https://github.com/rust-vmm/vm-allocator" }
 vm-device = { path = "../vm-device" }
 vm-memory = { version = "0.16.1", features = [
   "backend-mmap",

src/vmm/Cargo.toml

@@ -90,6 +90,10 @@ impl AcpiTableWriter<'_> {
             .acpi_devices
             .append_aml_bytes(&mut dsdt_data)?;
 
+        if let Some(pci_segment) = &device_manager.pci_devices.pci_segment {
+            pci_segment.append_aml_bytes(&mut dsdt_data)?;
+        }
+
         // Architecture specific DSDT data
         setup_arch_dsdt(&mut dsdt_data)?;
 

src/vmm/src/acpi/mod.rs

@@ -47,7 +47,9 @@ pub use crate::arch::x86_64::{
     layout::IOAPIC_ADDR, layout::IRQ_BASE, layout::IRQ_MAX, layout::MEM_32BIT_DEVICES_SIZE,
     layout::MEM_32BIT_DEVICES_START, layout::MEM_64BIT_DEVICES_SIZE,
     layout::MEM_64BIT_DEVICES_START, layout::MMIO32_MEM_SIZE, layout::MMIO32_MEM_START,
-    layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START, load_kernel,
+    layout::PCI_MMCONFIG_SIZE, layout::PCI_MMCONFIG_START,
+    layout::PCI_MMIO_CONFIG_SIZE_PER_SEGMENT, layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START,
+    load_kernel,
 };
 use crate::utils::u64_to_usize;
 

src/vmm/src/arch/mod.rs

@@ -24,6 +24,7 @@ use crate::cpu_config::templates::{
 };
 #[cfg(target_arch = "aarch64")]
 use crate::device_manager::AttachLegacyMmioDeviceError;
+use crate::device_manager::pci_mngr::PciManagerError;
 use crate::device_manager::{
     AttachMmioDeviceError, AttachVmgenidError, DeviceManager, DevicePersistError, DeviceRestoreArgs,
 };
@@ -71,6 +72,8 @@ pub enum StartMicrovmError {
     CreateLegacyDevice(device_manager::legacy::LegacyDeviceError),
     /// Error creating VMGenID device: {0}
     CreateVMGenID(VmGenIdError),
+    /// Error enabling PCIe support: {0}
+    EnablePciDevices(#[from] PciManagerError),
     /// Error enabling pvtime on vcpu: {0}
     #[cfg(target_arch = "aarch64")]
     EnablePVTime(crate::arch::VcpuArchError),
@@ -214,6 +217,8 @@ pub fn build_microvm_for_boot(
         .map(|vcpu| vcpu.copy_kvm_vcpu_fd(vmm.vm()))
         .collect::<Result<Vec<_>, _>>()?;
 
+    vmm.device_manager.enable_pci()?;
+
     // The boot timer device needs to be the first device attached in order
     // to maintain the same MMIO address referenced in the documentation
     // and tests.

src/vmm/src/builder.rs

@@ -17,6 +17,7 @@ use linux_loader::loader::Cmdline;
 use log::error;
 use mmio::{MMIODeviceManager, MmioError};
 use pci::DeviceRelocation;
+use pci_mngr::{PciDevices, PciManagerError};
 use persist::{ACPIDeviceManagerConstructorArgs, MMIODevManagerConstructorArgs};
 use resources::ResourceAllocator;
 use serde::{Deserialize, Serialize};
@@ -44,6 +45,8 @@ pub mod acpi;
 pub mod legacy;
 /// Memory Mapped I/O Manager.
 pub mod mmio;
+/// PCIe device manager
+pub mod pci_mngr;
 /// Device managers (de)serialization support.
 pub mod persist;
 /// Resource manager for devices.
@@ -105,6 +108,8 @@ pub struct DeviceManager {
     pub legacy_devices: PortIODeviceManager,
     /// ACPI devices
     pub acpi_devices: ACPIDeviceManager,
+    /// PCIe devices
+    pub pci_devices: PciDevices,
 }
 
 impl DeviceManager {
@@ -166,6 +171,7 @@ impl DeviceManager {
             #[cfg(target_arch = "x86_64")]
             legacy_devices,
             acpi_devices: ACPIDeviceManager::new(),
+            pci_devices: PciDevices::new(),
         })
     }
 
@@ -245,6 +251,12 @@ impl DeviceManager {
             .register_mmio_rtc(&self.resource_allocator, rtc, None)?;
         Ok(())
     }
+
+    /// Enables PCIe support for Firecracker devices
+    pub fn enable_pci(&mut self) -> Result<(), PciManagerError> {
+        self.pci_devices
+            .attach_pci_segment(&self.resource_allocator)
+    }
 }
 
 impl DeviceRelocation for DeviceManager {
@@ -381,6 +393,7 @@ pub(crate) mod tests {
     pub(crate) fn default_device_manager() -> DeviceManager {
         let mmio_devices = MMIODeviceManager::new();
         let acpi_devices = ACPIDeviceManager::new();
+        let pci_devices = PciDevices::new();
         let resource_allocator = Arc::new(ResourceAllocator::new().unwrap());
 
         #[cfg(target_arch = "x86_64")]
@@ -400,6 +413,7 @@ pub(crate) mod tests {
             #[cfg(target_arch = "x86_64")]
             legacy_devices,
             acpi_devices,
+            pci_devices,
         }
     }
 

src/vmm/src/device_manager/mod.rs

@@ -0,0 +1,49 @@
+// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::sync::Arc;
+
+use vm_device::BusError;
+
+use super::resources::ResourceAllocator;
+use crate::devices::pci::PciSegment;
+
+#[derive(Debug, Default)]
+pub struct PciDevices {
+    /// PCIe segment of the VMM, if PCI is enabled. We currently support a single PCIe segment.
+    pub pci_segment: Option<PciSegment>,
+}
+
+#[derive(Debug, thiserror::Error, displaydoc::Display)]
+pub enum PciManagerError {
+    /// Resource allocation error: {0}
+    ResourceAllocation(#[from] vm_allocator::Error),
+    /// Bus error: {0}
+    Bus(#[from] BusError),
+}
+
+impl PciDevices {
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    pub fn attach_pci_segment(
+        &mut self,
+        resource_allocator: &Arc<ResourceAllocator>,
+    ) -> Result<(), PciManagerError> {
+        // We only support a single PCIe segment. Calling this function twice is a Firecracker
+        // internal error.
+        assert!(self.pci_segment.is_none());
+
+        let irqs = resource_allocator.allocate_gsi(8)?;
+        let mut pci_irq_slots = [0u8; 32];
+        for i in 0..32 {
+            pci_irq_slots[i] = irqs[i % 8].try_into().unwrap();
+        }
+
+        let pci_segment = PciSegment::new(0, resource_allocator, &pci_irq_slots)?;
+        self.pci_segment = Some(pci_segment);
+
+        Ok(())
+    }
+}

src/vmm/src/device_manager/pci_mngr.rs

@@ -7,10 +7,13 @@
 
 //! Emulates virtual and hardware devices.
 
+#![allow(unused)]
+
 use std::io;
 
 pub mod acpi;
 pub mod legacy;
+pub mod pci;
 pub mod pseudo;
 pub mod virtio;