pci: add support for PCIe segment

Add a PCIe segment which includes a single PCIe root port and a bus.

At the moment, the PCIe segment is always enabled. Later commit will
make it optional and enable it only when a command line argument flag is
passed to Firecracker binary.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

src/firecracker/src/main.rs

@@ -449,7 +449,7 @@ fn main_exec() -> Result<(), MainError> {
 /// the default the jailer would set).
 ///
 /// We do this resizing because the kernel default is 64, with a reallocation happening whenever
-/// the tabel fills up. This was happening for some larger microVMs, and reallocating the
+/// the table fills up. This was happening for some larger microVMs, and reallocating the
 /// fdtable while a lot of file descriptors are active (due to being eventfds/timerfds registered
 /// to epoll) incurs a penalty of 30ms-70ms on the snapshot restore path.
 fn resize_fdtable() -> Result<(), ResizeFdTableError> {

src/vmm/src/acpi/mod.rs

@@ -90,6 +90,10 @@ impl AcpiTableWriter<'_> {
             .acpi_devices
             .append_aml_bytes(&mut dsdt_data)?;
 
+        if let Some(pci_segment) = &device_manager.pci_devices.pci_segment {
+            pci_segment.append_aml_bytes(&mut dsdt_data)?;
+        }
+
         // Architecture specific DSDT data
         setup_arch_dsdt(&mut dsdt_data)?;
 

src/vmm/src/arch/mod.rs

@@ -47,7 +47,9 @@ pub use crate::arch::x86_64::{
     layout::IOAPIC_ADDR, layout::IRQ_BASE, layout::IRQ_MAX, layout::MEM_32BIT_DEVICES_SIZE,
     layout::MEM_32BIT_DEVICES_START, layout::MEM_64BIT_DEVICES_SIZE,
     layout::MEM_64BIT_DEVICES_START, layout::MMIO32_MEM_SIZE, layout::MMIO32_MEM_START,
-    layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START, load_kernel,
+    layout::PCI_MMCONFIG_SIZE, layout::PCI_MMCONFIG_START,
+    layout::PCI_MMIO_CONFIG_SIZE_PER_SEGMENT, layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START,
+    load_kernel,
 };
 
 /// Types of devices that can get attached to this platform.

src/vmm/src/builder.rs

@@ -24,6 +24,7 @@ use crate::cpu_config::templates::{
 };
 #[cfg(target_arch = "aarch64")]
 use crate::device_manager::AttachLegacyMmioDeviceError;
+use crate::device_manager::pci_mngr::PciManagerError;
 use crate::device_manager::{
     AttachMmioDeviceError, AttachVmgenidError, DeviceManager, DevicePersistError, DeviceRestoreArgs,
 };
@@ -71,6 +72,8 @@ pub enum StartMicrovmError {
     CreateLegacyDevice(device_manager::legacy::LegacyDeviceError),
     /// Error creating VMGenID device: {0}
     CreateVMGenID(VmGenIdError),
+    /// Error enabling PCIe support: {0}
+    EnablePciDevices(#[from] PciManagerError),
     /// Error enabling pvtime on vcpu: {0}
     #[cfg(target_arch = "aarch64")]
     EnablePVTime(crate::arch::VcpuArchError),
@@ -214,6 +217,8 @@ pub fn build_microvm_for_boot(
         .map(|vcpu| vcpu.copy_kvm_vcpu_fd(vmm.vm()))
         .collect::<Result<Vec<_>, _>>()?;
 
+    vmm.device_manager.enable_pci()?;
+
     // The boot timer device needs to be the first device attached in order
     // to maintain the same MMIO address referenced in the documentation
     // and tests.

src/vmm/src/device_manager/mod.rs

@@ -16,6 +16,7 @@ use legacy::{LegacyDeviceError, PortIODeviceManager};
 use linux_loader::loader::Cmdline;
 use log::error;
 use mmio::{MMIODeviceManager, MmioError};
+use pci_mngr::{PciDevices, PciManagerError};
 use persist::{ACPIDeviceManagerConstructorArgs, MMIODevManagerConstructorArgs};
 use resources::ResourceAllocator;
 use serde::{Deserialize, Serialize};
@@ -43,6 +44,8 @@ pub mod acpi;
 pub mod legacy;
 /// Memory Mapped I/O Manager.
 pub mod mmio;
+/// PCIe device manager
+pub mod pci_mngr;
 /// Device managers (de)serialization support.
 pub mod persist;
 /// Resource manager for devices.
@@ -104,6 +107,8 @@ pub struct DeviceManager {
     pub legacy_devices: PortIODeviceManager,
     /// ACPI devices
     pub acpi_devices: ACPIDeviceManager,
+    /// PCIe devices
+    pub pci_devices: PciDevices,
 }
 
 impl DeviceManager {
@@ -165,6 +170,7 @@ impl DeviceManager {
             #[cfg(target_arch = "x86_64")]
             legacy_devices,
             acpi_devices: ACPIDeviceManager::new(),
+            pci_devices: PciDevices::new(),
         })
     }
 
@@ -244,6 +250,12 @@ impl DeviceManager {
             .register_mmio_rtc(&self.resource_allocator, rtc, None)?;
         Ok(())
     }
+
+    /// Enables PCIe support for Firecracker devices
+    pub fn enable_pci(&mut self) -> Result<(), PciManagerError> {
+        self.pci_devices
+            .attach_pci_segment(&self.resource_allocator)
+    }
 }
 
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
@@ -367,6 +379,7 @@ pub(crate) mod tests {
     pub(crate) fn default_device_manager() -> DeviceManager {
         let mmio_devices = MMIODeviceManager::new();
         let acpi_devices = ACPIDeviceManager::new();
+        let pci_devices = PciDevices::new();
         let resource_allocator = Arc::new(ResourceAllocator::new().unwrap());
 
         #[cfg(target_arch = "x86_64")]
@@ -386,6 +399,7 @@ pub(crate) mod tests {
             #[cfg(target_arch = "x86_64")]
             legacy_devices,
             acpi_devices,
+            pci_devices,
         }
     }
 

src/vmm/src/device_manager/pci_mngr.rs

@@ -0,0 +1,45 @@
+// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::sync::Arc;
+
+use vm_device::BusError;
+
+use super::resources::ResourceAllocator;
+use crate::devices::pci::PciSegment;
+
+#[derive(Debug, Default)]
+pub struct PciDevices {
+    /// PCIe segment of the VMM, if PCI is enabled. We currently support a single PCIe segment.
+    pub pci_segment: Option<PciSegment>,
+}
+
+#[derive(Debug, thiserror::Error, displaydoc::Display)]
+pub enum PciManagerError {
+    /// Resource allocation error: {0}
+    ResourceAllocation(#[from] vm_allocator::Error),
+    /// Bus error: {0}
+    Bus(#[from] BusError),
+}
+
+impl PciDevices {
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    pub fn attach_pci_segment(
+        &mut self,
+        resource_allocator: &Arc<ResourceAllocator>,
+    ) -> Result<(), PciManagerError> {
+        // We only support a single PCIe segment. Calling this function twice is a Firecracker
+        // internal error.
+        assert!(self.pci_segment.is_none());
+
+        // Currently we don't assign any IRQs to PCI devices. We will be using MSI-X interrupts
+        // only.
+        let pci_segment = PciSegment::new(0, resource_allocator, &[0u8; 32])?;
+        self.pci_segment = Some(pci_segment);
+
+        Ok(())
+    }
+}

src/vmm/src/devices/mod.rs

@@ -7,10 +7,13 @@
 
 //! Emulates virtual and hardware devices.
 
+#![allow(unused)]
+
 use std::io;
 
 pub mod acpi;
 pub mod legacy;
+pub mod pci;
 pub mod pseudo;
 pub mod virtio;
 

src/vmm/src/devices/pci/mod.rs

@@ -0,0 +1,6 @@
+// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod pci_segment;
+
+pub use pci_segment::*;