pci: make PCIe support optional

Add a command line argument to enable PCIe support. By default, PCIe
is disabled. The reason for making PCIe off by default is that users
need to explicitly enable PCI support in their kernels. Requiring users
to explicitly enable it, does not break existing deployments, i.e. users
can upgrade Firecracker within their existing environments without
breaking any deployment.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

src/firecracker/src/api_server_adapter.rs

@@ -143,6 +143,7 @@ pub(crate) fn run_with_api(
     instance_info: InstanceInfo,
     process_time_reporter: ProcessTimeReporter,
     boot_timer_enabled: bool,
+    pci_enabled: bool,
     api_payload_limit: usize,
     mmds_size_limit: usize,
     metadata_json: Option<&str>,
@@ -212,6 +213,7 @@ pub(crate) fn run_with_api(
             json,
             instance_info,
             boot_timer_enabled,
+            pci_enabled,
             mmds_size_limit,
             metadata_json,
         )
@@ -224,6 +226,7 @@ pub(crate) fn run_with_api(
             &to_api,
             &api_event_fd,
             boot_timer_enabled,
+            pci_enabled,
             mmds_size_limit,
             metadata_json,
         )

src/firecracker/src/main.rs

@@ -260,6 +260,11 @@ fn main_exec() -> Result<(), MainError> {
                 Argument::new("mmds-size-limit")
                     .takes_value(true)
                     .help("Mmds data store limit, in bytes."),
+            )
+            .arg(
+                Argument::new("enable-pci")
+                    .takes_value(false)
+                    .help("Enables PCIe support."),
             );
 
     arg_parser.parse_from_cmdline()?;
@@ -369,6 +374,7 @@ fn main_exec() -> Result<(), MainError> {
         .map(|x| x.expect("Unable to open or read from the mmds content file"));
 
     let boot_timer_enabled = arguments.flag_present("boot-timer");
+    let pci_enabled = arguments.flag_present("enable-pci");
     let api_enabled = !arguments.flag_present("no-api");
     let api_payload_limit = arg_parser
         .arguments()
@@ -422,6 +428,7 @@ fn main_exec() -> Result<(), MainError> {
             instance_info,
             process_time_reporter,
             boot_timer_enabled,
+            pci_enabled,
             api_payload_limit,
             mmds_size_limit,
             metadata_json.as_deref(),
@@ -437,6 +444,7 @@ fn main_exec() -> Result<(), MainError> {
             vmm_config_json,
             instance_info,
             boot_timer_enabled,
+            pci_enabled,
             mmds_size_limit,
             metadata_json.as_deref(),
         )
@@ -554,19 +562,22 @@ pub enum BuildFromJsonError {
 }
 
 // Configure and start a microVM as described by the command-line JSON.
+#[allow(clippy::too_many_arguments)]
 fn build_microvm_from_json(
     seccomp_filters: &BpfThreadMap,
     event_manager: &mut EventManager,
     config_json: String,
     instance_info: InstanceInfo,
     boot_timer_enabled: bool,
+    pci_enabled: bool,
     mmds_size_limit: usize,
     metadata_json: Option<&str>,
 ) -> Result<(VmResources, Arc<Mutex<vmm::Vmm>>), BuildFromJsonError> {
     let mut vm_resources =
         VmResources::from_json(&config_json, &instance_info, mmds_size_limit, metadata_json)
             .map_err(BuildFromJsonError::ParseFromJson)?;
     vm_resources.boot_timer = boot_timer_enabled;
+    vm_resources.pci_enabled = pci_enabled;
     let vmm = vmm::builder::build_and_boot_microvm(
         &instance_info,
         &vm_resources,
@@ -593,6 +604,7 @@ fn run_without_api(
     config_json: Option<String>,
     instance_info: InstanceInfo,
     bool_timer_enabled: bool,
+    pci_enabled: bool,
     mmds_size_limit: usize,
     metadata_json: Option<&str>,
 ) -> Result<(), RunWithoutApiError> {
@@ -610,6 +622,7 @@ fn run_without_api(
         config_json.unwrap(),
         instance_info,
         bool_timer_enabled,
+        pci_enabled,
         mmds_size_limit,
         metadata_json,
     )

src/vmm/src/builder.rs

@@ -217,7 +217,11 @@ pub fn build_microvm_for_boot(
         .map(|vcpu| vcpu.copy_kvm_vcpu_fd(vmm.vm()))
         .collect::<Result<Vec<_>, _>>()?;
 
-    vmm.device_manager.enable_pci()?;
+    if vm_resources.pci_enabled {
+        vmm.device_manager.enable_pci()?;
+    } else {
+        boot_cmdline.insert("pci", "off")?;
+    }
 
     // The boot timer device needs to be the first device attached in order
     // to maintain the same MMIO address referenced in the documentation

src/vmm/src/resources.rs

@@ -114,6 +114,8 @@ pub struct VmResources {
     pub mmds_size_limit: usize,
     /// Whether or not to load boot timer device.
     pub boot_timer: bool,
+    /// Whether or not to use PCIe transport for VirtIO devices.
+    pub pci_enabled: bool,
 }
 
 impl VmResources {
@@ -613,6 +615,7 @@ mod tests {
             boot_timer: false,
             mmds_size_limit: HTTP_MAX_PAYLOAD_SIZE,
             entropy: Default::default(),
+            pci_enabled: false,
         }
     }
 

src/vmm/src/rpc_interface.rs

@@ -327,18 +327,16 @@ impl<'a> PrebootApiController<'a> {
         to_api: &std::sync::mpsc::Sender<ApiResponse>,
         api_event_fd: &vmm_sys_util::eventfd::EventFd,
         boot_timer_enabled: bool,
+        pci_enabled: bool,
         mmds_size_limit: usize,
         metadata_json: Option<&str>,
     ) -> Result<(VmResources, Arc<Mutex<Vmm>>), BuildMicrovmFromRequestsError> {
-        let mut vm_resources = VmResources::default();
-        // Silence false clippy warning. Clippy suggests using
-        // VmResources { boot_timer: boot_timer_enabled, ..Default::default() }; but this will
-        // generate build errors because VmResources contains private fields.
-        #[allow(clippy::field_reassign_with_default)]
-        {
-            vm_resources.mmds_size_limit = mmds_size_limit;
-            vm_resources.boot_timer = boot_timer_enabled;
-        }
+        let mut vm_resources = VmResources {
+            boot_timer: boot_timer_enabled,
+            mmds_size_limit,
+            pci_enabled,
+            ..Default::default()
+        };
 
         // Init the data store from file, if present.
         if let Some(data) = metadata_json {

src/vmm/src/vmm_config/boot_source.rs

@@ -9,14 +9,13 @@ use serde::{Deserialize, Serialize};
 /// Default guest kernel command line:
 /// - `reboot=k` shut down the guest on reboot, instead of well... rebooting;
 /// - `panic=1` on panic, reboot after 1 second;
-/// - `pci=off` do not scan for PCI devices (save boot time);
 /// - `nomodule` disable loadable kernel module support;
 /// - `8250.nr_uarts=0` disable 8250 serial interface;
 /// - `i8042.noaux` do not probe the i8042 controller for an attached mouse (save boot time);
 /// - `i8042.nomux` do not probe i8042 for a multiplexing controller (save boot time);
 /// - `i8042.dumbkbd` do not attempt to control kbd state via the i8042 (save boot time).
 pub const DEFAULT_KERNEL_CMDLINE: &str =
-    "reboot=k panic=1 pci=off nomodule 8250.nr_uarts=0 i8042.noaux i8042.nomux i8042.dumbkbd";
+    "reboot=k panic=1 nomodule 8250.nr_uarts=0 i8042.noaux i8042.nomux i8042.dumbkbd";
 
 /// Strongly typed data structure used to configure the boot source of the
 /// microvm.