Wire up ACPI table generation via fw_cfg

Integrate the new ACPI table generation into propolis-standalone and
propolis-server. Also replace hardcoded memory region addresses with
constants that align with ACPI table definitions.

The PCIe ECAM base is kept same as before at 0xe000_0000 (3.5GB) to
match existing i440fx chipset ECAM placement. ECAM is no longer added
to the E820 map as reserved memory since it is MMIO space properly
described in the MCFG ACPI table.

Guest physical memory map:
0x0000_0000 - 0xbfff_ffff    Low RAM (up to 3 GiB)
0xc000_0000 - 0xffff_ffff    PCI hole (1 GiB MMIO region)
  0xc000_0000 - 0xdfff_ffff    32-bit PCI MMIO
  0xe000_0000 - 0xefff_ffff    PCIe ECAM (256 MiB, 256 buses)
  0xfec0_0000                  IOAPIC
  0xfed0_0000                  HPET
  0xffe0_0000 - 0xffff_ffff    Bootrom (2 MiB)
0x1_0000_0000+               High RAM + 64-bit PCI MMIO

e820 map as seen by guest:
0x0000_0000 - 0x0009_ffff    Usable (640 KiB low memory)
0x0010_0000 - 0xbeaf_ffff    Usable (~3 GiB main RAM)
0xbeb0_0000 - 0xbfb6_cfff    Reserved (UEFI runtime/data)
0xbfb6_d000 - 0xbfbf_efff    ACPI Tables + NVS
0xbfbf_f000 - 0xbffd_ffff    Usable (top of low memory)
0xbffe_0000 - 0xffff_ffff    Reserved (PCI hole)
0x1_0000_0000 - highmem      Usable (high RAM above 4 GiB)

To stay on safe side only enable using new ACPI tables for newly
launched VMs. Old VMs using OVMF tables would keep using the same OVMF
tables throughout multiple migrations.  To verify this add the phd test
as well for new VM launched with native tables, native tables preserved
through migration and VM launched from old propolis without native
tables stays with OVMF through multiple future migrations.

Signed-off-by: Amey Narkhede <[email protected]>

Author: glitzflitz

bin/propolis-cli/src/main.rs

@@ -340,6 +340,7 @@ impl VmConfig {
                 } else {
                     Default::default()
                 },
+                native_acpi_tables: Some(true),
             },
             components: Default::default(),
             smbios: None,

bin/propolis-server/src/lib/initializer.rs

@@ -111,6 +111,12 @@ pub enum MachineInitError {
 /// Arbitrary ROM limit for now
 const MAX_ROM_SIZE: usize = 0x20_0000;
 
+const PCIE_ECAM_BASE: usize = 0xe000_0000;
+const PCIE_ECAM_SIZE: usize = 0x1000_0000;
+const MEM_32BIT_DEVICES_START: usize = 0xc000_0000;
+const MEM_32BIT_DEVICES_END: usize = 0xfc00_0000;
+const HIGHMEM_START: usize = 0x1_0000_0000;
+
 fn get_spec_guest_ram_limits(spec: &Spec) -> (usize, usize) {
     let memsize = spec.board.memory_mb as usize * MB;
     let lowmem = memsize.min(3 * GB);
@@ -141,19 +147,22 @@ pub fn build_instance(
         .context("failed to add low memory region")?
         .add_rom_region(0x1_0000_0000 - MAX_ROM_SIZE, MAX_ROM_SIZE, "bootrom")
         .context("failed to add bootrom region")?
-        .add_mmio_region(0xc000_0000_usize, 0x2000_0000_usize, "dev32")
+        .add_mmio_region(lowmem, PCIE_ECAM_BASE - lowmem, "dev32")
         .context("failed to add low device MMIO region")?
-        .add_mmio_region(0xe000_0000_usize, 0x1000_0000_usize, "pcicfg")
+        .add_mmio_region(
+            PCIE_ECAM_BASE,
+            MEM_32BIT_DEVICES_END - PCIE_ECAM_BASE,
+            "pcicfg",
+        )
         .context("failed to add PCI config region")?;
 
-    let highmem_start = 0x1_0000_0000;
     if highmem > 0 {
         builder = builder
-            .add_mem_region(highmem_start, highmem, "highmem")
+            .add_mem_region(HIGHMEM_START, highmem, "highmem")
             .context("failed to add high memory region")?;
     }
 
-    let dev64_start = highmem_start + highmem;
+    let dev64_start = HIGHMEM_START + highmem;
     builder = builder
         .add_mmio_region(dev64_start, vmm::MAX_PHYSMEM - dev64_start, "dev64")
         .context("failed to add high device MMIO region")?;
@@ -1156,9 +1165,17 @@ impl MachineInitializer<'_> {
                 propolis::vmm::MapType::Dram => {
                     e820_table.add_mem(addr, len);
                 }
-                _ => {
+                propolis::vmm::MapType::Rom => {
                     e820_table.add_reserved(addr, len);
                 }
+                propolis::vmm::MapType::Mmio => {
+                    // With native ACPI tables, MMIO is described in the DSDT
+                    // _CRS and should not appear in E820. Without native
+                    // tables, preserve original E820 layout.
+                    if !self.spec.board.native_acpi_tables {
+                        e820_table.add_reserved(addr, len);
+                    }
+                }
             }
         }
 
@@ -1270,6 +1287,49 @@ impl MachineInitializer<'_> {
             .insert_named("etc/e820", e820_entry)
             .map_err(|e| MachineInitError::FwcfgInsertFailed("e820", e))?;
 
+        if self.spec.board.native_acpi_tables {
+            let (_, highmem) = get_spec_guest_ram_limits(self.spec);
+            let dev64_start = HIGHMEM_START + highmem;
+
+            let acpi_tables = fwcfg::formats::build_acpi_tables(
+                &fwcfg::formats::AcpiConfig {
+                    num_cpus: cpus,
+                    pcie_ecam_base: PCIE_ECAM_BASE as u64,
+                    pcie_ecam_size: PCIE_ECAM_SIZE as u64,
+                    pcie_mmio32_base: MEM_32BIT_DEVICES_START as u64,
+                    pcie_mmio32_limit: (MEM_32BIT_DEVICES_END - 1) as u64,
+                    pcie_mmio64_base: dev64_start as u64,
+                    pcie_mmio64_limit: vmm::MAX_PHYSMEM as u64 - 1,
+                    com_ports: vec![
+                        (ibmpc::PORT_COM1, ibmpc::IRQ_COM1),
+                        (ibmpc::PORT_COM2, ibmpc::IRQ_COM2),
+                    ],
+                    ..Default::default()
+                },
+            );
+            fwcfg
+                .insert_named(
+                    "etc/acpi/tables",
+                    Entry::Bytes(acpi_tables.tables),
+                )
+                .map_err(|e| {
+                    MachineInitError::FwcfgInsertFailed("acpi/tables", e)
+                })?;
+            fwcfg
+                .insert_named("etc/acpi/rsdp", Entry::Bytes(acpi_tables.rsdp))
+                .map_err(|e| {
+                    MachineInitError::FwcfgInsertFailed("acpi/rsdp", e)
+                })?;
+            fwcfg
+                .insert_named(
+                    "etc/table-loader",
+                    Entry::Bytes(acpi_tables.loader),
+                )
+                .map_err(|e| {
+                    MachineInitError::FwcfgInsertFailed("table-loader", e)
+                })?;
+        }
+
         let ramfb = ramfb::RamFb::create(
             self.log.new(slog::o!("component" => "ramfb")),
         );

bin/propolis-server/src/lib/server.rs

@@ -248,7 +248,10 @@ impl PropolisServerApi for PropolisServerImpl {
         let vm_init = match init {
             InstanceInitializationMethod::Spec { spec } => spec
                 .try_into()
-                .map(|s| VmInitializationMethod::Spec(Box::new(s)))
+                .map(|mut s: crate::spec::Spec| {
+                    s.board.native_acpi_tables = true;
+                    VmInitializationMethod::Spec(Box::new(s))
+                })
                 .map_err(|e| {
                     if let Some(s) = e.source() {
                         format!("{e}: {s}")
@@ -337,7 +340,10 @@ impl PropolisServerApi for PropolisServerImpl {
         let vm_init = match init {
             InstanceInitializationMethodV0::Spec { spec } => spec
                 .try_into()
-                .map(|s| VmInitializationMethod::Spec(Box::new(s)))
+                .map(|mut s: crate::spec::Spec| {
+                    s.board.native_acpi_tables = true;
+                    VmInitializationMethod::Spec(Box::new(s))
+                })
                 .map_err(|e| {
                     if let Some(s) = e.source() {
                         format!("{e}: {s}")

bin/propolis-server/src/lib/spec/api_spec_v0.rs

@@ -101,6 +101,7 @@ impl From<Spec> for InstanceSpecV0 {
             chipset: board.chipset,
             guest_hv_interface: board.guest_hv_interface,
             cpuid: Some(cpuid.into_instance_spec_cpuid()),
+            native_acpi_tables: Some(board.native_acpi_tables),
         };
         let mut spec = InstanceSpecV0 { board, components: Default::default() };
 

bin/propolis-server/src/lib/spec/builder.rs

@@ -96,6 +96,9 @@ impl SpecBuilder {
                     memory_mb: board.memory_mb,
                     chipset: board.chipset,
                     guest_hv_interface: board.guest_hv_interface,
+                    native_acpi_tables: board
+                        .native_acpi_tables
+                        .unwrap_or(false),
                 },
                 cpuid,
                 ..Default::default()
@@ -380,6 +383,7 @@ mod test {
             memory_mb: 512,
             chipset: Chipset::I440Fx(I440Fx { enable_pcie: false }),
             guest_hv_interface: GuestHypervisorInterface::Bhyve,
+            native_acpi_tables: false,
         };
 
         SpecBuilder {

bin/propolis-server/src/lib/spec/mod.rs

@@ -129,6 +129,7 @@ pub(crate) struct Board {
     pub memory_mb: u64,
     pub chipset: Chipset,
     pub guest_hv_interface: GuestHypervisorInterface,
+    pub native_acpi_tables: bool,
 }
 
 impl Default for Board {
@@ -138,6 +139,7 @@ impl Default for Board {
             memory_mb: 0,
             chipset: Chipset::I440Fx(I440Fx { enable_pcie: false }),
             guest_hv_interface: GuestHypervisorInterface::Bhyve,
+            native_acpi_tables: false,
         }
     }
 }

bin/propolis-standalone/src/main.rs

@@ -43,6 +43,12 @@ const PAGE_OFFSET: u64 = 0xfff;
 // Arbitrary ROM limit for now
 const MAX_ROM_SIZE: usize = 0x20_0000;
 
+const PCIE_ECAM_BASE: usize = 0xe000_0000;
+const PCIE_ECAM_SIZE: usize = 0x1000_0000;
+const MEM_32BIT_DEVICES_START: usize = 0xc000_0000;
+const MEM_32BIT_DEVICES_END: usize = 0xfc00_0000;
+const HIGHMEM_START: usize = 0x1_0000_0000;
+
 const MIN_RT_THREADS: usize = 8;
 const BASE_RT_THREADS: usize = 4;
 
@@ -738,15 +744,18 @@ fn build_machine(
     .max_cpus(max_cpu)?
     .add_mem_region(0, lowmem, "lowmem")?
     .add_rom_region(0x1_0000_0000 - MAX_ROM_SIZE, MAX_ROM_SIZE, "bootrom")?
-    .add_mmio_region(0xc000_0000, 0x2000_0000, "dev32")?
-    .add_mmio_region(0xe000_0000, 0x1000_0000, "pcicfg")?;
+    .add_mmio_region(lowmem, PCIE_ECAM_BASE - lowmem, "dev32")?
+    .add_mmio_region(
+        PCIE_ECAM_BASE,
+        MEM_32BIT_DEVICES_END - PCIE_ECAM_BASE,
+        "pcicfg",
+    )?;
 
-    let highmem_start = 0x1_0000_0000;
     if highmem > 0 {
-        builder = builder.add_mem_region(highmem_start, highmem, "highmem")?;
+        builder = builder.add_mem_region(HIGHMEM_START, highmem, "highmem")?;
     }
 
-    let dev64_start = highmem_start + highmem;
+    let dev64_start = HIGHMEM_START + highmem;
     builder = builder.add_mmio_region(
         dev64_start,
         vmm::MAX_PHYSMEM - dev64_start,
@@ -974,9 +983,13 @@ fn generate_e820(
             MapType::Dram => {
                 e820_table.add_mem(addr, len);
             }
-            _ => {
+            MapType::Rom => {
                 e820_table.add_reserved(addr, len);
             }
+            MapType::Mmio => {
+                // MMIO is described in the DSDT _CRS and should not
+                // appear in E820.
+            }
         }
     }
 
@@ -1375,6 +1388,37 @@ fn setup_instance(
     let e820_entry = generate_e820(machine, log).expect("can build E820 table");
     fwcfg.insert_named("etc/e820", e820_entry).unwrap();
 
+    let acpi_tables =
+        fwcfg::formats::build_acpi_tables(&fwcfg::formats::AcpiConfig {
+            num_cpus: cpus,
+            pcie_ecam_base: PCIE_ECAM_BASE as u64,
+            pcie_ecam_size: PCIE_ECAM_SIZE as u64,
+            pcie_mmio32_base: MEM_32BIT_DEVICES_START as u64,
+            pcie_mmio32_limit: (MEM_32BIT_DEVICES_END - 1) as u64,
+            pcie_mmio64_base: (HIGHMEM_START + highmem) as u64,
+            pcie_mmio64_limit: vmm::MAX_PHYSMEM as u64 - 1,
+            com_ports: vec![
+                (ibmpc::PORT_COM1, ibmpc::IRQ_COM1),
+                (ibmpc::PORT_COM2, ibmpc::IRQ_COM2),
+            ],
+            ..Default::default()
+        });
+    fwcfg
+        .insert_named(
+            "etc/acpi/tables",
+            fwcfg::Entry::Bytes(acpi_tables.tables),
+        )
+        .context("failed to insert ACPI tables")?;
+    fwcfg
+        .insert_named("etc/acpi/rsdp", fwcfg::Entry::Bytes(acpi_tables.rsdp))
+        .context("failed to insert ACPI RSDP")?;
+    fwcfg
+        .insert_named(
+            "etc/table-loader",
+            fwcfg::Entry::Bytes(acpi_tables.loader),
+        )
+        .context("failed to insert ACPI table-loader")?;
+
     fwcfg.attach(pio, &machine.acc_mem);
 
     guard.inventory.register(&fwcfg);

crates/propolis-api-types/src/instance_spec/components/board.rs

@@ -178,5 +178,14 @@ pub struct Board {
     /// default values from the host's CPUID values.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub cpuid: Option<Cpuid>,
+
+    /// Use native ACPI tables (via fw_cfg) instead of OVMF provided tables.
+    ///
+    /// VMs created before propolis supported ACPI table generation will not
+    /// have this field. For backwards compatibility with live migration,
+    /// `None` is treated as `false` (use OVMF tables). New VMs should set
+    /// this to `true`.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub native_acpi_tables: Option<bool>,
     // TODO: Processor and NUMA topology.
 }

phd-tests/framework/src/test_vm/config.rs

@@ -56,6 +56,7 @@ pub struct VmConfig<'dr> {
     disks: Vec<DiskRequest<'dr>>,
     migration_failure: Option<MigrationFailureInjector>,
     guest_hv_interface: Option<GuestHypervisorInterface>,
+    native_acpi_tables: bool,
 }
 
 impl<'dr> VmConfig<'dr> {
@@ -76,6 +77,7 @@ impl<'dr> VmConfig<'dr> {
             disks: Vec::new(),
             migration_failure: None,
             guest_hv_interface: None,
+            native_acpi_tables: true,
         };
 
         config.boot_disk(
@@ -151,6 +153,11 @@ impl<'dr> VmConfig<'dr> {
         self
     }
 
+    pub fn native_acpi_tables(&mut self, enabled: bool) -> &mut Self {
+        self.native_acpi_tables = enabled;
+        self
+    }
+
     /// Add a new disk to the VM config, and add it to the front of the VM's
     /// boot order.
     ///
@@ -221,6 +228,7 @@ impl<'dr> VmConfig<'dr> {
             disks,
             migration_failure,
             guest_hv_interface,
+            native_acpi_tables,
         } = self;
 
         let bootrom_path = framework
@@ -302,6 +310,7 @@ impl<'dr> VmConfig<'dr> {
                     .as_ref()
                     .cloned()
                     .unwrap_or_default(),
+                native_acpi_tables: Some(*native_acpi_tables),
             },
             components: Default::default(),
             smbios: None,