feat: support memory over 4GiB

This commits adds support for allocating memory over 4GiB to the VM.
It creates a new memory region after the APIC and MSI interrupt region.

Signed-off-by: Nils Ponsard <nilsponsard@gmail.com>

Author: nponsard

src/vmm/src/kernel.rs

@@ -75,6 +75,8 @@ fn add_e820_entry(
 pub fn build_bootparams(
     guest_memory: &GuestMemoryMmap,
     himem_start: GuestAddress,
+    reserved_memory_start: GuestAddress,
+    reserved_memory_end: GuestAddress,
 ) -> std::result::Result<boot_params, Error> {
     let mut params = boot_params::default();
 
@@ -88,14 +90,35 @@ pub fn build_bootparams(
 
     // Add entries for the usable RAM regions.
     let last_addr = guest_memory.last_addr();
-    add_e820_entry(
-        &mut params,
-        himem_start.raw_value() as u64,
-        last_addr
-            .checked_offset_from(himem_start)
-            .ok_or(Error::HimemStartPastMemEnd)?,
-        E820_RAM,
-    )?;
+
+    if reserved_memory_start > last_addr {
+        add_e820_entry(
+            &mut params,
+            himem_start.raw_value() as u64,
+            last_addr
+                .checked_offset_from(himem_start)
+                .ok_or(Error::HimemStartPastMemEnd)?,
+            E820_RAM,
+        )?;
+    } else {
+        add_e820_entry(
+            &mut params,
+            himem_start.raw_value() as u64,
+            reserved_memory_start
+                .checked_offset_from(himem_start)
+                .ok_or(Error::HimemStartPastMemEnd)?,
+            E820_RAM,
+        )?;
+
+        add_e820_entry(
+            &mut params,
+            reserved_memory_end.raw_value() as u64,
+            last_addr
+                .checked_offset_from(reserved_memory_end)
+                .ok_or(Error::HimemStartPastMemEnd)?,
+            E820_RAM,
+        )?;
+    }
 
     Ok(params)
 }
@@ -109,6 +132,8 @@ pub fn build_bootparams(
 pub fn kernel_setup(
     guest_memory: &GuestMemoryMmap,
     kernel_path: PathBuf,
+    reserved_memory_start: GuestAddress,
+    reserved_memory_end: GuestAddress,
 ) -> Result<KernelLoaderResult> {
     let mut kernel_image = File::open(kernel_path).map_err(Error::IO)?;
     let zero_page_addr = GuestAddress(ZEROPG_START);
@@ -123,7 +148,12 @@ pub fn kernel_setup(
     .map_err(Error::KernelLoad)?;
 
     // Generate boot parameters.
-    let mut bootparams = build_bootparams(guest_memory, GuestAddress(HIMEM_START))?;
+    let mut bootparams = build_bootparams(
+        guest_memory,
+        GuestAddress(HIMEM_START),
+        reserved_memory_start,
+        reserved_memory_end,
+    )?;
 
     // Add the kernel command line to the boot parameters.
     bootparams.hdr.cmd_line_ptr = CMDLINE_START as u32;

src/vmm/src/lib.rs

@@ -30,6 +30,11 @@ mod epoll_context;
 use epoll_context::{EpollContext, EPOLL_EVENTS_LEN};
 mod kernel;
 
+/// avoid the memory space where the APIC and MSI interrupts are located
+const HIGH_RESERVED_MEMORY_END: usize = 1 << 32;
+const HIGH_RESERVED_MEMORY_SIZE: usize = 0x1400000;
+const HIGH_RESERVED_MEMORY_START: usize = HIGH_RESERVED_MEMORY_END - HIGH_RESERVED_MEMORY_SIZE;
+
 #[derive(Debug)]
 
 /// VMM errors.
@@ -112,8 +117,17 @@ impl VMM {
         // Convert memory size from MBytes to bytes.
         let mem_size = ((mem_size_mb as u64) << 20) as usize;
 
-        // Create one single memory region, from zero to mem_size.
-        let mem_regions = vec![(GuestAddress(0), mem_size)];
+        // Check if the memory is overlapping with the APIC memory region.
+        // If it is, split the memory into two regions.
+
+        let mem_regions = if mem_size < HIGH_RESERVED_MEMORY_START {
+            vec![(GuestAddress(0), mem_size)]
+        } else {
+            vec![
+                (GuestAddress(0), HIGH_RESERVED_MEMORY_START),
+                (GuestAddress(HIGH_RESERVED_MEMORY_END as u64), mem_size - HIGH_RESERVED_MEMORY_END),
+            ]
+        };
 
         // Allocate the guest memory from the memory region.
         let guest_memory = GuestMemoryMmap::from_ranges(&mem_regions).map_err(Error::Memory)?;
@@ -269,7 +283,12 @@ impl VMM {
     pub fn configure(&mut self, num_vcpus: u8, mem_size_mb: u32, kernel_path: &str, console: Option<String>) -> Result<()> {
         self.configure_console(console)?;
         self.configure_memory(mem_size_mb)?;
-        let kernel_load = kernel::kernel_setup(&self.guest_memory, PathBuf::from(kernel_path))?;
+        let kernel_load = kernel::kernel_setup(
+            &self.guest_memory,
+            PathBuf::from(kernel_path),
+            GuestAddress(HIGH_RESERVED_MEMORY_START as u64),
+            GuestAddress(HIGH_RESERVED_MEMORY_END as u64),
+        )?;
         self.configure_io()?;
         self.configure_vcpus(num_vcpus, kernel_load)?;