[host/hypervisor_handler,host/mem/{layout,mgr},docs/paging] moved guest code to start at 0x0

instead of after paging sections

This is the first step in moving memory sections around to provide a more generic
layout. After this, I'll move continue moving sections to fulfill the layout shown
in the description for PR hyperlight-dev#297.

Changes:
- removed no longer applicable checks in hypervisor_handler
- changed the way we build memory in get_memory_regions
- added functions to get the offsets/addresses to paging sections
as they are no longer static from 0x0 and are now relative to code size
- updated code docs
- changed set_up_shared_memory to properly place paging sections
- updated paging docs

Signed-off-by: danbugs <[email protected]>

Author: danbugs

docs/paging-development-notes.md

@@ -29,11 +29,12 @@ The following page table structs are set up in memory before running a Hyperligh
 ### PML4 (Page Map Level 4) Table
 
 The PML4 table is located at physical address specified in CR3. In Hyperlight we set 
-`CR3=0x0`, which means the PML4 table is located at physical address `0x0`. The PML4 
-table comprises 512 64-bit entries.
+`CR3=pml4_address` (i.e., i.e., base address (0x0) + aligned guest code size), which 
+means the PML4 table is located at physical address `0x0`. The PML4 table comprises 
+512 64-bit entries.
 
-In Hyperlight, we only initialize the first entry (at address `0x0`), with value  
-`0x1_000`, implying that we only have a single PDPT.
+In Hyperlight, we only initialize the first entry, with value `0x1_000`, implying that 
+we only have a single PDPT.
 
 ### PDPT (Page-directory-pointer Table)
 
@@ -63,7 +64,7 @@ created to cover the size of memory mapped into the VM.
 Given a 64-bit virtual address X, the corresponding physical address is obtained as 
 follows:
 
-1. PML4 table's physical address is located using CR3 (CR3 is `0x0`).
+1. PML4 table's physical address is located using CR3.
 2. Bits 47:39 of X are used to index into PML4, giving us the address of the PDPT.
 3. Bits 38:30 of X are used to index into PDPT, giving us the address of the PD.
 4. Bits 29:21 of X are used to index into PD, giving us the address of the PT.

src/hyperlight_host/src/hypervisor/hypervisor_handler.rs

@@ -45,7 +45,6 @@ use crate::hypervisor::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
 #[cfg(target_os = "windows")]
 use crate::hypervisor::wrappers::HandleWrapper;
 use crate::hypervisor::Hypervisor;
-use crate::mem::layout::SandboxMemoryLayout;
 use crate::mem::mgr::SandboxMemoryManager;
 use crate::mem::ptr::{GuestPtr, RawPtr};
 use crate::mem::ptr_offset::Offset;
@@ -859,28 +858,14 @@ fn set_up_hypervisor_partition(
     }?;
     let base_ptr = GuestPtr::try_from(Offset::from(0))?;
     let pml4_ptr = {
-        let pml4_offset_u64 = u64::try_from(SandboxMemoryLayout::PML4_OFFSET)?;
+        let pml4_offset_u64 = u64::try_from(mgr.layout.get_pml4_offset())?;
         base_ptr + Offset::from(pml4_offset_u64)
     };
     let entrypoint_ptr = {
         let entrypoint_total_offset = mgr.load_addr.clone() + mgr.entrypoint_offset;
         GuestPtr::try_from(entrypoint_total_offset)
     }?;
 
-    if base_ptr != pml4_ptr {
-        log_then_return!(
-            "Error: base_ptr ({:#?}) does not equal pml4_ptr ({:#?})",
-            base_ptr,
-            pml4_ptr
-        );
-    }
-    if entrypoint_ptr <= pml4_ptr {
-        log_then_return!(
-            "Error: entrypoint_ptr ({:#?}) is not greater than pml4_ptr ({:#?})",
-            entrypoint_ptr,
-            pml4_ptr
-        );
-    }
     if mgr.is_in_process() {
         cfg_if::cfg_if! {
             if #[cfg(inprocess)] {

src/hyperlight_host/src/mem/layout.rs

@@ -61,16 +61,16 @@ use crate::{log_then_return, new_error, Result};
 // +-------------------------------------------+
 // |                PEB Struct (0x98)          |
 // +-------------------------------------------+
-// |               Guest Code                  |
-// +-------------------------------------------+
 // |                    PT                     |
-// +-------------------------------------------+ 0x3_000
+// +-------------------------------------------+ guest_code_offset + 0x3_000
 // |                    PD                     |
-// +-------------------------------------------+ 0x2_000
+// +-------------------------------------------+ guest_code_offset + 0x2_000
 // |                   PDPT                    |
-// +-------------------------------------------+ 0x1_000
+// +-------------------------------------------+ guest_code_offset + 0x1_000
 // |                   PML4                    |
-// +-------------------------------------------+ 0x0_000
+// +-------------------------------------------+ guest_code_offset
+// |               Guest Code                  |
+// +-------------------------------------------+ 0x0
 
 ///
 /// - `HostDefinitions` - the length of this is the `HostFunctionDefinitionSize`
@@ -160,6 +160,8 @@ pub(crate) struct SandboxMemoryLayout {
     total_page_table_size: usize,
     // The offset in the sandbox memory where the code starts
     guest_code_offset: usize,
+    // The offset in the sandbox memory where the PML4 Table is located
+    paging_sections_offset: usize,
 }
 
 impl Debug for SandboxMemoryLayout {
@@ -283,24 +285,13 @@ impl Debug for SandboxMemoryLayout {
 }
 
 impl SandboxMemoryLayout {
-    /// The offset into the sandbox's memory where the PML4 Table is located.
-    /// See https://www.pagetable.com/?p=14 for more information.
-    pub(crate) const PML4_OFFSET: usize = 0x0000;
-    /// The offset into the sandbox's memory where the Page Directory Pointer
-    /// Table starts.
-    pub(super) const PDPT_OFFSET: usize = 0x1000;
+    /// The offset from the start of the paging section region into the sandbox's memory where the
+    /// Page Directory Pointer Table starts.
+    const PDPT_OFFSET: usize = 0x1000;
     /// The offset into the sandbox's memory where the Page Directory starts.
-    pub(super) const PD_OFFSET: usize = 0x2000;
+    const PD_OFFSET: usize = 0x2000;
     /// The offset into the sandbox's memory where the Page Tables start.
-    pub(super) const PT_OFFSET: usize = 0x3000;
-    /// The address (not the offset) to the start of the page directory
-    pub(super) const PD_GUEST_ADDRESS: usize = Self::BASE_ADDRESS + Self::PD_OFFSET;
-    /// The address (not the offset) into sandbox memory where the Page
-    /// Directory Pointer Table starts
-    pub(super) const PDPT_GUEST_ADDRESS: usize = Self::BASE_ADDRESS + Self::PDPT_OFFSET;
-    /// The address (not the offset) into sandbox memory where the Page
-    /// Tables start
-    pub(super) const PT_GUEST_ADDRESS: usize = Self::BASE_ADDRESS + Self::PT_OFFSET;
+    const PT_OFFSET: usize = 0x3000;
     /// The maximum amount of memory a single sandbox will be allowed.
     /// The addressable virtual memory with current paging setup is virtual address 0x0 - 0x40000000,
     /// excluding the memory up to BASE_ADDRESS (which is 0 by default).
@@ -321,9 +312,10 @@ impl SandboxMemoryLayout {
         stack_size: usize,
         heap_size: usize,
     ) -> Result<Self> {
+        let guest_code_offset = 0x0;
         let total_page_table_size =
             Self::get_total_page_table_size(cfg, code_size, stack_size, heap_size);
-        let guest_code_offset = total_page_table_size;
+        let paging_sections_offset = guest_code_offset + round_up_to(code_size, PAGE_SIZE_USIZE);
         // The following offsets are to the fields of the PEB struct itself!
         let peb_offset = total_page_table_size + round_up_to(code_size, PAGE_SIZE_USIZE);
         let peb_security_cookie_seed_offset =
@@ -424,9 +416,34 @@ impl SandboxMemoryLayout {
             kernel_stack_guard_page_offset,
             kernel_stack_size_rounded,
             boot_stack_buffer_offset,
+            paging_sections_offset,
         })
     }
 
+    /// Gets the PML4 offset
+    /// (i.e., the `paging_sections_offset` == aligned code size)
+    pub fn get_pml4_offset(&self) -> usize {
+        self.paging_sections_offset
+    }
+
+    /// Gets the PDPT offset
+    /// (i.e., the `paging_sections_offset` + 0x1000)
+    pub fn get_pdpt_offset(&self) -> usize {
+        self.paging_sections_offset + Self::PDPT_OFFSET
+    }
+
+    /// Gets the PD offset
+    /// (i.e., the `paging_sections_offset` + 0x2000)
+    pub fn get_pd_offset(&self) -> usize {
+        self.paging_sections_offset + Self::PD_OFFSET
+    }
+
+    /// Gets the PT offset
+    /// (i.e., the `paging_sections_offset` + 0x3000)
+    pub fn get_pt_offset(&self) -> usize {
+        self.paging_sections_offset + Self::PT_OFFSET
+    }
+
     /// Gets the offset in guest memory to the RunMode field in the PEB struct.
     pub fn get_run_mode_offset(&self) -> usize {
         self.peb_runmode_offset
@@ -778,28 +795,22 @@ impl SandboxMemoryLayout {
     pub fn get_memory_regions(&self, shared_mem: &GuestSharedMemory) -> Result<Vec<MemoryRegion>> {
         let mut builder = MemoryRegionVecBuilder::new(Self::BASE_ADDRESS, shared_mem.base_addr());
 
-        // PML4, PDPT, PD
-        let code_offset = builder.push_page_aligned(
-            self.total_page_table_size,
-            MemoryRegionFlags::READ | MemoryRegionFlags::WRITE,
-            PageTables,
-        );
-
-        if code_offset != self.guest_code_offset {
-            return Err(new_error!(
-                "Code offset does not match expected code offset expected:  {}, actual:  {}",
-                self.guest_code_offset,
-                code_offset
-            ));
-        }
+        assert_eq!(self.guest_code_offset, 0x0);
 
-        // code
-        let peb_offset = builder.push_page_aligned(
+        // Code
+        builder.push_page_aligned(
             self.code_size,
             MemoryRegionFlags::READ | MemoryRegionFlags::WRITE | MemoryRegionFlags::EXECUTE,
             Code,
         );
 
+        // PML4, PDPT, PD
+        let peb_offset = builder.push_page_aligned(
+            self.total_page_table_size,
+            MemoryRegionFlags::READ | MemoryRegionFlags::WRITE,
+            PageTables,
+        );
+
         let expected_peb_offset = TryInto::<usize>::try_into(self.peb_offset)?;
 
         if peb_offset != expected_peb_offset {

src/hyperlight_host/src/mem/mgr.rs

@@ -152,22 +152,21 @@ where
             - 0x28;
 
         self.shared_mem.with_exclusivity(|shared_mem| {
-            // Create PDL4 table with only 1 PML4E
+            // Create PML4 table with only 1 PML4E
             shared_mem.write_u64(
-                SandboxMemoryLayout::PML4_OFFSET,
-                SandboxMemoryLayout::PDPT_GUEST_ADDRESS as u64 | PAGE_PRESENT | PAGE_RW,
+                self.layout.get_pml4_offset(),
+                self.layout.get_pdpt_offset() as u64 | PAGE_PRESENT | PAGE_RW,
             )?;
 
             // Create PDPT with only 1 PDPTE
             shared_mem.write_u64(
-                SandboxMemoryLayout::PDPT_OFFSET,
-                SandboxMemoryLayout::PD_GUEST_ADDRESS as u64 | PAGE_PRESENT | PAGE_RW,
+                self.layout.get_pdpt_offset(),
+                self.layout.get_pd_offset() as u64 | PAGE_PRESENT | PAGE_RW,
             )?;
 
             for i in 0..512 {
-                let offset = SandboxMemoryLayout::PD_OFFSET + (i * 8);
-                let val_to_write: u64 = (SandboxMemoryLayout::PT_GUEST_ADDRESS as u64
-                    + (i * 4096) as u64)
+                let offset = self.layout.get_pd_offset() + (i * 8);
+                let val_to_write: u64 = (self.layout.get_pt_offset() as u64 + (i * 4096) as u64)
                     | PAGE_PRESENT
                     | PAGE_RW;
                 shared_mem.write_u64(offset, val_to_write)?;
@@ -186,7 +185,7 @@ where
             // Create num_pages PT with 512 PTEs
             for p in 0..num_pages {
                 for i in 0..512 {
-                    let offset = SandboxMemoryLayout::PT_OFFSET + (p * 4096) + (i * 8);
+                    let offset = self.layout.get_pt_offset() + (p * 4096) + (i * 8);
                     // Each PTE maps a 4KB page
                     let val_to_write = {
                         let flags = match Self::get_page_flags(p, i, regions) {