page allocator: support allocating pages within an address range (theseus-os#970)

* Currently this is only used for allocating pages for new
  executable .text sections on aarch64, which itself is a
  workaround to enable runtime loading of crates (see theseus-os#940).
  * Based on the limitations of aarch64's ISA (branch instructions),
    we reserve 128MiB of virtual address space for this purpose.
  * This 128MiB region is for executable .text sections only,
    and is contiguous with the base kernel image's .text section.

* This is available but not used by default on x86_64 yet.

Author: kevinaboos

kernel/kernel_config/src/memory.rs

@@ -1,4 +1,4 @@
-//! The basic virtual memory map that Theseus assumes.
+//! The basic virtual address ranges (virtual memory map) defined by Theseus.
 //!
 //! Current P4 (top-level page table) mappings:
 //! * 511: kernel text sections.
@@ -53,15 +53,15 @@ pub const TEMPORARY_PAGE_VIRT_ADDR: usize = MAX_VIRTUAL_ADDRESS;
 
 /// Value: 512.
 pub const ENTRIES_PER_PAGE_TABLE: usize = PAGE_SIZE / BYTES_PER_ADDR;
-/// Value: 511. The 511th entry is used for kernel text sections
+/// Value: 511. The 511th entry is used (in part) for kernel text sections.
 pub const KERNEL_TEXT_P4_INDEX: usize = ENTRIES_PER_PAGE_TABLE - 1;
 /// Value: 510. The 510th entry is used to recursively map the current P4 root page table frame
-//              such that it can be accessed and modified just like any other level of page table.
+///             such that it can be accessed and modified just like any other level of page table.
 pub const RECURSIVE_P4_INDEX: usize = ENTRIES_PER_PAGE_TABLE - 2;
-/// Value: 509. The 509th entry is used for the kernel heap
+/// Value: 509. The 509th entry is used for the kernel heap.
 pub const KERNEL_HEAP_P4_INDEX: usize = ENTRIES_PER_PAGE_TABLE - 3;
 /// Value: 508. The 508th entry is used to temporarily recursively map the P4 root page table frame
-//              of an upcoming (new) page table such that it can be accessed and modified.
+///             of an upcoming (new) page table such that it can be accessed and modified.
 pub const UPCOMING_PAGE_TABLE_RECURSIVE_P4_INDEX: usize = ENTRIES_PER_PAGE_TABLE - 4;
 
 
@@ -89,12 +89,9 @@ pub const KERNEL_OFFSET: usize = canonicalize(MAX_VIRTUAL_ADDRESS - (TWO_GIGABYT
 /// Actual value on x86_64: 0o177777_777_000_000_000_0000, or 0xFFFF_FF80_0000_0000
 pub const KERNEL_TEXT_START: usize = canonicalize(KERNEL_TEXT_P4_INDEX << (P4_INDEX_SHIFT + PAGE_SHIFT));
 
-/// The size in bytes, not in pages.
-///
-/// the KERNEL_OFFSET starts at (MAX_ADDR - 2GiB),
-/// and .text contains nano_core, so this is the
-/// first 510GiB of the 511th P4 entry.
-pub const KERNEL_TEXT_MAX_SIZE: usize = ADDRESSABILITY_PER_P4_ENTRY - TWO_GIGABYTES;
+/// The start of the virtual address range covered by the 510th P4 entry,
+/// i.e., [`RECURSIVE_P4_INDEX`];
+pub const RECURSIVE_P4_START: usize = canonicalize(RECURSIVE_P4_INDEX << (P4_INDEX_SHIFT + PAGE_SHIFT));
 
 /// The higher-half heap gets the 512GB address range starting at the 509th P4 entry,
 /// which is the slot right below the recursive P4 entry (510).
@@ -103,12 +100,12 @@ pub const KERNEL_HEAP_START: usize = canonicalize(KERNEL_HEAP_P4_INDEX << (P4_IN
 
 #[cfg(not(debug_assertions))]
 pub const KERNEL_HEAP_INITIAL_SIZE: usize = 64 * 1024 * 1024; // 64 MiB
-
 #[cfg(debug_assertions)]
 pub const KERNEL_HEAP_INITIAL_SIZE: usize = 256 * 1024 * 1024; // 256 MiB, debug builds require more heap space.
 
-/// the kernel heap gets the whole 509th P4 entry.
+/// The kernel heap is allowed to grow to fill the entirety of its P4 entry.
 pub const KERNEL_HEAP_MAX_SIZE: usize = ADDRESSABILITY_PER_P4_ENTRY;
 
-/// The system (page allocator) must not use addresses at or above this address.
-pub const UPCOMING_PAGE_TABLE_RECURSIVE_MEMORY_START: usize = canonicalize(UPCOMING_PAGE_TABLE_RECURSIVE_P4_INDEX << (P4_INDEX_SHIFT + PAGE_SHIFT));
+/// The start of the virtual address range covered by the 508th P4 entry,
+/// i.e., [`UPCOMING_PAGE_TABLE_RECURSIVE_P4_INDEX`];
+pub const UPCOMING_PAGE_TABLE_RECURSIVE_P4_START: usize = canonicalize(UPCOMING_PAGE_TABLE_RECURSIVE_P4_INDEX << (P4_INDEX_SHIFT + PAGE_SHIFT));

kernel/memory/src/lib.rs

@@ -23,15 +23,8 @@ pub use self::paging::{
 };
 
 pub use memory_structs::*;
-pub use page_allocator::{
-    AllocatedPages, allocate_pages, allocate_pages_at,
-    allocate_pages_by_bytes, allocate_pages_by_bytes_at,
-};
-
-pub use frame_allocator::{
-    AllocatedFrames, MemoryRegionType, PhysicalMemoryRegion,
-    allocate_frames, allocate_frames_at, allocate_frames_by_bytes_at, allocate_frames_by_bytes,
-};
+pub use page_allocator::*;
+pub use frame_allocator::*;
 
 #[cfg(target_arch = "x86_64")]
 use memory_x86_64::{ tlb_flush_virt_addr, tlb_flush_all, get_p4, find_section_memory_bounds, get_vga_mem_addr };

kernel/mod_mgmt/src/lib.rs

@@ -14,7 +14,7 @@ use alloc::{
 };
 use spin::{Mutex, Once};
 use xmas_elf::{ElfFile, sections::{SHF_ALLOC, SHF_EXECINSTR, SHF_TLS, SHF_WRITE, SectionData, ShType}, symbol_table::{Binding, Type}};
-use memory::{MmiRef, MemoryManagementInfo, VirtualAddress, MappedPages, PteFlags, allocate_pages_by_bytes, allocate_frames_by_bytes_at};
+use memory::{MmiRef, MemoryManagementInfo, VirtualAddress, MappedPages, PteFlags, allocate_pages_by_bytes, allocate_frames_by_bytes_at, PageRange, allocate_pages_by_bytes_in_range};
 use bootloader_modules::BootloaderModule;
 use cow_arc::CowArc;
 use rustc_demangle::demangle;
@@ -33,6 +33,7 @@ pub mod parse_nano_core;
 pub mod replace_nano_core_crates;
 mod serde;
 
+
 /// The name of the directory that contains all of the CrateNamespace files.
 pub const NAMESPACES_DIRECTORY_NAME: &str = "namespaces";
 
@@ -2882,6 +2883,35 @@ struct SectionPages {
 }
 
 
+/// The range of virtual addresses from which we allocate pages for executable .text sections.
+///
+/// This is mostly an architecture-specific design choice (hopefully a temporary one):
+/// * On aarch64, even with the large code model, we are not (yet) able to generate
+///   code with branch instructions (call/jump) that can address instructions more than
+///   128 MiB away from the current instruction.
+///   Thus, we restrict the range of .text section locations to ensure they are within 128 MiB.
+///   At some point in the future, this will be a limitation, but not for a long, long time.
+/// * On x86_64, this is not necessary, so the range is `None`.
+pub const KERNEL_TEXT_ADDR_RANGE: Option<PageRange> = {
+    #[cfg(target_arch = "x86_64")] {
+        None
+    }
+    #[cfg(target_arch = "aarch64")] {
+        use {memory::Page, kernel_config::memory::KERNEL_OFFSET};
+
+        const ONE_MIB: usize = 0x10_0000;
+        let start_vaddr = VirtualAddress::new_canonical(KERNEL_OFFSET + ONE_MIB);
+        let end_vaddr = VirtualAddress::new_canonical(start_vaddr.value() + (128 * ONE_MIB) - 1);
+        Some(PageRange::new(
+            // the start of the base kernel image's .text section.
+            Page::containing_address(start_vaddr),
+            // the start of the base kernel image's .text section, plus 128 MiB.
+            Page::containing_address(end_vaddr),
+        ))
+    }
+};
+
+
 /// Allocates and maps memory sufficient to hold the sections that are found in the given `ElfFile`.
 /// Only sections that are marked "allocated" (`ALLOC`) in the ELF object file will contribute to the mappings' sizes.
 fn allocate_section_pages(elf_file: &ElfFile, kernel_mmi_ref: &MmiRef) -> Result<SectionPages, &'static str> {
@@ -2953,10 +2983,37 @@ fn allocate_section_pages(elf_file: &ElfFile, kernel_mmi_ref: &MmiRef) -> Result
     // trace!("\n\texec_bytes: {exec_bytes} {exec_bytes:#X}\n\tro_bytes:   {ro_bytes} {ro_bytes:#X}\n\trw_bytes:   {rw_bytes} {rw_bytes:#X}");
 
     // Allocate contiguous virtual memory pages for each section and map them to random frames as writable.
-    // We must allocate these pages separately because they will have different flags later.
-    let executable_pages = if exec_bytes > 0 { Some(allocate_and_map_as_writable(exec_bytes, TEXT_SECTION_FLAGS,     kernel_mmi_ref)?) } else { None };
-    let read_only_pages  = if ro_bytes   > 0 { Some(allocate_and_map_as_writable(ro_bytes,   RODATA_SECTION_FLAGS,   kernel_mmi_ref)?) } else { None };
-    let read_write_pages = if rw_bytes   > 0 { Some(allocate_and_map_as_writable(rw_bytes,   DATA_BSS_SECTION_FLAGS, kernel_mmi_ref)?) } else { None };
+    // We must allocate these pages separately because they use different flags.
+    let alloc_sec = |size_in_bytes: usize, within_range: Option<&PageRange>, flags: PteFlags| {
+        let allocated_pages = if let Some(range) = within_range {
+            allocate_pages_by_bytes_in_range(size_in_bytes, range)
+                .map_err(|_| "Couldn't allocate pages in text section address range")?
+        } else {
+            allocate_pages_by_bytes(size_in_bytes)
+                .ok_or("Couldn't allocate pages for new section")?
+        };
+    
+        kernel_mmi_ref.lock().page_table.map_allocated_pages(
+            allocated_pages,
+            flags.valid(true).writable(true)
+        )
+    };
+
+    let executable_pages = if exec_bytes > 0 {
+        Some(alloc_sec(exec_bytes, KERNEL_TEXT_ADDR_RANGE.as_ref(), TEXT_SECTION_FLAGS)?)
+    } else {
+        None
+    };
+    let read_only_pages  = if ro_bytes > 0 {
+        Some(alloc_sec(ro_bytes, None, RODATA_SECTION_FLAGS)?)
+    } else {
+        None
+    };
+    let read_write_pages = if rw_bytes > 0 {
+        Some(alloc_sec(rw_bytes, None, DATA_BSS_SECTION_FLAGS)?)
+    } else {
+        None
+    };
 
     let range_tuple = |mp: MappedPages, size_in_bytes: usize| {
         let start = mp.start_address();
@@ -2971,26 +3028,6 @@ fn allocate_section_pages(elf_file: &ElfFile, kernel_mmi_ref: &MmiRef) -> Result
 }
 
 
-/// A convenience function for allocating virtual pages and mapping them to random physical frames. 
-/// 
-/// The returned `MappedPages` will be at least as large as `size_in_bytes`,
-/// rounded up to the nearest `Page` size, 
-/// and is mapped as writable along with the other specified `flags`
-/// to ensure we can copy content into it.
-fn allocate_and_map_as_writable(
-    size_in_bytes: usize,
-    flags: PteFlags,
-    kernel_mmi_ref: &MmiRef,
-) -> Result<MappedPages, &'static str> {
-    let allocated_pages = allocate_pages_by_bytes(size_in_bytes)
-        .ok_or("Couldn't allocate_pages_by_bytes, out of virtual address space")?;
-    kernel_mmi_ref.lock().page_table.map_allocated_pages(
-        allocated_pages,
-        flags.valid(true).writable(true)
-    )
-}
-
-
 #[allow(dead_code)]
 fn dump_dependent_crates(krate: &LoadedCrate, prefix: String) {
 	for weak_crate_ref in krate.crates_dependent_on_me() {

kernel/nano_core/linker_higher_half-aarch64.ld

@@ -27,6 +27,18 @@ SECTIONS {
         *(.text .text.*)
     }
 
+    /*
+     * Currently, we are unable to force aarch64 to emit branch (call/jump) instructions
+     * that are capable of addressing a destination instruction pointer more than 128MiB away,
+     * even when specifying the "large" code model with `-C code-model=large`.
+     *
+     * Thus, as a workaround, we reserve the 128MiB chunk of virtual address space that
+     * directly follows the initial base kernel image's executable .text section,
+     * ensuring it can only be used by the page allocator when allocating pages for
+     * newly-loaded .text sections.
+     */
+    . = ALIGN(128M);
+
     .rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET)
     {
         *(.rodata .rodata.*)