feat(mm): offset device memory when being careful

Author: mkroening

Cargo.toml

@@ -73,6 +73,7 @@ vsock = ["pci"]
 
 [lints.rust]
 rust_2018_idioms = "warn"
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(careful)'] }
 unsafe_op_in_unsafe_fn = "warn"
 
 [lints.clippy]

src/mm/device_alloc.rs

@@ -2,9 +2,10 @@ use core::alloc::{AllocError, Allocator, Layout};
 use core::ptr::{self, NonNull};
 
 use align_address::Align;
-use memory_addresses::PhysAddr;
+use memory_addresses::{PhysAddr, VirtAddr};
 
 use crate::arch::mm::paging::{BasePageSize, PageSize};
+use crate::mm::virtualmem;
 
 /// An [`Allocator`] for memory that is used to communicate with devices.
 ///
@@ -18,7 +19,7 @@ unsafe impl Allocator for DeviceAlloc {
 
 		let phys_addr = super::physicalmem::allocate(size).unwrap();
 
-		let ptr = self.ptr(phys_addr.as_usize());
+		let ptr = self.ptr(phys_addr);
 		let slice = ptr::slice_from_raw_parts_mut(ptr, size);
 		Ok(NonNull::new(slice).unwrap())
 	}
@@ -35,15 +36,27 @@ unsafe impl Allocator for DeviceAlloc {
 
 impl DeviceAlloc {
 	/// Returns a pointer corresponding to `phys_addr`.
-	pub fn ptr<T>(&self, phys_addr: usize) -> *mut T {
-		ptr::with_exposed_provenance_mut(phys_addr)
+	pub fn ptr<T>(&self, phys_addr: PhysAddr) -> *mut T {
+		let addr = phys_addr.as_usize() + self.phys_offset().as_usize();
+		ptr::with_exposed_provenance_mut(addr)
 	}
 
 	/// Returns the physical address of `ptr`.
 	///
 	/// The address is only correct if `ptr` has been allocated by this allocator.
 	pub fn phys_addr<T: ?Sized>(&self, ptr: *mut T) -> PhysAddr {
-		let addr = u64::try_from(ptr.expose_provenance()).unwrap();
+		let addr = u64::try_from(ptr.expose_provenance()).unwrap() - self.phys_offset().as_u64();
 		PhysAddr::new(addr)
 	}
+
+	/// Returns the physical address offset.
+	///
+	/// This device allocator expects the complete physical memory to be mapped device-readable at this offset.
+	pub fn phys_offset(&self) -> VirtAddr {
+		if cfg!(careful) {
+			virtualmem::kernel_heap_end().as_u64().div_ceil(4).into()
+		} else {
+			0u64.into()
+		}
+	}
 }

src/mm/mod.rs

@@ -1,5 +1,4 @@
 pub(crate) mod allocator;
-#[cfg(any(feature = "tcp", feature = "udp", feature = "fuse", feature = "vsock"))]
 pub(crate) mod device_alloc;
 pub(crate) mod physicalmem;
 pub(crate) mod virtualmem;

src/mm/physicalmem.rs

@@ -5,8 +5,11 @@ use free_list::{AllocError, FreeList, PageLayout, PageRange};
 use hermit_sync::InterruptTicketMutex;
 use memory_addresses::{PhysAddr, VirtAddr};
 
-use crate::arch::mm::paging::{self, BasePageSize, PageSize};
+#[cfg(target_arch = "x86_64")]
+use crate::arch::mm::paging::PageTableEntryFlagsExt;
+use crate::arch::mm::paging::{self, BasePageSize, HugePageSize, PageSize, PageTableEntryFlags};
 use crate::env;
+use crate::mm::device_alloc::DeviceAlloc;
 
 pub static PHYSICAL_FREE_LIST: InterruptTicketMutex<FreeList<16>> =
 	InterruptTicketMutex::new(FreeList::new());
@@ -43,6 +46,21 @@ pub unsafe fn init_frame_range(frame_range: PageRange) {
 		.map(|addr| PhysAddr::new(addr.try_into().unwrap()))
 		.for_each(paging::identity_map::<IdentityPageSize>);
 
+	if DeviceAlloc.phys_offset() != VirtAddr::zero() {
+		let flags = {
+			let mut flags = PageTableEntryFlags::empty();
+			flags.normal().writable().execute_disable();
+			flags
+		};
+		(start..end)
+			.step_by(IdentityPageSize::SIZE.try_into().unwrap())
+			.for_each(|addr| {
+				let phys_addr = PhysAddr::new(addr.try_into().unwrap());
+				let virt_addr = VirtAddr::from_ptr(DeviceAlloc.ptr::<()>(phys_addr));
+				paging::map::<IdentityPageSize>(virt_addr, phys_addr, 1, flags);
+			});
+	}
+
 	TOTAL_MEMORY.fetch_add(frame_range.len().get(), Ordering::Relaxed);
 }
 
@@ -118,6 +136,13 @@ fn detect_from_limits() -> Result<(), ()> {
 }
 
 pub fn init() {
+	if env::is_uefi() && DeviceAlloc.phys_offset() != VirtAddr::zero() {
+		let start = DeviceAlloc.phys_offset();
+		let count = DeviceAlloc.phys_offset().as_u64() / HugePageSize::SIZE;
+		let count = usize::try_from(count).unwrap();
+		paging::unmap::<HugePageSize>(start, count);
+	}
+
 	if let Err(_err) = detect_from_fdt() {
 		cfg_if::cfg_if! {
 			if #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] {