diff --git a/src/aarch64/asm.rs b/src/aarch64/asm.rs index f556d3d..aa54f1b 100644 --- a/src/aarch64/asm.rs +++ b/src/aarch64/asm.rs @@ -64,7 +64,8 @@ pub fn read_kernel_page_table() -> PhysAddr { /// Reads the current page table root register for user space (`TTBR0_EL1`). /// /// When the "arm-el2" feature is enabled, for user-mode programs, -/// virtualization is completely transparent to them, so there is no need to modify +/// virtualization is completely transparent to them, so there is no need to +/// modify /// /// Returns the physical address of the page table root. #[inline] @@ -88,13 +89,15 @@ pub fn read_user_page_table() -> PhysAddr { pub unsafe fn write_kernel_page_table(root_paddr: PhysAddr) { #[cfg(not(feature = "arm-el2"))] { - // kernel space page table use TTBR1 (0xffff_0000_0000_0000..0xffff_ffff_ffff_ffff) + // kernel space page table use TTBR1 + // (0xffff_0000_0000_0000..0xffff_ffff_ffff_ffff) TTBR1_EL1.set(root_paddr.as_usize() as _); } #[cfg(feature = "arm-el2")] { - // kernel space page table at EL2 use TTBR0_EL2 (0x0000_0000_0000_0000..0x0000_ffff_ffff_ffff) + // kernel space page table at EL2 use TTBR0_EL2 + // (0x0000_0000_0000_0000..0x0000_ffff_ffff_ffff) TTBR0_EL2.set(root_paddr.as_usize() as _); } } @@ -102,7 +105,8 @@ pub unsafe fn write_kernel_page_table(root_paddr: PhysAddr) { /// Writes the register to update the current page table root for user space /// (`TTBR1_EL0`). /// When the "arm-el2" feature is enabled, for user-mode programs, -/// virtualization is completely transparent to them, so there is no need to modify +/// virtualization is completely transparent to them, so there is no need to +/// modify /// /// Note that the TLB is **NOT** flushed after this operation. /// @@ -201,3 +205,19 @@ pub fn enable_fp() { CPACR_EL1.write(CPACR_EL1::FPEN::TrapNothing); barrier::isb(barrier::SY); } + +core::arch::global_asm!(include_str!("user_copy.S")); + +extern "C" { + /// Copy data from user space to kernel space safely. + /// + /// # Arguments + /// * `dst` - Destination pointer in kernel space + /// * `src` - Source pointer in user space + /// * `size` - Number of bytes to copy + /// + /// # Returns + /// * `0` - Success, all bytes copied + /// * `>0` - Number of bytes that could not be copied due to page fault + pub fn user_copy(dst: *mut u8, src: *const u8, size: usize) -> usize; +} diff --git a/src/aarch64/context.rs b/src/aarch64/context.rs index f2869dd..970d062 100644 --- a/src/aarch64/context.rs +++ b/src/aarch64/context.rs @@ -1,5 +1,5 @@ -use core::arch::naked_asm; -use core::fmt; +use core::{arch::naked_asm, fmt}; + use memory_addr::VirtAddr; /// Saved registers when a trap (exception) occurs. @@ -124,6 +124,16 @@ impl TrapFrame { self.r[0] as _ } + /// Get the syscall number. + pub const fn sysno(&self) -> usize { + self.r[8] as usize + } + + /// Sets the syscall number. + pub const fn set_sysno(&mut self, sysno: usize) { + self.r[8] = sysno as _; + } + /// Sets the return value register. pub const fn set_retval(&mut self, r0: usize) { self.r[0] = r0 as _; @@ -235,8 +245,9 @@ impl TaskContext { /// Changes the page table root in this context. /// - /// The hardware register for user page table root (`ttbr0_el1` for aarch64 in EL1) - /// will be updated to the next task's after [`Self::switch_to`]. + /// The hardware register for user page table root (`ttbr0_el1` for aarch64 + /// in EL1) will be updated to the next task's after + /// [`Self::switch_to`]. #[cfg(feature = "uspace")] pub fn set_page_table_root(&mut self, ttbr0_el1: memory_addr::PhysAddr) { self.ttbr0_el1 = ttbr0_el1; @@ -244,8 +255,8 @@ impl TaskContext { /// Switches to another task. /// - /// It first saves the current task's context from CPU to this place, and then - /// restores the next task's context from `next_ctx` to CPU. + /// It first saves the current task's context from CPU to this place, and + /// then restores the next task's context from `next_ctx` to CPU. pub fn switch_to(&mut self, next_ctx: &Self) { #[cfg(feature = "tls")] { diff --git a/src/aarch64/trap.S b/src/aarch64/trap.S index 6638e7f..1eb82c5 100644 --- a/src/aarch64/trap.S +++ b/src/aarch64/trap.S @@ -1,5 +1,4 @@ .macro SAVE_REGS - sub sp, sp, {trapframe_size} stp x0, x1, [sp] stp x2, x3, [sp, 2 * 8] stp x4, x5, [sp, 4 * 8] @@ -36,7 +35,7 @@ # backup kernel tpidr_el0 mrs x1, tpidr_el0 msr tpidrro_el0, x1 - + ldp x11, x12, [sp, 33 * 8] ldp x9, x10, [sp, 31 * 8] msr sp_el0, x9 @@ -65,6 +64,7 @@ .macro INVALID_EXCP, kind, source .p2align 7 + sub sp, sp, {trapframe_size} SAVE_REGS mov x0, sp mov x1, \kind @@ -73,8 +73,20 @@ b .Lexception_return .endm +.macro TASK_EXIT, kind +.p2align 7 + SAVE_REGS + mov x0, \kind + b _user_trap_entry + +.endm + +.Ldeadloop: + b .Ldeadloop + .macro HANDLE_SYNC .p2align 7 + sub sp, sp, {trapframe_size} SAVE_REGS mov x0, sp bl handle_sync_exception @@ -83,6 +95,7 @@ .macro HANDLE_IRQ .p2align 7 + sub sp, sp, {trapframe_size} SAVE_REGS mov x0, sp bl handle_irq_exception @@ -93,30 +106,38 @@ .p2align 11 .global exception_vector_base exception_vector_base: - // current EL, with SP_EL0 - INVALID_EXCP 0 0 - INVALID_EXCP 1 0 - INVALID_EXCP 2 0 - INVALID_EXCP 3 0 - - // current EL, with SP_ELx - HANDLE_SYNC - HANDLE_IRQ - INVALID_EXCP 2 1 - INVALID_EXCP 3 1 - - // TODO: handle user trap - // lower EL, aarch64 - HANDLE_SYNC - HANDLE_IRQ - INVALID_EXCP 2 2 - INVALID_EXCP 3 2 - - // lower EL, aarch32 - INVALID_EXCP 0 3 - INVALID_EXCP 1 3 - INVALID_EXCP 2 3 - INVALID_EXCP 3 3 +curr_el_sp0_sync: + INVALID_EXCP 0 0 +curr_el_sp0_irq: + INVALID_EXCP 1 0 +curr_el_sp0_fiq: + INVALID_EXCP 2 0 +curr_el_sp0_serror: + INVALID_EXCP 3 0 +curr_el_spx_sync: + HANDLE_SYNC +curr_el_spx_irq: + HANDLE_IRQ +curr_el_spx_fiq: + INVALID_EXCP 2 1 +curr_el_spx_serror: + INVALID_EXCP 3 1 +lower_el_aarch64_sync: + TASK_EXIT {kind_sync} +lower_el_aarch64_irq: + TASK_EXIT {kind_irq} +lower_el_aarch64_fiq: + INVALID_EXCP 2 2 +lower_el_aarch64_serror: + INVALID_EXCP 3 2 +lower_el_aarch32_sync: + INVALID_EXCP 0 3 +lower_el_aarch32_irq: + INVALID_EXCP 1 3 +lower_el_aarch32_fiq: + INVALID_EXCP 2 3 +lower_el_aarch32_serror: + INVALID_EXCP 3 3 .Lexception_return: RESTORE_REGS diff --git a/src/aarch64/trap.rs b/src/aarch64/trap.rs index 2c711c7..b6776ba 100644 --- a/src/aarch64/trap.rs +++ b/src/aarch64/trap.rs @@ -6,17 +6,19 @@ use crate::trap::PageFaultFlags; core::arch::global_asm!( include_str!("trap.S"), - trapframe_size = const core::mem::size_of::() + trapframe_size = const core::mem::size_of::(), + kind_irq = const TrapKind::Irq as u8, + kind_sync = const TrapKind::Synchronous as u8, ); #[repr(u8)] #[derive(Debug)] #[allow(dead_code)] -enum TrapKind { +pub(crate) enum TrapKind { Synchronous = 0, - Irq = 1, - Fiq = 2, - SError = 3, + Irq = 1, + Fiq = 2, + SError = 3, } #[repr(u8)] diff --git a/src/aarch64/user_copy.S b/src/aarch64/user_copy.S new file mode 100644 index 0000000..abb7eca --- /dev/null +++ b/src/aarch64/user_copy.S @@ -0,0 +1,76 @@ +// AArch64 user space safe memory copy +// Based on RISC-V implementation and optimized for AArch64 + +.section .text +.global user_copy +user_copy: + // x0 = dst, x1 = src, x2 = size + // Return: 0 on success, remaining bytes on fault + + // Quick exit for zero size + cbz x2, .Lsuccess + + // Save the end address for fault handling + add x3, x0, x2 // x3 = dst_end + mov x4, x2 // x4 = original_size (for fault calculation) + + // Use different strategies based on size + cmp x2, #64 + b.lo .Lbyte_copy + + // For larger copies, try to align and use word copy + // First align dst to 8-byte boundary + and x5, x0, #7 + cbz x5, .Laligned_copy + + // Copy bytes until dst is 8-byte aligned + sub x5, x2, x5 // remaining after alignment +.Lalign_loop: + ldrb w6, [x1], #1 + strb w6, [x0], #1 + tbnz x0, #0, .Lalign_loop + tbnz x0, #1, .Lalign_loop + tbnz x0, #2, .Lalign_loop + mov x2, x5 // update remaining size + +.Laligned_copy: + // Now dst is 8-byte aligned, do word copy + cmp x2, #8 + b.lo .Lbyte_copy + + // Unrolled 8-word (64-byte) copy loop + sub x5, x2, #64 +.Lword_loop: + ldp x6, x7, [x1], #16 + ldp x8, x9, [x1], #16 + ldp x10, x11, [x1], #16 + ldp x12, x13, [x1], #16 + stp x6, x7, [x0], #16 + stp x8, x9, [x0], #16 + stp x10, x11, [x0], #16 + stp x12, x13, [x0], #16 + subs x2, x2, #64 + cmp x2, #64 + b.hs .Lword_loop + + // Handle remaining words (8-byte chunks) +.Lword_remainder: + cmp x2, #8 + b.lo .Lbyte_copy + ldr x6, [x1], #8 + str x6, [x0], #8 + sub x2, x2, #8 + b .Lword_remainder + +.Lbyte_copy: + // Copy remaining bytes + cbz x2, .Lsuccess +.Lbyte_loop: + ldrb w6, [x1], #1 + strb w6, [x0], #1 + subs x2, x2, #1 + b.ne .Lbyte_loop + +.Lsuccess: + mov x0, #0 + ret \ No newline at end of file diff --git a/src/aarch64/uspace.rs b/src/aarch64/uspace.rs index a7d9f7c..76807b7 100644 --- a/src/aarch64/uspace.rs +++ b/src/aarch64/uspace.rs @@ -1,58 +1,98 @@ //! Structures and functions for user space. +use core::{ + arch::naked_asm, + mem::offset_of, + ops::{Deref, DerefMut}, +}; -use core::ops::{Deref, DerefMut}; - -use aarch64_cpu::registers::ESR_EL1; +use aarch64_cpu::registers::{ESR_EL1, FAR_EL1, Readable}; use memory_addr::VirtAddr; +use page_table_entry::MappingFlags; +use tock_registers::LocalRegisterCopy; use crate::{ - trap::{ExceptionKind, ReturnReason}, TrapFrame, + aarch64::trap::TrapKind, + trap::{ExceptionKind, ReturnReason}, }; -/// Context to enter user space. -pub struct UserContext(TrapFrame); +#[derive(Debug, Clone, Copy)] +pub struct ExceptionInfo { + pub esr: LocalRegisterCopy, + pub stval: usize, +} -impl UserContext { - /// Creates an empty context with all registers set to zero. - pub const fn empty() -> Self { - Self(TrapFrame::new()) +impl ExceptionInfo { + pub fn kind(&self) -> ExceptionKind { + match self.esr.read_as_enum(ESR_EL1::EC) { + Some(ESR_EL1::EC::Value::BreakpointLowerEL) => ExceptionKind::Breakpoint, + Some(ESR_EL1::EC::Value::IllegalExecutionState) => ExceptionKind::IllegalInstruction, + Some(ESR_EL1::EC::Value::PCAlignmentFault) + | Some(ESR_EL1::EC::Value::SPAlignmentFault) => ExceptionKind::Misaligned, + _ => ExceptionKind::Other, + } } +} - /// Creates a new context with the given entry point, user stack pointer, - /// and the argument. - pub fn new(entry: usize, ustack_top: VirtAddr, arg0: usize) -> Self { - use aarch64_cpu::registers::SPSR_EL1; - let mut regs = [0; 31]; - regs[0] = arg0 as _; - Self(TrapFrame { - r: regs, - usp: ustack_top.as_usize() as _, - tpidr: 0, - elr: entry as _, - spsr: (SPSR_EL1::M::EL0t - + SPSR_EL1::D::Masked - + SPSR_EL1::A::Masked - + SPSR_EL1::I::Unmasked - + SPSR_EL1::F::Masked) - .value, - }) - } +#[repr(C)] +#[derive(Debug, Clone)] +pub struct UserContext { + tf: TrapFrame, + sp_el1: u64, +} + +impl UserContext { + pub fn run(&mut self) -> ReturnReason { + let tp_kind = unsafe { _enter_user(self) }; + + if matches!(tp_kind, TrapKind::Irq) { + handle_trap!(IRQ, 0); + return ReturnReason::Interrupt; + } - /// Creates a new context from the given [`TrapFrame`]. - pub const fn from(trap_frame: &TrapFrame) -> Self { - Self(*trap_frame) + let esr = ESR_EL1.extract(); + let iss = esr.read(ESR_EL1::ISS); + + match esr.read_as_enum(ESR_EL1::EC) { + Some(ESR_EL1::EC::Value::SVC64) => ReturnReason::Syscall, + Some(ESR_EL1::EC::Value::InstrAbortLowerEL) => { + handle_instruction_abort_lower(&self.tf, iss, true) + } + Some(ESR_EL1::EC::Value::BreakpointLowerEL) + | Some(ESR_EL1::EC::Value::IllegalExecutionState) + | Some(ESR_EL1::EC::Value::PCAlignmentFault) + | Some(ESR_EL1::EC::Value::SPAlignmentFault) => { + ReturnReason::Exception(ExceptionInfo { + esr, + stval: FAR_EL1.get() as usize, + }) + } + Some(ESR_EL1::EC::Value::DataAbortLowerEL) => { + info!("task return because DataAbortLowerEL ..."); + handle_data_abort_lower(&self.tf, iss, true) + } + _ => ReturnReason::Unknown, + } } - /// Enters user space. - /// - /// It restores the user registers and jumps to the user entry point - /// (saved in `elr`). - /// - /// This function returns when an exception or syscall occurs. - pub fn run(&mut self) -> ReturnReason { - // TODO: implement - ReturnReason::Unknown + pub fn new(entry: usize, ustack_top: VirtAddr, arg0: usize) -> Self { + info!( + "new ctx: entry={:#x}, ustack_top={:#x}", + entry, + ustack_top.as_usize() + ); + let mut r = [0u64; 31]; + r[0] = arg0 as u64; + Self { + tf: TrapFrame { + r, + usp: ustack_top.as_usize() as u64, // 假设 VirtAddr 有 as_u64 方法 + tpidr: 0, + elr: entry as u64, + spsr: 0, // recommend to set to 0 + }, + sp_el1: 0, // stack pointer for EL1, will be set in _enter_user + } } } @@ -60,32 +100,154 @@ impl Deref for UserContext { type Target = TrapFrame; fn deref(&self) -> &Self::Target { - &self.0 + &self.tf } } impl DerefMut for UserContext { fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 + &mut self.tf } } impl From for UserContext { fn from(tf: TrapFrame) -> Self { - Self(tf) + Self { + tf, + sp_el1: 0, // 默认初始化 + } } } -#[derive(Debug, Clone, Copy)] -pub struct ExceptionInfo { - pub ec: ESR_EL1::EC::Value, - pub il: bool, - pub iss: u32, +fn handle_instruction_abort_lower(tf: &TrapFrame, iss: u64, is_user: bool) -> ReturnReason { + let mut access_flags = MappingFlags::EXECUTE; + if is_user { + access_flags |= MappingFlags::USER; + } + let vaddr = va!(FAR_EL1.get() as usize); + + // Only handle Translation fault and Permission fault + if !matches!(iss & 0b111100, 0b0100 | 0b1100) + // IFSC or DFSC bits + { + panic!( + "Unhandled {} Instruction Abort @ {:#x}, fault_vaddr={:#x}, ESR={:#x} \ + ({:?}):\n{:#x?}\n{}", + if is_user { "EL0" } else { "EL1" }, + tf.elr, + vaddr, + ESR_EL1.get(), + access_flags, + tf, + tf.backtrace() + ); + } else { + ReturnReason::PageFault(vaddr, access_flags) + } } -impl ExceptionInfo { - pub fn kind(&self) -> ExceptionKind { - // TODO: implement - ExceptionKind::Other +fn handle_data_abort_lower(tf: &TrapFrame, iss: u64, is_user: bool) -> ReturnReason { + let wnr = (iss & (1 << 6)) != 0; // WnR: Write not Read + let cm = (iss & (1 << 8)) != 0; // CM: Cache maintenance + let mut access_flags = if wnr & !cm { + MappingFlags::WRITE + } else { + MappingFlags::READ + }; + if is_user { + access_flags |= MappingFlags::USER; + } + let vaddr = va!(FAR_EL1.get() as usize); + + // Only handle Translation fault and Permission fault + if !matches!(iss & 0b111100, 0b0100 | 0b1100) + // IFSC or DFSC bits + { + panic!( + "Unhandled {} Data Abort @ {:#x}, fault_vaddr={:#x}, ESR={:#x} ({:?}):\n{:#x?}\n{}", + if is_user { "EL0" } else { "EL1" }, + tf.elr, + vaddr, + ESR_EL1.get(), + access_flags, + tf, + tf.backtrace() + ); + } else { + ReturnReason::PageFault(vaddr, access_flags) } } + +#[unsafe(naked)] +unsafe extern "C" fn _enter_user(_ctx: &mut UserContext) -> TrapKind { + naked_asm!( + " + // -- save kernel context -- + sub sp, sp, 12 * 8 + stp x29, x30, [sp, 10 * 8] + stp x27, x28, [sp, 8 * 8] + stp x25, x26, [sp, 6 * 8] + stp x23, x24, [sp, 4 * 8] + stp x21, x22, [sp, 2 * 8] + stp x19, x20, [sp] + + mov x8, sp + str x8, [x0, {sp_el1}] // save sp_el1 to ctx.sp_el1 + + // -- restore user context -- + mov sp, x0 + + mrs x8, tpidr_el0 + msr tpidrro_el0, x8 + + ldp x8, x9, [sp, {elr_el1}] + msr elr_el1, x8 + msr spsr_el1, x9 + + ldp x8, x9, [sp, {sp_el0}] + msr sp_el0, x8 + msr tpidr_el0, x9 + + ldr x30, [sp, 30 * 8] + ldp x28, x29, [sp, 28 * 8] + ldp x26, x27, [sp, 26 * 8] + ldp x24, x25, [sp, 24 * 8] + ldp x22, x23, [sp, 22 * 8] + ldp x20, x21, [sp, 20 * 8] + ldp x18, x19, [sp, 18 * 8] + ldp x16, x17, [sp, 16 * 8] + ldp x14, x15, [sp, 14 * 8] + ldp x12, x13, [sp, 12 * 8] + ldp x10, x11, [sp, 10 * 8] + ldp x8, x9, [sp, 8 * 8] + ldp x6, x7, [sp, 6 * 8] + ldp x4, x5, [sp, 4 * 8] + ldp x2, x3, [sp, 2 * 8] + ldp x0, x1, [sp] + eret + ", + sp_el1 = const offset_of!(UserContext, sp_el1), + elr_el1 = const offset_of!(TrapFrame, elr), + sp_el0 = const offset_of!(TrapFrame, usp), + ) +} + +#[unsafe(no_mangle)] +#[unsafe(naked)] +pub unsafe extern "C" fn _user_trap_entry() -> ! { + naked_asm!( + " + ldr x8, [sp, {sp_el1}] // load ctx.sp_el1 to x8 + mov sp, x8 + ldp x19, x20, [sp] + ldp x21, x22, [sp, 2 * 8] + ldp x23, x24, [sp, 4 * 8] + ldp x25, x26, [sp, 6 * 8] + ldp x27, x28, [sp, 8 * 8] + ldp x29, x30, [sp, 10 * 8] + add sp, sp, 12 * 8 + ret + ", + sp_el1 = const offset_of!(UserContext, sp_el1), + ) +} diff --git a/src/trap.rs b/src/trap.rs index 7fc2506..31b9c13 100644 --- a/src/trap.rs +++ b/src/trap.rs @@ -2,12 +2,13 @@ use core::fmt::Debug; +pub use linkme::{ + distributed_slice as def_trap_handler, distributed_slice as register_trap_handler, +}; use memory_addr::VirtAddr; +pub use page_table_entry::MappingFlags as PageFaultFlags; pub use crate::TrapFrame; -pub use linkme::distributed_slice as def_trap_handler; -pub use linkme::distributed_slice as register_trap_handler; -pub use page_table_entry::MappingFlags as PageFaultFlags; /// A slice of IRQ handler functions. #[def_trap_handler]