Migrate to Rust Edition 2024, refactor current el irq handling routine

.github/workflows/ci.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        rust-toolchain: [nightly, nightly-2024-05-02]
+        rust-toolchain: [nightly-2024-12-25, nightly]
         targets: [aarch64-unknown-none-softfloat]
     steps:
     - uses: actions/checkout@v4

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "arm_vcpu"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 [dependencies]
 log = "0.4.21"

src/context_frame.rs

@@ -211,43 +211,45 @@ impl GuestSystemRegisters {
     /// This method uses inline assembly to read the values of various system registers
     /// and stores them in the corresponding fields of the `GuestSystemRegisters` structure.
     pub unsafe fn store(&mut self) {
-        asm!("mrs {0}, CNTVOFF_EL2", out(reg) self.cntvoff_el2);
-        asm!("mrs {0}, CNTV_CVAL_EL0", out(reg) self.cntv_cval_el0);
-        asm!("mrs {0:x}, CNTKCTL_EL1", out(reg) self.cntkctl_el1);
-        asm!("mrs {0:x}, CNTP_CTL_EL0", out(reg) self.cntp_ctl_el0);
-        asm!("mrs {0:x}, CNTV_CTL_EL0", out(reg) self.cntv_ctl_el0);
-        asm!("mrs {0:x}, CNTP_TVAL_EL0", out(reg) self.cntp_tval_el0);
-        asm!("mrs {0:x}, CNTV_TVAL_EL0", out(reg) self.cntv_tval_el0);
-        asm!("mrs {0}, CNTVCT_EL0", out(reg) self.cntvct_el0);
-        // MRS!("self.vpidr_el2, VPIDR_EL2, "x");
-        asm!("mrs {0}, VMPIDR_EL2", out(reg) self.vmpidr_el2);
+        unsafe {
+            asm!("mrs {0}, CNTVOFF_EL2", out(reg) self.cntvoff_el2);
+            asm!("mrs {0}, CNTV_CVAL_EL0", out(reg) self.cntv_cval_el0);
+            asm!("mrs {0:x}, CNTKCTL_EL1", out(reg) self.cntkctl_el1);
+            asm!("mrs {0:x}, CNTP_CTL_EL0", out(reg) self.cntp_ctl_el0);
+            asm!("mrs {0:x}, CNTV_CTL_EL0", out(reg) self.cntv_ctl_el0);
+            asm!("mrs {0:x}, CNTP_TVAL_EL0", out(reg) self.cntp_tval_el0);
+            asm!("mrs {0:x}, CNTV_TVAL_EL0", out(reg) self.cntv_tval_el0);
+            asm!("mrs {0}, CNTVCT_EL0", out(reg) self.cntvct_el0);
+            // MRS!("self.vpidr_el2, VPIDR_EL2, "x");
+            asm!("mrs {0}, VMPIDR_EL2", out(reg) self.vmpidr_el2);
 
-        asm!("mrs {0}, SP_EL0", out(reg) self.sp_el0);
-        asm!("mrs {0}, SP_EL1", out(reg) self.sp_el1);
-        asm!("mrs {0}, ELR_EL1", out(reg) self.elr_el1);
-        asm!("mrs {0:x}, SPSR_EL1", out(reg) self.spsr_el1);
-        asm!("mrs {0:x}, SCTLR_EL1", out(reg) self.sctlr_el1);
-        asm!("mrs {0:x}, CPACR_EL1", out(reg) self.cpacr_el1);
-        asm!("mrs {0}, TTBR0_EL1", out(reg) self.ttbr0_el1);
-        asm!("mrs {0}, TTBR1_EL1", out(reg) self.ttbr1_el1);
-        asm!("mrs {0}, TCR_EL1", out(reg) self.tcr_el1);
-        asm!("mrs {0:x}, ESR_EL1", out(reg) self.esr_el1);
-        asm!("mrs {0}, FAR_EL1", out(reg) self.far_el1);
-        asm!("mrs {0}, PAR_EL1", out(reg) self.par_el1);
-        asm!("mrs {0}, MAIR_EL1", out(reg) self.mair_el1);
-        asm!("mrs {0}, AMAIR_EL1", out(reg) self.amair_el1);
-        asm!("mrs {0}, VBAR_EL1", out(reg) self.vbar_el1);
-        asm!("mrs {0:x}, CONTEXTIDR_EL1", out(reg) self.contextidr_el1);
-        asm!("mrs {0}, TPIDR_EL0", out(reg) self.tpidr_el0);
-        asm!("mrs {0}, TPIDR_EL1", out(reg) self.tpidr_el1);
-        asm!("mrs {0}, TPIDRRO_EL0", out(reg) self.tpidrro_el0);
+            asm!("mrs {0}, SP_EL0", out(reg) self.sp_el0);
+            asm!("mrs {0}, SP_EL1", out(reg) self.sp_el1);
+            asm!("mrs {0}, ELR_EL1", out(reg) self.elr_el1);
+            asm!("mrs {0:x}, SPSR_EL1", out(reg) self.spsr_el1);
+            asm!("mrs {0:x}, SCTLR_EL1", out(reg) self.sctlr_el1);
+            asm!("mrs {0:x}, CPACR_EL1", out(reg) self.cpacr_el1);
+            asm!("mrs {0}, TTBR0_EL1", out(reg) self.ttbr0_el1);
+            asm!("mrs {0}, TTBR1_EL1", out(reg) self.ttbr1_el1);
+            asm!("mrs {0}, TCR_EL1", out(reg) self.tcr_el1);
+            asm!("mrs {0:x}, ESR_EL1", out(reg) self.esr_el1);
+            asm!("mrs {0}, FAR_EL1", out(reg) self.far_el1);
+            asm!("mrs {0}, PAR_EL1", out(reg) self.par_el1);
+            asm!("mrs {0}, MAIR_EL1", out(reg) self.mair_el1);
+            asm!("mrs {0}, AMAIR_EL1", out(reg) self.amair_el1);
+            asm!("mrs {0}, VBAR_EL1", out(reg) self.vbar_el1);
+            asm!("mrs {0:x}, CONTEXTIDR_EL1", out(reg) self.contextidr_el1);
+            asm!("mrs {0}, TPIDR_EL0", out(reg) self.tpidr_el0);
+            asm!("mrs {0}, TPIDR_EL1", out(reg) self.tpidr_el1);
+            asm!("mrs {0}, TPIDRRO_EL0", out(reg) self.tpidrro_el0);
 
-        asm!("mrs {0}, PMCR_EL0", out(reg) self.pmcr_el0);
-        asm!("mrs {0}, VTCR_EL2", out(reg) self.vtcr_el2);
-        asm!("mrs {0}, VTTBR_EL2", out(reg) self.vttbr_el2);
-        asm!("mrs {0}, HCR_EL2", out(reg) self.hcr_el2);
-        asm!("mrs {0}, ACTLR_EL1", out(reg) self.actlr_el1);
-        // println!("save sctlr {:x}", self.sctlr_el1);
+            asm!("mrs {0}, PMCR_EL0", out(reg) self.pmcr_el0);
+            asm!("mrs {0}, VTCR_EL2", out(reg) self.vtcr_el2);
+            asm!("mrs {0}, VTTBR_EL2", out(reg) self.vttbr_el2);
+            asm!("mrs {0}, HCR_EL2", out(reg) self.hcr_el2);
+            asm!("mrs {0}, ACTLR_EL1", out(reg) self.actlr_el1);
+            // println!("save sctlr {:x}", self.sctlr_el1);
+        }
     }
 
     /// Restores the values of all relevant system registers from the `GuestSystemRegisters` structure.
@@ -259,38 +261,40 @@ impl GuestSystemRegisters {
     /// Each system register is restored with its corresponding value from the `GuestSystemRegisters`, ensuring
     /// that the virtual machine or thread resumes execution with the correct context.
     pub unsafe fn restore(&self) {
-        asm!("msr CNTV_CVAL_EL0, {0}", in(reg) self.cntv_cval_el0);
-        asm!("msr CNTKCTL_EL1, {0:x}", in (reg) self.cntkctl_el1);
-        asm!("msr CNTV_CTL_EL0, {0:x}", in (reg) self.cntv_ctl_el0);
-        // The restoration of SP_EL0 is done in `exception_return_el2`,
-        // which move the value from `self.ctx.sp_el0` to `SP_EL0`.
-        // asm!("msr SP_EL0, {0}", in(reg) self.sp_el0);
-        asm!("msr SP_EL1, {0}", in(reg) self.sp_el1);
-        asm!("msr ELR_EL1, {0}", in(reg) self.elr_el1);
-        asm!("msr SPSR_EL1, {0:x}", in(reg) self.spsr_el1);
-        asm!("msr SCTLR_EL1, {0:x}", in(reg) self.sctlr_el1);
-        asm!("msr CPACR_EL1, {0:x}", in(reg) self.cpacr_el1);
-        asm!("msr TTBR0_EL1, {0}", in(reg) self.ttbr0_el1);
-        asm!("msr TTBR1_EL1, {0}", in(reg) self.ttbr1_el1);
-        asm!("msr TCR_EL1, {0}", in(reg) self.tcr_el1);
-        asm!("msr ESR_EL1, {0:x}", in(reg) self.esr_el1);
-        asm!("msr FAR_EL1, {0}", in(reg) self.far_el1);
-        asm!("msr PAR_EL1, {0}", in(reg) self.par_el1);
-        asm!("msr MAIR_EL1, {0}", in(reg) self.mair_el1);
-        asm!("msr AMAIR_EL1, {0}", in(reg) self.amair_el1);
-        asm!("msr VBAR_EL1, {0}", in(reg) self.vbar_el1);
-        asm!("msr CONTEXTIDR_EL1, {0:x}", in(reg) self.contextidr_el1);
-        asm!("msr TPIDR_EL0, {0}", in(reg) self.tpidr_el0);
-        asm!("msr TPIDR_EL1, {0}", in(reg) self.tpidr_el1);
-        asm!("msr TPIDRRO_EL0, {0}", in(reg) self.tpidrro_el0);
+        unsafe {
+            asm!("msr CNTV_CVAL_EL0, {0}", in(reg) self.cntv_cval_el0);
+            asm!("msr CNTKCTL_EL1, {0:x}", in (reg) self.cntkctl_el1);
+            asm!("msr CNTV_CTL_EL0, {0:x}", in (reg) self.cntv_ctl_el0);
+            // The restoration of SP_EL0 is done in `exception_return_el2`,
+            // which move the value from `self.ctx.sp_el0` to `SP_EL0`.
+            // asm!("msr SP_EL0, {0}", in(reg) self.sp_el0);
+            asm!("msr SP_EL1, {0}", in(reg) self.sp_el1);
+            asm!("msr ELR_EL1, {0}", in(reg) self.elr_el1);
+            asm!("msr SPSR_EL1, {0:x}", in(reg) self.spsr_el1);
+            asm!("msr SCTLR_EL1, {0:x}", in(reg) self.sctlr_el1);
+            asm!("msr CPACR_EL1, {0:x}", in(reg) self.cpacr_el1);
+            asm!("msr TTBR0_EL1, {0}", in(reg) self.ttbr0_el1);
+            asm!("msr TTBR1_EL1, {0}", in(reg) self.ttbr1_el1);
+            asm!("msr TCR_EL1, {0}", in(reg) self.tcr_el1);
+            asm!("msr ESR_EL1, {0:x}", in(reg) self.esr_el1);
+            asm!("msr FAR_EL1, {0}", in(reg) self.far_el1);
+            asm!("msr PAR_EL1, {0}", in(reg) self.par_el1);
+            asm!("msr MAIR_EL1, {0}", in(reg) self.mair_el1);
+            asm!("msr AMAIR_EL1, {0}", in(reg) self.amair_el1);
+            asm!("msr VBAR_EL1, {0}", in(reg) self.vbar_el1);
+            asm!("msr CONTEXTIDR_EL1, {0:x}", in(reg) self.contextidr_el1);
+            asm!("msr TPIDR_EL0, {0}", in(reg) self.tpidr_el0);
+            asm!("msr TPIDR_EL1, {0}", in(reg) self.tpidr_el1);
+            asm!("msr TPIDRRO_EL0, {0}", in(reg) self.tpidrro_el0);
 
-        asm!("msr PMCR_EL0, {0}", in(reg) self.pmcr_el0);
-        asm!("msr ACTLR_EL1, {0}", in(reg) self.actlr_el1);
+            asm!("msr PMCR_EL0, {0}", in(reg) self.pmcr_el0);
+            asm!("msr ACTLR_EL1, {0}", in(reg) self.actlr_el1);
 
-        asm!("msr VTCR_EL2, {0}", in(reg) self.vtcr_el2);
-        asm!("msr VTTBR_EL2, {0}", in(reg) self.vttbr_el2);
-        asm!("msr HCR_EL2, {0}", in(reg) self.hcr_el2);
-        asm!("msr VMPIDR_EL2, {0}", in(reg) self.vmpidr_el2);
-        asm!("msr CNTVOFF_EL2, {0}", in(reg) self.cntvoff_el2);
+            asm!("msr VTCR_EL2, {0}", in(reg) self.vtcr_el2);
+            asm!("msr VTTBR_EL2, {0}", in(reg) self.vttbr_el2);
+            asm!("msr HCR_EL2, {0}", in(reg) self.hcr_el2);
+            asm!("msr VMPIDR_EL2, {0}", in(reg) self.vmpidr_el2);
+            asm!("msr CNTVOFF_EL2, {0}", in(reg) self.cntvoff_el2);
+        }
     }
 }

src/exception.S

@@ -68,7 +68,21 @@
     b       .Lexception_return_el2
 .endm
 
-.macro HANDLE_IRQ_VCPU
+.macro HANDLE_CURRENT_IRQ
+.p2align 7
+    SAVE_REGS_FROM_EL1
+    bl      current_el_irq_handler
+    b       .Lexception_return_el2
+.endm
+
+.macro HANDLE_CURRENT_SYNC
+.p2align 7
+    SAVE_REGS_FROM_EL1
+    bl      current_el_sync_handler
+    b       .Lexception_return_el2
+.endm
+
+.macro HANDLE_LOWER_IRQ_VCPU
 .p2align 7
     SAVE_REGS_FROM_EL1
     mov    x0, {exception_irq}
@@ -96,14 +110,14 @@ exception_vector_base_vcpu:
     INVALID_EXCP_EL2 3 0
 
     // current EL, with SP_ELx
-    INVALID_EXCP_EL2 0 1
-    HANDLE_IRQ_VCPU
+    HANDLE_CURRENT_SYNC
+    HANDLE_CURRENT_IRQ
     INVALID_EXCP_EL2 2 1
     INVALID_EXCP_EL2 3 1
 
     // lower EL, aarch64
     HANDLE_LOWER_SYNC_VCPU
-    HANDLE_IRQ_VCPU
+    HANDLE_LOWER_IRQ_VCPU
     INVALID_EXCP_EL2 2 2
     INVALID_EXCP_EL2 3 2
 

src/exception.rs

@@ -1,9 +1,10 @@
-use aarch64_cpu::registers::{Readable, ESR_EL2, HCR_EL2, SCTLR_EL1, VTCR_EL2, VTTBR_EL2};
+use aarch64_cpu::registers::{ESR_EL2, HCR_EL2, Readable, SCTLR_EL1, VTCR_EL2, VTTBR_EL2};
 
 use axaddrspace::GuestPhysAddr;
 use axerrno::{AxError, AxResult};
 use axvcpu::{AccessWidth, AxVCpuExitReason};
 
+use crate::TrapFrame;
 use crate::exception_utils::{
     exception_class, exception_class_value, exception_data_abort_access_is_write,
     exception_data_abort_access_reg, exception_data_abort_access_reg_width,
@@ -12,7 +13,6 @@ use crate::exception_utils::{
     exception_esr, exception_fault_addr, exception_next_instruction_step, exception_sysreg_addr,
     exception_sysreg_direction_write, exception_sysreg_gpr,
 };
-use crate::TrapFrame;
 
 numeric_enum_macro::numeric_enum! {
 #[repr(u8)]
@@ -260,7 +260,7 @@ fn handle_psci_call(ctx: &mut TrapFrame) -> Option<AxResult<AxVCpuExitReason>> {
 fn handle_smc64_exception(ctx: &mut TrapFrame) -> AxResult<AxVCpuExitReason> {
     // Is this a psci call?
     if let Some(result) = handle_psci_call(ctx) {
-        return result;
+        result
     } else {
         // We just forward the SMC call to the ATF directly.
         // The args are from lower EL, so it is safe to call the ATF.
@@ -270,47 +270,23 @@ fn handle_smc64_exception(ctx: &mut TrapFrame) -> AxResult<AxVCpuExitReason> {
     }
 }
 
+/// Handles IRQ exceptions that occur from the current exception level.
 /// Dispatches IRQs to the appropriate handler provided by the underlying host OS,
 /// which is registered at [`crate::pcpu::IRQ_HANDLER`] during `Aarch64PerCpu::new()`.
-fn dispatch_irq() {
+#[unsafe(no_mangle)]
+fn current_el_irq_handler(_tf: &mut TrapFrame) {
     unsafe { crate::pcpu::IRQ_HANDLER.current_ref_raw() }
         .get()
         .unwrap()()
 }
 
-/// A trampoline function for handling exceptions (VM exits) in EL2.
-///
-/// Functionality:
-///
-/// 1. **Check if VCPU is running:**
-///    - The `vcpu_running` function is called to check if the VCPU is currently running.
-///      If the VCPU is running, the control flow is transferred to the `return_run_guest` function.
-///
-/// 2. **Dispatch IRQ:**
-///   - If there is no active vcpu running, the `dispatch_irq` function is called to handle the IRQ,
-///     which will dispatch this irq routine to the underlining host OS.
-///   - The IRQ handling routine will end up calling `exception_return_el2` here.
-///
-/// Note that the `return_run_guest` will never return.
-#[naked]
-#[no_mangle]
-unsafe extern "C" fn vmexit_trampoline() {
-    core::arch::asm!(
-        "bl {vcpu_running}", // Check if vcpu is running.
-        // If vcpu_running returns true, jump to `return_run_guest`,
-        // after that the control flow is handed back to Aarch64VCpu.run(),
-        // simulating the normal return of the `run_guest` function.
-        "cbnz x0, {return_run_guest}",
-        // If vcpu_running returns false, there is no active vcpu running,
-        // jump to `dispatch_irq`.
-        "bl {dispatch_irq}",
-        // Return from exception.
-        "b  .Lexception_return_el2",
-        vcpu_running = sym crate::vcpu::vcpu_running,
-        return_run_guest = sym return_run_guest,
-        dispatch_irq = sym dispatch_irq,
-        options(noreturn),
-    )
+/// Handles synchronous exceptions that occur from the current exception level.
+#[unsafe(no_mangle)]
+fn current_el_sync_handler(tf: &mut TrapFrame) {
+    panic!(
+        "Unhandled synchronous exception from current EL: {:#x?}",
+        tf
+    );
 }
 
 /// A trampoline function for sp switching during handling VM exits,
@@ -349,22 +325,23 @@ unsafe extern "C" fn vmexit_trampoline() {
 /// - This function is not typically called directly from Rust code. Instead, it is
 ///   invoked as part of the low-level hypervisor or VM exit handling routines.
 #[naked]
-#[no_mangle]
-unsafe extern "C" fn return_run_guest() -> ! {
-    core::arch::asm!(
-        // Curretly `sp` points to the base address of `Aarch64VCpu.ctx`, which stores guest's `TrapFrame`.
-        "add x9, sp, 34 * 8", // Skip the exception frame.
-        // Currently `x9` points to `&Aarch64VCpu.host_stack_top`, see `run_guest()` in vcpu.rs.
-        "ldr x10, [x9]", // Get `host_stack_top` value from `&Aarch64VCpu.host_stack_top`.
-        "mov sp, x10",   // Set `sp` as the host stack top.
-        restore_regs_from_stack!(), // Restore host function context frame.
-        "ret", // Control flow is handed back to Aarch64VCpu.run(), simulating the normal return of the `run_guest` function.
-        options(noreturn),
-    )
+#[unsafe(no_mangle)]
+unsafe extern "C" fn vmexit_trampoline() -> ! {
+    unsafe {
+        core::arch::naked_asm!(
+            // Curretly `sp` points to the base address of `Aarch64VCpu.ctx`, which stores guest's `TrapFrame`.
+            "add x9, sp, 34 * 8", // Skip the exception frame.
+            // Currently `x9` points to `&Aarch64VCpu.host_stack_top`, see `run_guest()` in vcpu.rs.
+            "ldr x10, [x9]", // Get `host_stack_top` value from `&Aarch64VCpu.host_stack_top`.
+            "mov sp, x10",   // Set `sp` as the host stack top.
+            restore_regs_from_stack!(), // Restore host function context frame.
+            "ret", // Control flow is handed back to Aarch64VCpu.run(), simulating the normal return of the `run_guest` function.
+        )
+    }
 }
 
 /// Deal with invalid aarch64 exception.
-#[no_mangle]
+#[unsafe(no_mangle)]
 fn invalid_exception_el2(tf: &mut TrapFrame, kind: TrapKind, source: TrapSource) {
     panic!(
         "Invalid exception {:?} from {:?}:\n{:#x?}",

src/exception_utils.rs

@@ -2,7 +2,7 @@ use aarch64_cpu::registers::{ESR_EL2, FAR_EL2, PAR_EL1};
 use tock_registers::interfaces::*;
 
 use axaddrspace::GuestPhysAddr;
-use axerrno::{ax_err, AxResult};
+use axerrno::{AxResult, ax_err};
 
 /// Retrieves the Exception Syndrome Register (ESR) value from EL2.
 ///

src/lib.rs

@@ -1,8 +1,6 @@
 #![no_std]
 #![feature(naked_functions)]
 #![feature(doc_cfg)]
-#![feature(asm_const)]
-#![feature(exclusive_range_pattern)]
 #![doc = include_str!("../README.md")]
 
 #[macro_use]

src/pcpu.rs

@@ -25,7 +25,7 @@ static ORI_EXCEPTION_VECTOR_BASE: usize = 0;
 #[percpu::def_percpu]
 pub static IRQ_HANDLER: OnceCell<&(dyn Fn() + Send + Sync)> = OnceCell::new();
 
-extern "C" {
+unsafe extern "C" {
     fn exception_vector_base_vcpu();
 }
 

src/smc.rs

@@ -12,13 +12,15 @@ pub unsafe fn smc_call(x0: u64, x1: u64, x2: u64, x3: u64) -> (u64, u64, u64, u6
     let r1;
     let r2;
     let r3;
-    asm!(
-        "smc #0",
-        inout("x0") x0 => r0,
-        inout("x1") x1 => r1,
-        inout("x2") x2 => r2,
-        inout("x3") x3 => r3,
-        options(nomem, nostack)
-    );
+    unsafe {
+        asm!(
+            "smc #0",
+            inout("x0") x0 => r0,
+            inout("x1") x1 => r1,
+            inout("x2") x2 => r2,
+            inout("x3") x3 => r3,
+            options(nomem, nostack)
+        );
+    }
     (r0, r1, r2, r3)
 }