feat(aarch64): implement system register emulation and IRQ handling

- Add system register emulation support for CNTP_TVAL_EL0 and CNTP_CTL_EL0
- Implement IRQ handler registration and dispatching
- Update exception handling to use new system register emulator
- Refactor Aarch64VCpu to support generic AxVCpuHal

Author: luodeb

Cargo.toml

@@ -6,9 +6,10 @@ edition = "2021"
 [dependencies]
 log = "0.4.21"
 spin = "0.9"
+spin = "0.9"
+lazy_static = { version = "1.5", features = ["spin_no_std"] }
 
 aarch64-cpu = "9.3"
-smccc = "0.1.1"
 tock-registers = "0.8"
 numeric-enum-macro = "0.2"
 
@@ -17,6 +18,9 @@ percpu = "0.1.4"
 crate_interface = "0.1"
 lazy_static = { version = "1.5", features = ["spin_no_std"] }
 axhal = { path = "../../arceos/modules/axhal" }
+aarch64_sysreg = "0.1.1"
+
 axaddrspace = { git = "https://github.com/arceos-hypervisor/axaddrspace.git" }
-axvcpu = { git = "https://github.com/arceos-hypervisor/axvcpu.git", branch = "axvcpuhal" }
-aarch64_sysreg = "0.1.1"
+axvcpu = { git = "https://github.com/arceos-hypervisor/axvcpu.git" }
+axhal = { git = "https://github.com/arceos-hypervisor/arceos.git", branch = "vmm" }
+axdevice = { git = "https://github.com/arceos-hypervisor/axdevice.git" }

src/exception.rs

@@ -96,7 +96,6 @@ pub fn handle_exception_sync(ctx: &mut TrapFrame) -> AxResult<AxVCpuExitReason>
             })
         }
         Some(ESR_EL2::EC::Value::TrappedMsrMrs) => handle_system_register(ctx),
-        // Some(ESR_EL2::EC::Value::InstrAbortLowerEL) => Ok(AxVCpuExitReason::Nothing),
         _ => {
             panic!(
                 "handler not presents for EC_{} @ipa 0x{:x}, @pc 0x{:x}, @esr 0x{:x},
@@ -115,31 +114,6 @@ pub fn handle_exception_sync(ctx: &mut TrapFrame) -> AxResult<AxVCpuExitReason>
     }
 }
 
-/*
-ESR EC:011000 IL, bit [25]：Instruction Length for synchronous exceptions. Possible values of this bit are: 0：16b 1：32b
-Op0, bits [21:20]
-Op2, bits [19:17]
-Op1, bits [16:14]
-CRn, bits [13:10]
-CRm, bits [4:1]   This five parameters are for SYSREG
-
-Rt, bits [9:5]  general-purpose register used for the transfer.
-Direction, bit [0]  0：Write access, including MSR instructions.  1：Read access, including MRS instructions.
-
-#define SYS_CNTFRQ_EL0			sys_reg(3, 3, 14, 0, 0)
-
-#define SYS_CNTPCT_EL0			sys_reg(3, 3, 14, 0, 1)
-#define SYS_CNTPCTSS_EL0		sys_reg(3, 3, 14, 0, 5)
-#define SYS_CNTVCTSS_EL0		sys_reg(3, 3, 14, 0, 6)
-
-#define SYS_CNTP_TVAL_EL0		sys_reg(3, 3, 14, 2, 0)
-#define SYS_CNTP_CTL_EL0		sys_reg(3, 3, 14, 2, 1)
-#define SYS_CNTP_CVAL_EL0		sys_reg(3, 3, 14, 2, 2)
-
-#define SYS_CNTV_CTL_EL0		sys_reg(3, 3, 14, 3, 1)
-#define SYS_CNTV_CVAL_EL0		sys_reg(3, 3, 14, 3, 2)
-*/
-
 fn handle_system_register(context_frame: &mut TrapFrame) -> AxResult<AxVCpuExitReason> {
     let iss = ESR_EL2.read(ESR_EL2::ISS);
     let addr = exception_sysreg_addr(iss.try_into().unwrap());
@@ -150,13 +124,12 @@ fn handle_system_register(context_frame: &mut TrapFrame) -> AxResult<AxVCpuExitR
     context_frame.set_exception_pc(val);
     // TODO! gicv3 ICC_SRE_ADDR / ICC_SGIR_ADDR
     if write {
-        return Ok(AxVCpuExitReason::SysregWrite {
+        return Ok(AxVCpuExitReason::SysRegWrite {
             addr,
-            reg,
-            data: context_frame.gpr(reg as usize) as u64,
+            value: context_frame.gpr(reg as usize) as u64,
         });
     }
-    Ok(AxVCpuExitReason::SysregRead { addr, reg })
+    Ok(AxVCpuExitReason::SysRegRead { addr, reg })
 }
 
 fn handle_data_abort(context_frame: &mut TrapFrame) -> AxResult<AxVCpuExitReason> {
@@ -253,9 +226,12 @@ fn handle_psci_call(ctx: &mut TrapFrame) -> Option<AxResult<AxVCpuExitReason>> {
     })
 }
 
-/// Dispatches IRQs to the appropriate handler provided by the underlying host OS.
+/// Dispatches IRQs to the appropriate handler provided by the underlying host OS,
+/// which is registered at [`crate::pcpu::IRQ_HANDLER`] during `Aarch64PerCpu::new()`.
 fn dispatch_irq() {
-    crate_interface::call_interface!(crate::HalIf::irq_hanlder())
+    unsafe { crate::pcpu::IRQ_HANDLER.current_ref_raw() }
+        .get()
+        .unwrap()()
 }
 
 /// A trampoline function for handling exceptions (VM exits) in EL2.
@@ -276,11 +252,14 @@ fn dispatch_irq() {
 #[no_mangle]
 unsafe extern "C" fn vmexit_trampoline() {
     core::arch::asm!(
+        "mov x6, x0", // Save the exit reason.
         "bl {vcpu_running}", // Check if vcpu is running.
+        "mov x7, x0", // Save the return value of vcpu_running.
+        "mov x0, x6", // Restore the exit reason.
         // If vcpu_running returns true, jump to `return_run_guest`,
         // after that the control flow is handed back to Aarch64VCpu.run(),
         // simulating the normal return of the `run_guest` function.
-        "cbnz x0, {return_run_guest}",
+        "cbnz x7, {return_run_guest}",
         // If vcpu_running returns false, there is no active vcpu running,
         // jump to `dispatch_irq`.
         "bl {dispatch_irq}",

src/lib.rs

@@ -12,11 +12,11 @@ mod context_frame;
 mod exception_utils;
 mod exception;
 mod pcpu;
-mod registers;
+mod system_registers;
 mod vcpu;
 
 pub use self::pcpu::Aarch64PerCpu;
-pub use self::registers::{emu_register_add, emu_register_handle_read, emu_register_handle_write};
+pub use self::system_registers::Aarch64EmuRegs;
 pub use self::vcpu::{Aarch64VCpu, Aarch64VCpuCreateConfig};
 
 /// context frame for aarch64

src/pcpu.rs

@@ -1,4 +1,4 @@
-use core::marker::PhantomData;
+use core::{cell::OnceCell, marker::PhantomData};
 
 use aarch64_cpu::registers::*;
 use tock_registers::interfaces::ReadWriteable;
@@ -10,26 +10,34 @@ use axvcpu::{AxArchPerCpu, AxVCpuHal};
 #[repr(C)]
 #[repr(align(4096))]
 pub struct Aarch64PerCpu<H: AxVCpuHal> {
-    //stack_top_addr has no use yet?
     /// per cpu id
     pub cpu_id: usize,
-    /// context address of this cpu
-    pub ctx: Option<usize>,
     _phantom: PhantomData<H>,
 }
 
 #[percpu::def_percpu]
 static ORI_EXCEPTION_VECTOR_BASE: usize = 0;
 
+/// IRQ handler registered by underlying host OS during per-cpu initialization,
+/// for dispatching IRQs to the host OS.
+///
+/// Set `IRQ_HANDLER` as per-cpu variable to avoid the need of `OnceLock`.
+#[percpu::def_percpu]
+pub static IRQ_HANDLER: OnceCell<&(dyn Fn() + Send + Sync)> = OnceCell::new();
+
 extern "C" {
     fn exception_vector_base_vcpu();
 }
 
 impl<H: AxVCpuHal> AxArchPerCpu for Aarch64PerCpu<H> {
     fn new(cpu_id: usize) -> AxResult<Self> {
+        // Register IRQ handler for this CPU.
+        let _ = unsafe { IRQ_HANDLER.current_ref_mut_raw() }
+            .set(&|| H::irq_hanlder())
+            .map(|_| {});
+
         Ok(Self {
             cpu_id,
-            ctx: None,
             _phantom: PhantomData,
         })
     }

src/system_registers.rs

@@ -0,0 +1,124 @@
+extern crate alloc;
+use crate::vcpu::Aarch64VCpu;
+use aarch64_cpu::registers::Writeable;
+use aarch64_cpu::registers::{CNTFRQ_EL0, CNTPCT_EL0, CNTP_CTL_EL0, CNTP_TVAL_EL0};
+use aarch64_sysreg::SystemRegType;
+use alloc::sync::Arc;
+use alloc::vec::Vec;
+use axdevice::timer::{register_timer, VmmTimerEvent};
+use axvcpu::{AxVCpu, AxVCpuHal};
+use spin::RwLock;
+
+use axhal::irq::MyVgic;
+
+type RegVcpu<H> = Arc<AxVCpu<Aarch64VCpu<H>>>;
+
+/// Struct representing an entry in the emulator register list.
+pub struct EmuRegEntry<H: AxVCpuHal> {
+    /// The type of the emulator register.
+    pub emu_type: EmuRegType,
+    /// The address associated with the emulator register.
+    pub addr: SystemRegType,
+    /// The handler write function for the emulator register.
+    pub handle_write: fn(SystemRegType, u64, RegVcpu<H>) -> bool,
+    /// The handler read function for the emulator register.
+    pub handle_read: fn(SystemRegType, usize, RegVcpu<H>) -> bool,
+}
+
+/// Enumeration representing the type of emulator registers.
+pub enum EmuRegType {
+    /// System register type for emulator registers.
+    SysReg,
+}
+
+/// Struct representing the emulator registers.
+pub struct Aarch64EmuRegs<H: AxVCpuHal> {
+    /// The list of emulator registers.
+    pub emu_regs: RwLock<Vec<EmuRegEntry<H>>>,
+}
+
+impl<H: AxVCpuHal> Aarch64EmuRegs<H> {
+    const EMU_REGISTERS: [EmuRegEntry<H>; 2] = [
+        EmuRegEntry {
+            emu_type: EmuRegType::SysReg,
+            addr: SystemRegType::CNTP_TVAL_EL0,
+            handle_write: |addr, value, _| {
+                info!("Write to emulator register: {:?}, value: {}", addr, value);
+                // CNTP_TVAL_EL0.set(value);
+                let now = axhal::time::monotonic_time_nanos();
+                debug!("Current time: {}", now);
+                register_timer(
+                    value + now,
+                    VmmTimerEvent::new(|_| {
+                        info!("Timer expired");
+                        let gich = MyVgic::get_gich();
+                        let hcr = gich.get_hcr();
+                        gich.set_hcr(hcr | 1 << 0);
+                        let mut lr = 0;
+                        lr |= 30 << 0;
+                        lr |= 1 << 19;
+                        lr |= 1 << 28;
+                        gich.set_lr(0, lr);
+                    }),
+                );
+                true
+            },
+            handle_read: |_, _, _| true,
+        },
+        EmuRegEntry {
+            emu_type: EmuRegType::SysReg,
+            addr: SystemRegType::CNTP_CTL_EL0,
+            handle_write: |addr, value, _| {
+                info!("Write to emulator register: {:?}, value: {}", addr, value);
+                // if value & 0x1 != 0 {
+                //     axhal::irq::set_enable(30, true);
+                // } else {
+                //     axhal::irq::set_enable(30, false);
+                // }
+
+                // let gich = MyVgic::get_gich();
+                // let gicd = MyVgic::get_gicd();
+                // let gicc = MyVgic::get_gicv();
+                // CNTP_CTL_EL0.set(value);
+                // let hcr = gich.get_hcr();
+                // gich.set_hcr(hcr | 1 << 0);
+                // let mut lr = 0;
+                // lr |= 30 << 0;
+                // lr |= 1 << 19;
+                // lr |= 1 << 28;
+                // gich.set_lr(0, lr);
+                // gicd.lock().set_pend(30, true, 0);
+                // gicc.set_ctlr(ctlr);
+
+                true
+            },
+            handle_read: |_, _, _| true,
+        },
+    ];
+
+    /// Handle a write to an emulator register.
+    pub fn emu_register_handle_write(addr: SystemRegType, value: u64, vcpu: RegVcpu<H>) -> bool {
+        let emu_reg = Self::EMU_REGISTERS;
+
+        for entry in emu_reg.iter() {
+            if entry.addr == addr {
+                return (entry.handle_write)(addr, value, vcpu);
+            }
+        }
+        error!("Invalid emulated register write: {}", addr);
+        false
+    }
+
+    /// Handle a read from an emulator register.
+    pub fn emu_register_handle_read(addr: SystemRegType, reg: usize, vcpu: RegVcpu<H>) -> bool {
+        let emu_reg = Self::EMU_REGISTERS;
+
+        for entry in emu_reg.iter() {
+            if entry.addr == addr {
+                return (entry.handle_read)(addr, reg, vcpu);
+            }
+        }
+        error!("Invalid emulated register read: {}", addr);
+        false
+    }
+}

src/vcpu.rs

@@ -57,14 +57,15 @@ pub struct VmCpuRegisters {
 /// A virtual CPU within a guest
 #[repr(C)]
 #[derive(Debug)]
-pub struct Aarch64VCpu {
+pub struct Aarch64VCpu<H: AxVCpuHal> {
     // DO NOT modify `guest_regs` and `host_stack_top` and their order unless you do know what you are doing!
     // DO NOT add anything before or between them unless you do know what you are doing!
     ctx: TrapFrame,
     host_stack_top: u64,
     guest_system_regs: GuestSystemRegisters,
     /// The MPIDR_EL1 value for the vCPU.
     mpidr: u64,
+    _phantom: PhantomData<H>,
 }
 
 /// Configuration for creating a new `Aarch64VCpu`
@@ -76,7 +77,7 @@ pub struct Aarch64VCpuCreateConfig {
     pub mpidr_el1: u64,
 }
 
-impl axvcpu::AxArchVCpu for Aarch64VCpu {
+impl<H: AxVCpuHal> axvcpu::AxArchVCpu for Aarch64VCpu<H> {
     type CreateConfig = Aarch64VCpuCreateConfig;
 
     type SetupConfig = ();
@@ -87,6 +88,7 @@ impl axvcpu::AxArchVCpu for Aarch64VCpu {
             host_stack_top: 0,
             guest_system_regs: GuestSystemRegisters::default(),
             mpidr: config.mpidr_el1,
+            _phantom: PhantomData,
         })
     }
 
@@ -136,10 +138,14 @@ impl axvcpu::AxArchVCpu for Aarch64VCpu {
     fn set_gpr(&mut self, idx: usize, val: usize) {
         self.ctx.set_gpr(idx, val);
     }
+
+    fn inject_interrupt(&mut self, vector: usize) -> AxResult {
+        Ok(())
+    }
 }
 
 // Private function
-impl Aarch64VCpu {
+impl<H: AxVCpuHal> Aarch64VCpu<H> {
     fn init_hv(&mut self) {
         self.ctx.spsr = (SPSR_EL1::M::EL1h
             + SPSR_EL1::I::Masked
@@ -168,12 +174,11 @@ impl Aarch64VCpu {
         .into();
         self.guest_system_regs.hcr_el2 = (HCR_EL2::VM::Enable
             + HCR_EL2::RW::EL1IsAarch64
-            // + HCR_EL2::IMO::EnableVirtualIRQ
-            // + HCR_EL2::FMO::EnableVirtualFIQ
-            + HCR_EL2::TSC::EnableTrapEl1SmcToEl2)
+            + HCR_EL2::IMO::EnableVirtualIRQ
+            + HCR_EL2::FMO::EnableVirtualFIQ
+            + HCR_EL2::TSC::EnableTrapEl1SmcToEl2
+            + HCR_EL2::RW::EL1IsAarch64)
             .into();
-        self.guest_system_regs.cnthctl_el2 =
-            (CNTHCTL_EL2::EL1PCEN::CLEAR + CNTHCTL_EL2::EL1PCTEN::CLEAR).into();
         // self.system_regs.hcr_el2 |= 1<<27;
         // + HCR_EL2::IMO::EnableVirtualIRQ).into();
         // trap el1 smc to el2
@@ -200,7 +205,7 @@ impl Aarch64VCpu {
 }
 
 /// Private functions related to vcpu runtime control flow.
-impl Aarch64VCpu {
+impl<H: AxVCpuHal> Aarch64VCpu<H> {
     /// Save host context and run guest.
     ///
     /// When a VM-Exit happens when guest's vCpu is running,
@@ -228,7 +233,13 @@ impl Aarch64VCpu {
         );
 
         // the dummy return value, the real return value is in x0 when `return_run_guest` returns
-        0
+        let exit_reason: usize;
+        core::arch::asm!(
+            "mov {}, x0",
+            out(reg) exit_reason,
+            options(nostack)
+        );
+        exit_reason
     }
 
     /// Restores guest system control registers.
@@ -282,7 +293,7 @@ impl Aarch64VCpu {
         match exit_reason {
             TrapKind::Synchronous => handle_exception_sync(&mut self.ctx),
             TrapKind::Irq => Ok(AxVCpuExitReason::ExternalInterrupt {
-                vector: axhal::irq::fetch_irq() as _,
+                vector: H::irq_fetch() as _,
             }),
             _ => panic!("Unhandled exception {:?}", exit_reason),
         }