Move QEMU executables to a non-default member test-qemu, and add multi-hart tests

.github/workflows/qemu.yaml

@@ -32,9 +32,34 @@ jobs:
             qemu: riscv64
           - target: riscv64gc-unknown-none-elf
             qemu: riscv64
-        example:
-          - qemu_uart
-          - qemu_semihosting
+        example-features:
+          - example: qemu_uart
+            features: ""
+          - example: qemu_semihosting
+            features: ""
+          - example: qemu_uart
+            features: "s-mode"
+          - example: multi_hart
+            features: "multi-hart"
+        exclude:
+          # multi_hart requires atomics extension ('a'), exclude targets without it
+          - example-features:
+              example: multi_hart
+            target-qemu:
+              target: riscv32i-unknown-none-elf
+          - example-features:
+              example: multi_hart
+            target-qemu:
+              target: riscv32im-unknown-none-elf
+          - example-features:
+              example: multi_hart
+            target-qemu:
+              target: riscv32imc-unknown-none-elf
+          # s-mode on riscv32 requires OpenSBI which has issues with some targets
+          - example-features:
+              features: "s-mode"
+            target-qemu:
+              qemu: riscv32
 
     steps:
       - name: Checkout
@@ -55,5 +80,9 @@ jobs:
           sudo apt install -y qemu-system-${{ matrix.target-qemu.qemu }}
 
       - name: Run-pass tests
-        run: cargo run --package xtask -- qemu --target ${{ matrix.target-qemu.target }} --example ${{ matrix.example }}
-
+        run: |
+          if [ -n "${{ matrix.example-features.features }}" ]; then
+            cargo run --package xtask -- qemu --target ${{ matrix.target-qemu.target }} --example ${{ matrix.example-features.example }} --features ${{ matrix.example-features.features }}
+          else
+            cargo run --package xtask -- qemu --target ${{ matrix.target-qemu.target }} --example ${{ matrix.example-features.example }}
+          fi

Cargo.toml

@@ -10,6 +10,7 @@ members = [
     "riscv-target-parser",
     "riscv-types",
     "tests-build",
+    "tests-qemu",
     "tests-trybuild",
     "xtask",
 ]

ci/expected/multi_hart.run

@@ -0,0 +1,3 @@
+Hart 0: Initializing
+Hart 1: Running
+Hart 0: Both harts done

riscv-rt/CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
+- Moved QEMU test examples to dedicated `tests-qemu` crate with support for multiple configurations (m-mode, s-mode, single-hart, multi-hart)
 - Added examples for CI tests using semihosting and UART
 - New `no-mhartid` feature to load 0 to `a0` instead of reading `mhartid`.
 - New `no-xtvec` feature that removes interrupt stuff.

tests-qemu/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "tests-qemu"
+version = "0.1.0"
+edition = "2021"
+
+[build-dependencies]
+
+[dependencies]
+panic-halt = "1.0"
+riscv = { path = "../riscv" }
+riscv-rt = { path = "../riscv-rt" }
+riscv-semihosting = { path = "../riscv-semihosting" }
+
+[features]
+default = ["single-hart"]
+s-mode = ["riscv-rt/s-mode", "single-hart"]
+single-hart = ["riscv-rt/single-hart", "riscv/critical-section-single-hart"]
+multi-hart = []

tests-qemu/build.rs

@@ -0,0 +1,28 @@
+use std::{env, fs::File, io::Write, path::PathBuf};
+
+fn main() {
+    let out = &PathBuf::from(env::var_os("OUT_DIR").unwrap());
+
+    let s_mode = env::var_os("CARGO_FEATURE_S_MODE").is_some();
+    let multi_hart = env::var_os("CARGO_FEATURE_MULTI_HART").is_some();
+
+    // Multi-hart is only supported in M-mode; s-mode takes priority if both are enabled
+    let memory_x = match (s_mode, multi_hart) {
+        (true, _) => include_bytes!("memory-s-mode.x").as_slice(),
+        (_, true) => include_bytes!("memory-multihart.x").as_slice(),
+        _ => include_bytes!("memory.x").as_slice(),
+    };
+
+    File::create(out.join("memory.x"))
+        .unwrap()
+        .write_all(memory_x)
+        .unwrap();
+    println!("cargo:rustc-link-search={}", out.display());
+
+    println!("cargo:rustc-link-arg=-Tmemory.x");
+
+    println!("cargo:rerun-if-changed=memory.x");
+    println!("cargo:rerun-if-changed=memory-multihart.x");
+    println!("cargo:rerun-if-changed=memory-s-mode.x");
+    println!("cargo:rerun-if-changed=build.rs");
+}

tests-qemu/examples/multi_hart.rs

@@ -0,0 +1,133 @@
+//! Multi-hart example demonstrating IPI-based hart synchronization.
+//!
+//! Hart 0 initializes UART and wakes Hart 1 via software interrupt (CLINT).
+//! Both harts print messages and synchronize before exit.
+
+#![no_std]
+#![no_main]
+
+extern crate panic_halt;
+
+use core::arch::global_asm;
+use core::sync::atomic::{AtomicBool, Ordering};
+use riscv_rt::entry;
+use riscv_semihosting::debug::{self, EXIT_SUCCESS};
+
+const UART_BASE: usize = 0x1000_0000;
+const UART_THR: usize = UART_BASE;
+const UART_LCR: usize = UART_BASE + 3;
+const UART_LSR: usize = UART_BASE + 5;
+const LCR_DLAB: u8 = 1 << 7;
+const LCR_8N1: u8 = 0x03;
+const LSR_THRE: u8 = 1 << 5;
+
+static UART_LOCK: AtomicBool = AtomicBool::new(false);
+static HART1_DONE: AtomicBool = AtomicBool::new(false);
+
+fn uart_lock() {
+    while UART_LOCK
+        .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
+        .is_err()
+    {
+        core::hint::spin_loop();
+    }
+}
+
+fn uart_unlock() {
+    UART_LOCK.store(false, Ordering::Release);
+}
+
+unsafe fn uart_write_reg(off: usize, v: u8) {
+    (off as *mut u8).write_volatile(v);
+}
+
+unsafe fn uart_read_reg(off: usize) -> u8 {
+    (off as *const u8).read_volatile()
+}
+
+fn uart_init() {
+    unsafe {
+        uart_write_reg(UART_LCR, LCR_DLAB);
+        uart_write_reg(UART_THR, 0x01);
+        uart_write_reg(UART_BASE + 1, 0x00);
+        uart_write_reg(UART_LCR, LCR_8N1);
+        uart_write_reg(UART_BASE + 2, 0x07);
+    }
+}
+
+fn uart_write_byte(b: u8) {
+    unsafe {
+        while (uart_read_reg(UART_LSR) & LSR_THRE) == 0 {}
+        uart_write_reg(UART_THR, b);
+    }
+}
+
+fn uart_print(s: &str) {
+    uart_lock();
+    for &b in s.as_bytes() {
+        uart_write_byte(b);
+    }
+    uart_unlock();
+}
+
+// Custom _mp_hook implementation in assembly
+// Hart 0 returns 1 (true) to initialize RAM
+// Hart 1 polls for IPI via CLINT, then returns 0 (false) to skip RAM init
+global_asm!(
+    r#"
+.section .init.mp_hook, "ax"
+.global _mp_hook
+_mp_hook:
+    beqz a0, 2f         // if hart 0, return true
+
+    // Hart 1: Poll for IPI (no interrupts, just polling)
+    // Clear any pending software interrupt first
+    li t0, 0x02000004   // CLINT msip address for hart 1
+    sw zero, 0(t0)
+
+1:  // Poll mip register for software interrupt pending
+    csrr t0, mip
+    andi t0, t0, 8      // Check MSIP bit
+    beqz t0, 1b         // If not set, keep polling
+
+    // Clear the software interrupt
+    li t0, 0x02000004
+    sw zero, 0(t0)
+
+    // Return false (0) - don't initialize RAM again
+    li a0, 0
+    ret
+
+2:  // Hart 0: return true to initialize RAM
+    li a0, 1
+    ret
+"#
+);
+
+#[entry]
+fn main(hartid: usize) -> ! {
+    if hartid == 0 {
+        uart_init();
+        uart_print("Hart 0: Initializing\n");
+
+        // Send IPI to Hart 1 (write to CLINT msip register for hart 1)
+        unsafe {
+            (0x02000004usize as *mut u32).write_volatile(1);
+        }
+
+        while !HART1_DONE.load(Ordering::Acquire) {
+            core::hint::spin_loop();
+        }
+
+        uart_print("Hart 0: Both harts done\n");
+        debug::exit(EXIT_SUCCESS);
+    } else {
+        // Hart 1 reaches here after _mp_hook detects IPI
+        uart_print("Hart 1: Running\n");
+        HART1_DONE.store(true, Ordering::Release);
+    }
+
+    loop {
+        core::hint::spin_loop();
+    }
+}

riscv-rt/examples/qemu_semihosting.rs → tests-qemu/examples/qemu_semihosting.rs

@@ -2,7 +2,6 @@
 //!
 //! This example uses RISC-V semihosting to print output and cleanly exit QEMU.
 //! Run with: `qemu-system-riscv32 -machine virt -nographic -semihosting-config enable=on,target=native -bios none -kernel <binary>`
-
 #![no_std]
 #![no_main]
 

tests-qemu/memory-multihart.x

@@ -0,0 +1,12 @@
+MEMORY
+{
+    RAM : ORIGIN = 0x80000000, LENGTH = 16M
+}
+REGION_ALIAS("REGION_TEXT", RAM);
+REGION_ALIAS("REGION_RODATA", RAM);
+REGION_ALIAS("REGION_DATA", RAM);
+REGION_ALIAS("REGION_BSS", RAM);
+REGION_ALIAS("REGION_HEAP", RAM);
+REGION_ALIAS("REGION_STACK", RAM);
+_max_hart_id = 1;
+INCLUDE link.x

tests-qemu/memory-s-mode.x

@@ -0,0 +1,11 @@
+MEMORY
+{
+    RAM : ORIGIN = 0x80200000, LENGTH = 16M
+}
+REGION_ALIAS("REGION_TEXT", RAM);
+REGION_ALIAS("REGION_RODATA", RAM);
+REGION_ALIAS("REGION_DATA", RAM);
+REGION_ALIAS("REGION_BSS", RAM);
+REGION_ALIAS("REGION_HEAP", RAM);
+REGION_ALIAS("REGION_STACK", RAM);
+INCLUDE link.x

tests-qemu/src/lib.rs

@@ -0,0 +1 @@
+#![no_std]

xtask/src/main.rs

@@ -23,10 +23,32 @@ fn find_golden_file(target: &str, example: &str) -> Option<PathBuf> {
     None
 }
 
+fn filter_output(raw: &str, s_mode: bool) -> String {
+    let mut lines: Vec<&str> = raw.lines().collect();
+
+    if s_mode {
+        if let Some(pos) = lines.iter().position(|l| l.contains("Boot HART MEDELEG")) {
+            lines = lines.into_iter().skip(pos + 1).collect();
+        }
+    }
+
+    let filtered: Vec<&str> = lines
+        .into_iter()
+        .filter(|l| !l.starts_with("QEMU ") && !l.contains("monitor"))
+        .collect();
+
+    let result = filtered.join("\n");
+    if result.is_empty() {
+        String::new()
+    } else {
+        format!("{}\n", result.trim())
+    }
+}
+
 fn main() -> anyhow::Result<()> {
     let mut args = std::env::args().skip(1).collect::<Vec<_>>();
     if args.is_empty() || args[0] != "qemu" {
-        bail!("usage: cargo run -p xtask -- qemu --target <triple> --example <name>");
+        bail!("usage: cargo run -p xtask -- qemu --target <triple> --example <name> [--features <features>]");
     }
     args.remove(0);
     let mut target = None;
@@ -54,24 +76,27 @@ fn main() -> anyhow::Result<()> {
     }
     let target = target.context("--target required")?;
     let example = example.context("--example required")?;
-    let mut rustflags = "-C link-arg=-Triscv-rt/examples/device_virt.x".to_string();
-    if let Some(f) = &features {
-        if f.contains("s-mode") {
-            rustflags = "-C link-arg=-Triscv-rt/examples/device_virt_s.x".into();
-        }
-    }
+
+    let feat = features.as_deref().unwrap_or("");
+    let s_mode = feat.contains("s-mode");
+    let multi_hart = feat.contains("multi-hart");
 
     let mut cmd = Command::new("cargo");
-    cmd.env("RUSTFLAGS", rustflags).args([
+    cmd.args([
         "build",
         "--package",
-        "riscv-rt",
+        "tests-qemu",
         "--release",
         "--target",
         &target,
         "--example",
         &example,
     ]);
+    // Disable default features when specifying s-mode or multi-hart
+    // since they are mutually exclusive configurations
+    if s_mode || multi_hart {
+        cmd.arg("--no-default-features");
+    }
     cmd.apply_features(features.as_deref());
     let status = cmd.status()?;
     if !status.success() {
@@ -94,10 +119,14 @@ fn main() -> anyhow::Result<()> {
         "-semihosting-config",
         "enable=on,target=native",
     ];
-    if !features.as_deref().unwrap_or("").contains("s-mode") {
+    if !s_mode {
         qemu_args.push("-bios");
         qemu_args.push("none");
     }
+    if multi_hart {
+        qemu_args.push("-smp");
+        qemu_args.push("2");
+    }
     let kernel_path = format!("target/{}/release/examples/{}", target, example);
     let child = Command::new(qemu)
         .args(&qemu_args)
@@ -109,16 +138,7 @@ fn main() -> anyhow::Result<()> {
         .context("running qemu")?;
     let output = child.wait_with_output()?;
     let raw_stdout = String::from_utf8_lossy(&output.stdout).into_owned();
-    let stdout = raw_stdout
-        .lines()
-        .filter(|l| !l.starts_with("QEMU ") && !l.contains("monitor"))
-        .collect::<Vec<_>>()
-        .join("\n");
-    let stdout = if stdout.is_empty() {
-        String::new()
-    } else {
-        format!("{}\n", stdout.trim())
-    };
+    let stdout = filter_output(&raw_stdout, s_mode);
 
     let expected_path = match find_golden_file(&target, &example) {
         Some(p) => p,