feat: add vm_list and vcpu_list, import axtask for vcpu scheduling (arceos-hypervisor#17)

* feat: add vm_list and vcpu_list, import axtask for vcpu scheduling

* refactor: move vmm related codes into vmm

* Rebase to main branch, a few modifications to aarch64 related part of axvm

* Modify vCpu vm-exit handing and some modication related with axaddrspace

* Add doc comments in vmm/vcpus.rs

Author: hky1999

Cargo.lock

@@ -6,3 +6,27 @@ exclude = ["guest/nimbos"]
 
 [profile.release]
 lto = true
+
+[workspace.dependencies]
+axalloc = { git = "https://github.com/arceos-hypervisor/arceos.git", rev = "5c9ee79" }
+axhal = { git = "https://github.com/arceos-hypervisor/arceos.git", rev = "5c9ee79" }
+axstd = { git = "https://github.com/arceos-hypervisor/arceos.git", rev = "5c9ee79" }
+axtask = { git = "https://github.com/arceos-org/arceos.git", branch = "monolithic" }
+
+axvm = { path = "./crates/axvm" }
+axvcpu = { git = "https://github.com/arceos-hypervisor/axvcpu.git" }
+axaddrspace = { git = "https://github.com/arceos-hypervisor/axaddrspace.git", rev = "17378d" }
+
+# Since we need to rely on the `axtask` module from the `monolithic` branch of arceos-org/arceos repository, which supports TaskExt,
+# and this module depends on different versions of `axhal` and `axconfig`, 
+# (ref: https://github.com/arceos-org/arceos/blob/monolithic/modules/axtask/Cargo.toml)
+# we temporarily use [patch] here to unify the dependency versions of these modules.
+[patch."https://github.com/arceos-org/arceos.git"]
+axhal = { git = "https://github.com/arceos-hypervisor/arceos.git", rev = "5c9ee79" }
+axconfig = { git = "https://github.com/arceos-hypervisor/arceos.git", rev = "5c9ee79" }
+
+# The complexity: 
+# `axstd` depends on the `axtask` from the `arceos-hypervisor\arceos` repository when the 'multitask' feature is enabled. 
+# Here, we use [patch] to unify `axtask` to the one from the `monolithic` branch of the arceos-org/arceos repository.
+[patch."https://github.com/arceos-hypervisor/arceos.git"]
+axtask = { git = "https://github.com/arceos-org/arceos.git", branch = "monolithic" }

Cargo.toml

@@ -13,18 +13,20 @@ lazy_static = { version = "1.4", features = ["spin_no_std"] }
 cfg-if = "1.0"
 
 # System dependent modules provided by ArceOS.
-axalloc = { git = "https://github.com/arceos-hypervisor/arceos.git", rev = "5c9ee79" }
-axhal = { git = "https://github.com/arceos-hypervisor/arceos.git", rev = "5c9ee79", features = ["paging"] }
-axstd = { git = "https://github.com/arceos-hypervisor/arceos.git", rev = "5c9ee79", features = [
+axhal = { workspace = true, features = ["paging"] }
+axtask = { workspace = true, features = ["multitask"] }
+axstd = { workspace = true, features = [
     "alloc",
     "paging",
     "fs",
     # "irq",
     "hv",
-], optional = true }
+    "multitask",
+] }
 
-# System independent crates used for constructing hypervisor.
-axvm = { path = "../crates/axvm" }
+# System dependent modules provided by ArceOS-Hypervisor.
+axvm = { workspace = true }
+axvcpu = { workspace = true }
 
 # System independent crates provided by ArceOS, these crates could be imported by remote url. 
 axerrno = "0.1.0"

arceos-vmm/Cargo.toml

@@ -35,9 +35,9 @@ define cargo_clippy
 endef
 
 all_packages := \
-  $(shell ls $(CURDIR)/crates) \
-  $(shell ls $(CURDIR)/modules) \
-  axfeat arceos_api axstd axlibc
+  # $(shell ls $(CURDIR)/crates) \
+  # $(shell ls $(CURDIR)/modules) \
+  # axfeat arceos_api axstd axlibc
 
 define cargo_doc
   $(call run_cmd,cargo doc,--no-deps --all-features --workspace --exclude "arceos-*" $(verbose))

arceos-vmm/scripts/make/cargo.mk

@@ -1,5 +1,4 @@
 use axvm::AxVMHal;
-use memory_addr::PAGE_SIZE_4K as PAGE_SIZE;
 // Todo: should we know about HostPhysAddr and HostVirtAddr here???
 use memory_addr::{PhysAddr, VirtAddr};
 

arceos-vmm/src/hal.rs

@@ -1,28 +1,24 @@
-#![cfg_attr(feature = "axstd", no_std)]
-#![cfg_attr(feature = "axstd", no_main)]
-#![feature(naked_functions)]
-#![allow(warnings)]
-#[macro_use]
-#[cfg(feature = "axstd")]
-extern crate axstd as std;
-
-extern crate alloc;
+#![no_std]
+#![no_main]
 
 #[macro_use]
 extern crate log;
+#[macro_use]
+extern crate alloc;
+extern crate axstd as std;
 
 mod hal;
-mod images;
+mod task;
+mod vmm;
 
-use axvm::config::{AxVMConfig, AxVMCrateConfig};
-use axvm::{AxVM, AxVMPerCpu};
+use axvm::AxVMPerCpu;
 
 use crate::hal::AxVMHalImpl;
 
 #[percpu::def_percpu]
 pub static mut AXVM_PER_CPU: AxVMPerCpu<AxVMHalImpl> = AxVMPerCpu::new_uninit();
 
-#[cfg_attr(feature = "axstd", no_mangle)]
+#[no_mangle]
 fn main() {
     info!("Starting virtualization...");
 
@@ -37,32 +33,12 @@ fn main() {
         .hardware_enable()
         .expect("Failed to enable virtualization");
 
-    // Config file for guest VM should be read into memory in a more flexible way.
-    // FIXME: remove this hardcode.
-    #[cfg(target_arch = "x86_64")]
-    let raw_vm_config = core::include_str!("../configs/nimbos-x86.toml");
-    #[cfg(target_arch = "aarch64")]
-    let raw_vm_config = core::include_str!("../configs/nimbos-aarch64.toml");
-    #[cfg(target_arch = "riscv64")]
-    let raw_vm_config = core::include_str!("../configs/nimbos-riscv64.toml");
-
-    let vm_create_config =
-        AxVMCrateConfig::from_toml(raw_vm_config).expect("Failed to resolve VM config");
-
-    let config = AxVMConfig::from(vm_create_config.clone());
-
-    // Create VM.
-    let vm = AxVM::<AxVMHalImpl>::new(config).expect("Failed to create VM");
-
-    // Load corresponding images for VM.
-    info!("VM[{}] created success, loading images...", vm.id());
-    images::load_vm_images(vm_create_config, vm.clone()).expect("Failed to load VM images");
+    vmm::init();
 
-    info!("Boot VM[{}]...", vm.id());
+    vmm::start();
 
-    // Todo: remove this, details can be get from
-    // this [PR](https://github.com/arceos-hypervisor/arceos-umhv/pull/5).
-    vm.boot().unwrap();
+    // Todo: move this to `vmm::start()`.
+    axtask::WaitQueue::new().wait();
 
-    panic!("VM[{}] boot failed", vm.id());
+    unreachable!("VMM start failed")
 }

arceos-vmm/src/main.rs

@@ -0,0 +1,19 @@
+use axvm::{AxVCpuRef, AxVMRef};
+
+use crate::hal::AxVMHalImpl;
+
+/// Task extended data for the monolithic kernel.
+pub struct TaskExt {
+    /// The VM.
+    pub vm: AxVMRef<AxVMHalImpl>,
+    /// The virtual memory address space.
+    pub vcpu: AxVCpuRef<AxVMHalImpl>,
+}
+
+impl TaskExt {
+    pub const fn new(vm: AxVMRef<AxVMHalImpl>, vcpu: AxVCpuRef<AxVMHalImpl>) -> Self {
+        Self { vm, vcpu }
+    }
+}
+
+axtask::def_task_ext!(TaskExt);

arceos-vmm/src/task.rs

@@ -0,0 +1,30 @@
+use axvm::config::{AxVMConfig, AxVMCrateConfig};
+
+use crate::vmm::{images::load_vm_images, vm_list::push_vm, VM};
+
+pub fn init_guest_vms() {
+    // Config file for guest VM should be read into memory in a more flexible way.
+    // FIXME: remove this hardcode.
+    let gvm_raw_configs = vec![
+        #[cfg(target_arch = "x86_64")]
+        core::include_str!("../../configs/nimbos-x86.toml"),
+        #[cfg(target_arch = "aarch64")]
+        core::include_str!("../../configs/nimbos-aarch64.toml"),
+        #[cfg(target_arch = "riscv64")]
+        core::include_str!("../../configs/nimbos-riscv64.toml"),
+    ];
+
+    for raw_cfg_str in gvm_raw_configs {
+        let vm_create_config =
+            AxVMCrateConfig::from_toml(raw_cfg_str).expect("Failed to resolve VM config");
+        let vm_config = AxVMConfig::from(vm_create_config.clone());
+
+        // Create VM.
+        let vm = VM::new(vm_config).expect("Failed to create VM");
+        push_vm(vm.clone());
+
+        // Load corresponding images for VM.
+        info!("VM[{}] created success, loading images...", vm.id());
+        load_vm_images(vm_create_config, vm.clone()).expect("Failed to load VM images");
+    }
+}

arceos-vmm/src/vmm/config.rs

@@ -1,36 +1,35 @@
 use alloc::string::String;
-use alloc::sync::Arc;
-use alloc::vec::Vec;
 
 use std::fs::File;
 
-use axerrno::{ax_err, ax_err_type, AxError, AxResult};
+use axerrno::{ax_err, ax_err_type, AxResult};
 use memory_addr::VirtAddr;
 
 use axvm::config::AxVMCrateConfig;
-use axvm::AxVM;
 
-use crate::hal::AxVMHalImpl;
+use crate::vmm::VMRef;
 
-pub fn load_vm_images(config: AxVMCrateConfig, vm: Arc<AxVM<AxVMHalImpl>>) -> AxResult {
+/// Loads the VM image files from the filesystem
+/// into the guest VM's memory space based on the VM configuration.
+pub fn load_vm_images(config: AxVMCrateConfig, vm: VMRef) -> AxResult {
     // Load kernel image.
     load_vm_image(
         config.kernel_path,
         VirtAddr::from(config.kernel_load_addr),
         vm.clone(),
-    );
+    )?;
     // Load BIOS image if needed.
     if let Some(bios_path) = config.bios_path {
         if let Some(bios_load_addr) = config.bios_load_addr {
-            load_vm_image(bios_path, VirtAddr::from(bios_load_addr), vm.clone());
+            load_vm_image(bios_path, VirtAddr::from(bios_load_addr), vm.clone())?;
         } else {
             return ax_err!(NotFound, "BIOS load addr is missed");
         }
     };
     // Load Ramdisk image if needed.
     if let Some(ramdisk_path) = config.ramdisk_path {
         if let Some(ramdisk_load_addr) = config.ramdisk_load_addr {
-            load_vm_image(ramdisk_path, VirtAddr::from(ramdisk_load_addr), vm.clone());
+            load_vm_image(ramdisk_path, VirtAddr::from(ramdisk_load_addr), vm.clone())?;
         } else {
             return ax_err!(NotFound, "Ramdisk load addr is missed");
         }
@@ -39,19 +38,15 @@ pub fn load_vm_images(config: AxVMCrateConfig, vm: Arc<AxVM<AxVMHalImpl>>) -> Ax
     // Todo: generate DTB file for guest VM.
     if let Some(dtb_path) = config.dtb_path {
         if let Some(dtb_load_addr) = config.dtb_load_addr {
-            load_vm_image(dtb_path, VirtAddr::from(dtb_load_addr), vm.clone());
+            load_vm_image(dtb_path, VirtAddr::from(dtb_load_addr), vm.clone())?;
         } else {
             return ax_err!(NotFound, "DTB load addr is missed");
         }
     };
     Ok(())
 }
 
-fn load_vm_image(
-    image_path: String,
-    image_load_gpa: VirtAddr,
-    vm: Arc<AxVM<AxVMHalImpl>>,
-) -> AxResult {
+fn load_vm_image(image_path: String, image_load_gpa: VirtAddr, vm: VMRef) -> AxResult {
     use std::io::{BufReader, Read};
     let (image_file, image_size) = open_image_file(image_path.as_str())?;