diff --git a/.gitignore b/.gitignore index ae6943b0..621e4c01 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ rusty-tags.vi # dev env /crates /Cargo.toml.bk +/deps # tools should be downloaded from github tools/* diff --git a/Cargo.lock b/Cargo.lock index de14e8d6..d048c03a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,10 +117,9 @@ dependencies = [ [[package]] name = "arceos_api" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "axalloc", - "axconfig", + "axconfig 0.1.0", "axdriver", "axerrno", "axfeat", @@ -176,7 +175,6 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "axaddrspace" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/axaddrspace.git#f1ab1108c1477f6f4f56035b74ea76c8932d6b8d" dependencies = [ "axerrno", "bit_field", @@ -194,7 +192,6 @@ dependencies = [ [[package]] name = "axalloc" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "allocator", "axerrno", @@ -204,6 +201,13 @@ dependencies = [ "memory_addr", ] +[[package]] +name = "axconfig" +version = "0.1.0" +dependencies = [ + "axconfig-gen-macros", +] + [[package]] name = "axconfig" version = "0.1.0" @@ -263,10 +267,9 @@ dependencies = [ [[package]] name = "axdriver" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "axalloc", - "axconfig", + "axconfig 0.1.0", "axdriver_base", "axdriver_block", "axdriver_pci", @@ -320,7 +323,6 @@ dependencies = [ [[package]] name = "axfeat" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "axalloc", "axdriver", @@ -330,12 +332,12 @@ dependencies = [ "axruntime", "axsync", "axtask", + "kspin", ] [[package]] name = "axfs" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "axdriver", "axdriver_block", @@ -388,13 +390,12 @@ dependencies = [ [[package]] name = "axhal" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "aarch64-cpu 9.4.0", "arm_gicv2", "arm_pl011", "axalloc", - "axconfig", + "axconfig 0.1.0", "axlog", "bitflags 2.9.0", "cfg-if", @@ -415,11 +416,13 @@ dependencies = [ "raw-cpuid 11.5.0", "riscv 0.11.1", "sbi-rt", + "spin", "static_assertions", "tock-registers 0.8.1", "x2apic", "x86", "x86_64 0.15.2", + "x86_vcpu", ] [[package]] @@ -434,7 +437,6 @@ dependencies = [ [[package]] name = "axlog" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "cfg-if", "crate_interface", @@ -445,9 +447,8 @@ dependencies = [ [[package]] name = "axmm" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ - "axconfig", + "axconfig 0.1.0", "axerrno", "axhal", "kspin", @@ -459,10 +460,9 @@ dependencies = [ [[package]] name = "axruntime" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "axalloc", - "axconfig", + "axconfig 0.1.0", "axdriver", "axfs", "axhal", @@ -472,13 +472,13 @@ dependencies = [ "chrono", "crate_interface", "kernel_guard", + "lazyinit", "percpu", ] [[package]] name = "axstd" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "arceos_api", "axerrno", @@ -490,7 +490,6 @@ dependencies = [ [[package]] name = "axsync" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "axtask", "kspin", @@ -499,9 +498,8 @@ dependencies = [ [[package]] name = "axtask" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ - "axconfig", + "axconfig 0.1.0", "axhal", "cfg-if", "cpumask", @@ -519,11 +517,11 @@ dependencies = [ [[package]] name = "axvcpu" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/axvcpu.git#8414a575723f929d5fd24010ce16446d90ecf268" dependencies = [ "axaddrspace", "axerrno", "memory_addr", + "page_table_multiarch", "percpu", ] @@ -532,11 +530,12 @@ name = "axvisor" version = "0.1.0" dependencies = [ "axaddrspace", - "axconfig", + "axconfig 0.1.0 (git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm)", "axerrno", "axstd", "axvcpu", "axvm", + "axvmconfig", "bitflags 2.9.0", "crate_interface", "kspin", @@ -557,7 +556,6 @@ dependencies = [ [[package]] name = "axvm" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/axvm.git#224fe307b2ba4c83f31bc3e9a6ef288422d1b5a1" dependencies = [ "arm_vcpu", "axaddrspace", @@ -854,6 +852,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "iced-x86" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c447cff8c7f384a7d4f741cfcff32f75f3ad02b406432e8d6c878d56b1edf6b" +dependencies = [ + "lazy_static", +] + [[package]] name = "indexmap" version = "2.8.0" @@ -901,6 +908,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "lazyinit" @@ -1666,7 +1676,6 @@ dependencies = [ [[package]] name = "x86_vcpu" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/x86_vcpu.git#5f6eaf4157ee8385ba1a2bfe5060098be18b0245" dependencies = [ "axaddrspace", "axerrno", @@ -1675,10 +1684,12 @@ dependencies = [ "bitflags 2.9.0", "cfg-if", "crate_interface", + "iced-x86", "log", "memory_addr", "numeric-enum-macro", "page_table_entry", + "page_table_multiarch", "raw-cpuid 11.5.0", "x86", "x86_64 0.15.2", diff --git a/Cargo.toml b/Cargo.toml index d337d174..dfac87ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [profile.release] lto = true +debug = true [package] name = "axvisor" @@ -11,6 +12,7 @@ license = "GPL-3.0-or-later OR Apache-2.0 OR MulanPubL-2.0 OR MulanPSL2" [features] fs = ["axstd/fs"] +irq = ["axstd/irq"] [dependencies] log = "=0.4.21" @@ -25,16 +27,18 @@ axstd = { git = "https://github.com/arceos-hypervisor/arceos.git", branch = "vmm "alloc", "paging", # "fs", - "irq", + # "irq", "hv", "multitask", - # "sched_rr" -]} + # "sched_rr", + "smp", +] } # System dependent modules provided by ArceOS-Hypervisor. axvm = { git = "https://github.com/arceos-hypervisor/axvm.git" } axvcpu = { git = "https://github.com/arceos-hypervisor/axvcpu.git" } axaddrspace = { git = "https://github.com/arceos-hypervisor/axaddrspace.git" } +axvmconfig = { git = "https://github.com/arceos-hypervisor/axvmconfig.git", default-features = false } # System independent crates provided by ArceOS, these crates could be imported by remote url. crate_interface = "0.1" @@ -50,3 +54,18 @@ axconfig = { git = "https://github.com/arceos-hypervisor/arceos.git", branch = " prettyplease = "0.2" quote = "1.0" syn = "2.0" + +[patch."https://github.com/arceos-hypervisor/arceos.git"] +axstd = { path = "./crates/arceos/ulib/axstd" } + +[patch."https://github.com/arceos-hypervisor/x86_vcpu.git"] +x86_vcpu = { path = "./crates/x86_vcpu" } + +[patch."https://github.com/arceos-hypervisor/axvm.git"] +axvm = { path = "./crates/axvm" } + +[patch."https://github.com/arceos-hypervisor/axvcpu.git"] +axvcpu = { path = "./crates/axvcpu" } + +[patch."https://github.com/arceos-hypervisor/axaddrspace.git"] +axaddrspace = { path = "./crates/axaddrspace" } diff --git a/HOW-to.md b/HOW-to.md new file mode 100644 index 00000000..12ffcf1f --- /dev/null +++ b/HOW-to.md @@ -0,0 +1,68 @@ +``` +mkdir -p deps +cd deps +git clone git@github.com:EquationOS/jailhouse-equation.git + +``` + +## Getting Started + +### Build + +```bash +make PLATFORM=x86_64-qemu-linux defconfig +make PLATFORM=x86_64-qemu-linux SMP=4 LOG=debug scp_linux +``` + +### Test in QEMU (ubuntu as the guest OS) + +1. Download the guest image and run in QEMU: + + ```bash + cd scripts/vmm/host + make image # download image and configure for the first time + make qemu # execute this command only for subsequent runs + ``` + + You can login the guest OS via SSH. The default username and password is `ubuntu` and `123`. The default port is `2334` and can be changed by QEMU arguments. + +2. Copy helpful scripts into the guest OS: + + ```bash + scp -P 2334 scripts/vmm/guest/* ubuntu@localhost:/home/ubuntu # in host + ``` + +3. Setup in the guest OS: + + Here, you need to copy the [jailhouse-equation](https://github.com/EquationOS/jailhouse-equation) manually, because it is still WIP and not published. + + Copy jailhouse-equation dir: + ```bash + scp -P 2334 -r deps/jailhouse-equation ubuntu@localhost:~/ # in host + ``` + + Then run `setup.sh` in guest, (you only need to run it once see [`setup.sh`](scripts/guest/setup.sh) for details). + + ```bash + ssh -p 2334 ubuntu@localhost # in host + ./setup.sh # in guest + ``` + +4. Compile Jailhouse: + + You need to do this each time after modifing the jailhouse code. + + ``` + cd jailhouse-equation + make + ``` + + To change the CPU number reserved for ArceOS, modify the `RT_CPUS` macro in `jailhouse-equation/tools/jailhouse.c`. + +5. Enable AxVisor + + `./enable-axvisor.sh` + +## Development + + `cd scripts/ && ./dev_deps.sh` diff --git a/Makefile b/Makefile index dee0c9e1..575636ff 100644 --- a/Makefile +++ b/Makefile @@ -155,6 +155,7 @@ else ifeq ($(PLAT_NAME), aarch64-bsta1000b-virt-hv) else ifeq ($(PLAT_NAME), aarch64-rk3588j) include scripts/make/rk3588.mk endif +include scripts/vmm/scp.mk defconfig: _axconfig-gen $(call defconfig) @@ -175,6 +176,14 @@ justrun: debug: build $(call run_qemu_debug) +.PHONY: scp_linux +scp_linux: $(OUT_DIR) $(OUT_BIN) + $(call scp_bin) + +.PNONY: ssh +ssh: + ssh -p $(PORT) ubuntu@localhost + gdb: $(GDB) $(OUT_ELF) \ -ex 'target remote localhost:1234' \ diff --git a/build.rs b/build.rs index 0106e6ed..19dbe788 100644 --- a/build.rs +++ b/build.rs @@ -270,7 +270,11 @@ fn gen_linker_script(arch: &str, platform: &str) -> io::Result<()> { } else { arch }; - let ld_content = std::fs::read_to_string("scripts/lds/linker.lds.S")?; + let ld_content = if platform.contains("linux") { + std::fs::read_to_string("scripts/lds/linker_linux.lds.S") + } else { + std::fs::read_to_string("scripts/lds/linker.lds.S") + }?; let ld_content = ld_content.replace("%ARCH%", output_arch); let ld_content = ld_content.replace( "%KERNEL_BASE%", diff --git a/configs/platforms/x86_64-qemu-linux.toml b/configs/platforms/x86_64-qemu-linux.toml new file mode 100644 index 00000000..88b144d2 --- /dev/null +++ b/configs/platforms/x86_64-qemu-linux.toml @@ -0,0 +1,66 @@ +# Architecture identifier. +arch = "x86_64" # str +# Platform identifier. +platform = "x86_64-qemu-linux" # str + +# +# Platform configs +# +[plat] +# Platform family. +family = "x86-linux" # str + +# Base address of the whole physical memory. +phys-memory-base = "0x4000_0000" # uint +# Size of the whole physical memory. (# 256MB) +phys-memory-size = "0x1000_0000" # uint +# Base physical address of the kernel image. +# kernel-base-paddr = "0x10000_0000" # uint +kernel-base-paddr = "0x4000_0000" # uint +# Base virtual address of the kernel image. +# kernel-base-vaddr = "0xffff_8000_0000_0000" # uint +kernel-base-vaddr = "0xffff_ff80_0000_0000" # uint +# Linear mapping offset, for quick conversions between physical and virtual +# addresses. (# kernel-base-vaddr - phys-memory-base) +# 0xffff_ff80_0000_0000 - 0x4000_0000 +phys-virt-offset = "0xffff_ff7f_c000_0000" # uint + +# Offset of bus address and phys address. some boards, the bus address is +# different from the physical address. +phys-bus-offset = 0 # uint +# Kernel address space base. + +# kernel-aspace-base = "0xffff_ff80_0000_0000" # uint +# Note: this is extremely tricky, the kernel address space base should be 0xffff_ff80_0000_0000 in fact. +# But, to map Linux's low memory address (below 0x4000_0000) to the kernel address space, +# (for example, we need to map 0x1000 to 0xffffff7fc0001000), +# we just simply decreate the kernel address space base by 0x10_0000_0000, to 0xffff_ff70_0000_0000. +# So, the kernel address space base is 0xffff_ff70_0000_0000, and the kernel address space size is 0x0000_008f_ffff_f000. +kernel-aspace-base = "0xffff_ff70_0000_0000" # uint +# Kernel address space size. +# kernel-aspace-size = "0x0000_007f_ffff_f000" # uint +kernel-aspace-size = "0x0000_008f_ffff_f000" # uint + +# +# Device specifications +# +[devices] +# MMIO regions with format (`base_paddr`, `size`). +mmio-regions = [ + [0xb000_0000, 0x1000_0000], # PCI config space + [0xfe00_0000, 0xc0_0000], # PCI devices + [0xfec0_0000, 0x1000], # IO APIC + [0xfed0_0000, 0x1000], # HPET + [0xfee0_0000, 0x1000], # Local APIC +] # [(uint, uint)] +# VirtIO MMIO regions with format (`base_paddr`, `size`). +virtio-mmio-regions = [] # [(uint, uint)] +# Base physical address of the PCIe ECAM space (should read from ACPI 'MCFG' table). +pci-ecam-base = 0xb000_0000 # uint +# End PCI bus number. +pci-bus-end = 0xff # uint +# PCI device memory ranges (not used on x86). +pci-ranges = [] # [(uint, uint)] + +# Timer interrupt frequencyin Hz. (4.0GHz) +timer-frequency = 4_000_000_000 # uint diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 7c587ea0..3080634a 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -3,3 +3,6 @@ profile = "minimal" channel = "nightly-2024-12-25" components = ["rust-src", "llvm-tools", "rustfmt", "clippy"] targets = ["x86_64-unknown-none", "riscv64gc-unknown-none-elf", "aarch64-unknown-none", "aarch64-unknown-none-softfloat"] + +[profile.release] +debug = true \ No newline at end of file diff --git a/scripts/dev_deps.sh b/scripts/dev_deps.sh new file mode 100644 index 00000000..658be33f --- /dev/null +++ b/scripts/dev_deps.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Create directories in the parent directory +mkdir -p ../crates ../deps + +# Clone repositories into the crates directory +cd ../crates || exit +git clone git@github.com:EquationOS/arceos.git --branch vmm_type15 +git clone git@github.com:arceos-hypervisor/axaddrspace.git --branch type15 +git clone git@github.com:arceos-hypervisor/axvm.git --branch type15 +git clone git@github.com:arceos-hypervisor/axvcpu.git --branch type15 +git clone git@github.com:arceos-hypervisor/x86_vcpu.git --branch type15 + +# Clone repository into the deps directory +cd ../deps || exit +git clone git@github.com:EquationOS/jailhouse-equation.git --branch axvisor \ No newline at end of file diff --git a/scripts/lds/linker_linux.lds.S b/scripts/lds/linker_linux.lds.S new file mode 100644 index 00000000..c3daa62a --- /dev/null +++ b/scripts/lds/linker_linux.lds.S @@ -0,0 +1,101 @@ +OUTPUT_ARCH(%ARCH%) + +BASE_ADDRESS = %KERNEL_BASE%; + +ENTRY(_start) +SECTIONS +{ + . = BASE_ADDRESS; + _skernel = .; + + .header : { + __header_start = .; + KEEP(*(.header)) + . = ALIGN(4K); + __header_end = .; + } + + .text : ALIGN(4K) { + _stext = .; + *(.text.boot) + *(.text .text.*) + . = ALIGN(4K); + _etext = .; + } + + .rodata : ALIGN(4K) { + _srodata = .; + *(.rodata .rodata.*) + *(.srodata .srodata.*) + *(.sdata2 .sdata2.*) + . = ALIGN(4K); + _erodata = .; + } + + .data : ALIGN(4K) { + _sdata = .; + *(.data.boot_page_table) + . = ALIGN(4K); + *(.data .data.*) + *(.sdata .sdata.*) + *(.got .got.*) + } + + .tdata : ALIGN(0x10) { + _stdata = .; + *(.tdata .tdata.*) + _etdata = .; + } + + .tbss : ALIGN(0x10) { + _stbss = .; + *(.tbss .tbss.*) + *(.tcommon) + _etbss = .; + } + + . = ALIGN(4K); + _percpu_start = .; + _percpu_end = _percpu_start + SIZEOF(.percpu); + .percpu 0x0 : AT(_percpu_start) { + _percpu_load_start = .; + *(.percpu .percpu.*) + _percpu_load_end = .; + . = _percpu_load_start + ALIGN(64) * %SMP%; + } + . = _percpu_end; + + . = ALIGN(4K); + _edata = .; + + .bss : ALIGN(4K) { + boot_stack = .; + *(.bss.stack) + . = ALIGN(4K); + boot_stack_top = .; + + _sbss = .; + *(.bss .bss.*) + *(.sbss .sbss.*) + *(COMMON) + . = ALIGN(4K); + _ebss = .; + } + + _ekernel = .; + + __entry_offset = _start - BASE_ADDRESS; + __kernel_size = _ekernel - BASE_ADDRESS; + + /DISCARD/ : { + *(.comment) *(.gnu*) *(.note*) *(.eh_frame*) + } +} + +SECTIONS { + linkme_IRQ : { *(linkme_IRQ) } + linkm2_IRQ : { *(linkm2_IRQ) } + linkme_PAGE_FAULT : { *(linkme_PAGE_FAULT) } + linkm2_PAGE_FAULT : { *(linkm2_PAGE_FAULT) } +} +INSERT AFTER .tbss; diff --git a/scripts/vmm/guest/disable-axvisor.sh b/scripts/vmm/guest/disable-axvisor.sh new file mode 100755 index 00000000..2b46fa44 --- /dev/null +++ b/scripts/vmm/guest/disable-axvisor.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +JH_DIR=~/jailhouse-equation +JH=$JH_DIR/tools/jailhouse + +sudo $JH disable diff --git a/scripts/vmm/guest/enable-axvisor.sh b/scripts/vmm/guest/enable-axvisor.sh new file mode 100755 index 00000000..0b4c1a20 --- /dev/null +++ b/scripts/vmm/guest/enable-axvisor.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +JH_DIR=~/jailhouse-equation +JH=$JH_DIR/tools/jailhouse + +sudo $JH disable +sudo rmmod jailhouse +sudo insmod $JH_DIR/driver/jailhouse.ko +sudo chown $(whoami) /dev/jailhouse +sudo $JH enable diff --git a/scripts/vmm/guest/setup.sh b/scripts/vmm/guest/setup.sh new file mode 100755 index 00000000..7d141f4c --- /dev/null +++ b/scripts/vmm/guest/setup.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Install packages +sudo sed -i "s/http:\/\/archive.ubuntu.com/http:\/\/mirrors.tuna.tsinghua.edu.cn/g" /etc/apt/sources.list +sudo apt-get update +sudo apt-get install -y build-essential python3-mako + +# Create a hypervisor image link to /lib/firmware/evm-intel.bin +sudo mkdir -p /lib/firmware +sudo ln -sf ~/evm-intel.bin /lib/firmware + +# Clone jailhouse-equation, apply patches and build +# git clone https://github.com/EquationOS/jailhouse-equation +cd jailhouse-equation +make + +# Update grub config file to update kernel cmdline +./update-cmdline.sh +sudo update-grub + +echo +echo "Setup OK!" +echo "Press ENTER to reboot..." +read +sudo reboot diff --git a/scripts/vmm/guest/test_hypercall.c b/scripts/vmm/guest/test_hypercall.c new file mode 100644 index 00000000..ceeec157 --- /dev/null +++ b/scripts/vmm/guest/test_hypercall.c @@ -0,0 +1,40 @@ +#include +#include +#include + +#define HYPERCALL "vmcall" + +static void in_guest() { + printf("Execute VMCALL OK.\n"); + printf("You are in the Guest mode.\n"); + exit(0); +} + +static void in_host() { + printf("Execute VMCALL failed.\n"); + printf("You are in the Host mode.\n"); + exit(1); +} + +static void sig_handler(int signum) { + printf("Caught signal %d\n", signum); + in_host(); +} + +static inline long hypercall(int num) { + long ret; + asm volatile(HYPERCALL : "=a"(ret) : "a"(num) : "memory"); + return ret; +} + +int main () { + signal(SIGSEGV, sig_handler); + signal(SIGILL, sig_handler); + int ret = hypercall(2333); + if (ret == 2333) { + in_guest(); + } else { + in_host(); + } + return 0; +} diff --git a/scripts/vmm/host/.gitignore b/scripts/vmm/host/.gitignore new file mode 100644 index 00000000..37c1c846 --- /dev/null +++ b/scripts/vmm/host/.gitignore @@ -0,0 +1,2 @@ +*.img +user-data \ No newline at end of file diff --git a/scripts/vmm/host/Makefile b/scripts/vmm/host/Makefile new file mode 100644 index 00000000..5eedc515 --- /dev/null +++ b/scripts/vmm/host/Makefile @@ -0,0 +1,58 @@ +QEMU ?= qemu-system-x86_64 +PORT ?= 2334 +SMP ?= 4 +OUT_ELF ?= ../../../apps/vmm/vmm_x86_64-linux.elf + +TELNET_PORT := 4321 + +UV ?= 24.04 + +ifeq ($(UV),22.04) +RELEASE_NAME = focal +endif + +ifeq ($(UV),24.04) +RELEASE_NAME = noble +endif + +qemu_image := ubuntu-$(UV)-server-cloudimg-amd64.img +qemu_args := \ + -smp $(SMP) -m 4G -accel kvm -nographic \ + -machine q35,kernel_irqchip=split \ + -cpu host,-la57,-kvm-asyncpf,-kvm-pv-eoi,-kvm-pv-ipi,-kvm-pv-sched-yield,-kvm-pv-unhalt,-kvm-steal-time,-kvmclock \ + -drive file=$(qemu_image) \ + -net user,id=net,hostfwd=tcp::$(PORT)-:22 -net nic,model=e1000e \ + -serial mon:stdio \ + -serial telnet:localhost:$(TELNET_PORT),server,nowait + +$(qemu_image): + wget -nc https://cloud-images.ubuntu.com/releases/$(RELEASE_NAME)/release/$(qemu_image) + +.ONESHELL: +image: $(qemu_image) + cat >user-data < Option { - ::PagingHandler::alloc_frame() +impl axaddrspace::EPTTranslator for EPTTranslatorImpl { + fn guest_phys_to_host_phys(gpa: axaddrspace::GuestPhysAddr) -> Option { + use std::os::arceos::modules::axtask::{self, TaskExtRef}; + axtask::current().task_ext().vm.guest_phys_to_host_phys(gpa) } +} - fn dealloc_frame(paddr: HostPhysAddr) { - ::PagingHandler::dealloc_frame(paddr) - } +pub struct AxVCpuHalImpl; - #[inline] - fn phys_to_virt(paddr: HostPhysAddr) -> HostVirtAddr { - ::PagingHandler::phys_to_virt(paddr) - } +impl AxVCpuHal for AxVCpuHalImpl { + type EPTTranslator = EPTTranslatorImpl; + type PagingHandler = axhal::paging::PagingHandlerImpl; - fn virt_to_phys(vaddr: axaddrspace::HostVirtAddr) -> axaddrspace::HostPhysAddr { + fn virt_to_phys(vaddr: HostVirtAddr) -> axaddrspace::HostPhysAddr { std::os::arceos::modules::axhal::mem::virt_to_phys(vaddr) } @@ -104,7 +100,8 @@ pub(crate) fn enable_virtualization() { "Initialize CPU affinity failed!" ); - vmm::init_timer_percpu(); + #[cfg(feature = "irq")] + crate::vmm::init_timer_percpu(); let percpu = unsafe { AXVM_PER_CPU.current_ref_mut_raw() }; percpu diff --git a/src/vmm/config.rs b/src/vmm/config.rs index 260d785d..0f033441 100644 --- a/src/vmm/config.rs +++ b/src/vmm/config.rs @@ -10,12 +10,12 @@ pub mod config { #[allow(dead_code)] pub fn default_static_vm_configs() -> Vec<&'static str> { vec![ - #[cfg(target_arch = "x86_64")] - core::include_str!("../../configs/vms/nimbos-x86_64.toml"), - #[cfg(target_arch = "aarch64")] - core::include_str!("../../configs/vms/nimbos-aarch64.toml"), - #[cfg(target_arch = "riscv64")] - core::include_str!("../../configs/vms/nimbos-riscv64.toml"), + // #[cfg(target_arch = "x86_64")] + // core::include_str!("../../configs/vms/nimbos-x86_64.toml"), + // #[cfg(target_arch = "aarch64")] + // core::include_str!("../../configs/vms/nimbos-aarch64.toml"), + // #[cfg(target_arch = "riscv64")] + // core::include_str!("../../configs/vms/nimbos-riscv64.toml"), ] } @@ -41,3 +41,34 @@ pub fn init_guest_vms() { load_vm_images(vm_create_config, vm.clone()).expect("Failed to load VM images"); } } + +pub fn init_host_vm() { + use crate::alloc::string::ToString; + + use std::os::arceos::modules::axhal; + + use axvm::config::AxVMConfig; + use axvmconfig::{VmMemConfig, VmMemMappingType}; + + info!("Creating host VM..."); + + let mut host_vm_cfg = AxVMConfig::new_host( + 0, + "host".to_string(), + axhal::hvheader::HvHeader::get().reserved_cpus() as usize, + ); + + for region in axhal::host_memory_regions() { + host_vm_cfg.append_memory_region(VmMemConfig { + gpa: region.paddr.as_usize(), + size: region.size, + flags: region.flags.bits(), + map_type: VmMemMappingType::MapIentical, + }); + } + + // Create VM. + let vm = + VM::new_host(host_vm_cfg, axhal::get_linux_context_list()).expect("Failed to create VM"); + push_vm(vm.clone()); +} diff --git a/src/vmm/mod.rs b/src/vmm/mod.rs index a3d32041..7a043627 100644 --- a/src/vmm/mod.rs +++ b/src/vmm/mod.rs @@ -1,5 +1,7 @@ mod config; mod images; + +#[cfg(feature = "irq")] mod timer; mod vcpus; mod vm_list; @@ -10,6 +12,8 @@ use core::sync::atomic::AtomicUsize; use core::sync::atomic::Ordering; use crate::hal::{AxVCpuHalImpl, AxVMHalImpl}; + +#[cfg(feature = "irq")] pub use timer::init_percpu as init_timer_percpu; pub type VM = axvm::AxVM; @@ -22,28 +26,32 @@ static VMM: AxWaitQueueHandle = AxWaitQueueHandle::new(); static RUNNING_VM_COUNT: AtomicUsize = AtomicUsize::new(0); pub fn init() { + config::init_host_vm(); // Initialize guest VM according to config file. config::init_guest_vms(); // Setup vcpus, spawn axtask for primary VCpu. info!("Setting up vcpus..."); - for vm in vm_list::get_vm_list() { - vcpus::setup_vm_primary_vcpu(vm); - } + + vm_list::manipulate_each_vm(|vm| { + vcpus::setup_vm_cpu(vm); + }); } pub fn start() { info!("VMM starting, booting VMs..."); - for vm in vm_list::get_vm_list() { - match vm.boot() { - Ok(_) => { - vcpus::notify_primary_vcpu(vm.id()); + vm_list::manipulate_each_vm(|vm| { + let _ = vm + .boot() + .inspect(|_| { + vcpus::boot_vm_cpu(&vm); RUNNING_VM_COUNT.fetch_add(1, Ordering::Release); - info!("VM[{}] boot success", vm.id()) - } - Err(err) => warn!("VM[{}] boot failed, error {:?}", vm.id(), err), - } - } + info!("VM[{}] boot success", vm.id()); + }) + .inspect_err(|err| { + warn!("VM[{}] boot failed, error {:?}", vm.id(), err); + }); + }); // Do not exit until all VMs are stopped. task::ax_wait_queue_wait_until(&VMM, || RUNNING_VM_COUNT.load(Ordering::Acquire) == 0, None); diff --git a/src/vmm/timer.rs b/src/vmm/timer.rs index 0d98704c..4708d2ee 100644 --- a/src/vmm/timer.rs +++ b/src/vmm/timer.rs @@ -1,5 +1,3 @@ -extern crate alloc; - use core::sync::atomic::AtomicUsize; use core::sync::atomic::Ordering; diff --git a/src/vmm/vcpus.rs b/src/vmm/vcpus.rs index 5db545f2..eb4b5cef 100644 --- a/src/vmm/vcpus.rs +++ b/src/vmm/vcpus.rs @@ -43,7 +43,7 @@ impl VMVcpus { /// # Returns /// /// A new `VMVcpus` instance with an empty task list and a fresh wait queue. - fn new(vm: VMRef) -> Self { + fn new(vm: &VMRef) -> Self { Self { _vm_id: vm.id(), wait_queue: WaitQueue::new(), @@ -77,6 +77,10 @@ impl VMVcpus { fn notify_one(&mut self) { self.wait_queue.notify_one(false); } + + fn notify_all(&mut self) { + self.wait_queue.notify_all(false); + } } /// Blocks the current thread until it is explicitly woken up, using the wait queue @@ -116,13 +120,20 @@ where /// /// * `vm_id` - The ID of the VM whose vCPUs are to be notified. /// -pub(crate) fn notify_primary_vcpu(vm_id: usize) { +fn notify_primary_vcpu(vm_id: usize) { // Generally, the primary vCPU is the first and **only** vCPU in the list. unsafe { VM_VCPU_TASK_WAIT_QUEUE.get_mut(&vm_id) } .unwrap() .notify_one() } +/// Boots all vCPUs of the specified VM. +fn notify_all_vcpus(vm_id: usize) { + unsafe { VM_VCPU_TASK_WAIT_QUEUE.get_mut(&vm_id) } + .unwrap() + .notify_all() +} + /// Boot target vCPU on the specified VM. /// This function is used to boot a secondary vCPU on a VM, setting the entry point and argument for the vCPU. /// @@ -172,10 +183,10 @@ fn vcpu_on(vm: VMRef, vcpu_id: usize, entry_point: GuestPhysAddr, arg: usize) { /// # Arguments /// /// * `vm` - A reference to the VM for which the vCPUs are being set up. -pub fn setup_vm_primary_vcpu(vm: VMRef) { +fn setup_vm_primary_vcpu(vm: VMRef) { info!("Initializing VM[{}]'s {} vcpus", vm.id(), vm.vcpu_num()); let vm_id = vm.id(); - let mut vm_vcpus = VMVcpus::new(vm.clone()); + let mut vm_vcpus = VMVcpus::new(&vm); let primary_vcpu_id = 0; @@ -187,6 +198,53 @@ pub fn setup_vm_primary_vcpu(vm: VMRef) { } } +fn setup_vm_all_cpus(vm: VMRef) { + info!( + "Initializing VM[{}, {}]'s {} vcpus", + vm.id(), + vm.name(), + vm.vcpu_num() + ); + + if !vm.is_host_vm() { + warn!("setup_vm_all_cpus: not host vm"); + return; + } + + let vm_id = vm.id(); + unsafe { + VM_VCPU_TASK_WAIT_QUEUE.insert(vm_id, VMVcpus::new(&vm)); + } + + for vcpu_id in 0..vm.vcpu_num() { + let vcpu = vm.vcpu_list()[vcpu_id].clone(); + let vcpu_task = alloc_vcpu_task(vm.clone(), vcpu); + + unsafe { + VM_VCPU_TASK_WAIT_QUEUE + .get_mut(&vm_id) + .unwrap() + .add_vcpu_task(vcpu_task); + } + } +} + +pub fn setup_vm_cpu(vm: VMRef) { + if vm.is_host_vm() { + setup_vm_all_cpus(vm); + } else { + setup_vm_primary_vcpu(vm); + } +} + +pub fn boot_vm_cpu(vm: &VMRef) { + if vm.is_host_vm() { + notify_all_vcpus(vm.id()); + } else { + notify_primary_vcpu(vm.id()); + } +} + /// Allocates arceos task for vcpu, set the task's entry function to [`vcpu_run()`], /// alse initializes the CPU mask if the vCPU has a dedicated physical CPU set. /// @@ -205,7 +263,7 @@ pub fn setup_vm_primary_vcpu(vm: VMRef) { /// * The task is scheduled on the scheduler of arceos after it is spawned. fn alloc_vcpu_task(vm: VMRef, vcpu: VCpuRef) -> AxTaskRef { info!("Spawning task for VM[{}] Vcpu[{}]", vm.id(), vcpu.id()); - let mut vcpu_task = TaskInner::new( + let mut vcpu_task: TaskInner = TaskInner::new( vcpu_run, format!("VM[{}]-VCpu[{}]", vm.id(), vcpu.id()), KERNEL_STACK_SIZE, @@ -256,6 +314,7 @@ fn vcpu_run() { "VM[{}] VCpu[{}] run failed with exit code {}", vm_id, vcpu_id, hardware_entry_failure_reason ); + wait(vm_id) } AxVCpuExitReason::ExternalInterrupt { vector } => { debug!("VM[{}] run VCpu[{}] get irq {}", vm_id, vcpu_id, vector); diff --git a/src/vmm/vm_list.rs b/src/vmm/vm_list.rs index 56a2f45e..3e7ec348 100644 --- a/src/vmm/vm_list.rs +++ b/src/vmm/vm_list.rs @@ -1,5 +1,4 @@ use alloc::collections::BTreeMap; -use alloc::vec::Vec; use spin::Mutex; @@ -106,11 +105,13 @@ pub fn get_vm_by_id(vm_id: usize) -> Option { GLOBAL_VM_LIST.lock().get_vm_by_id(vm_id) } -pub fn get_vm_list() -> Vec { - let global_vm_list = GLOBAL_VM_LIST.lock().vm_list.clone(); - let mut vm_list = Vec::with_capacity(global_vm_list.len()); - for (_id, vm) in global_vm_list { - vm_list.push(vm.clone()); +pub fn manipulate_each_vm(f: F) +where + F: Fn(VMRef), +{ + let vm_list_locked = GLOBAL_VM_LIST.lock(); + for vm in vm_list_locked.vm_list.values() { + debug!("Manipulating VM[{}]...", vm.id()); + f(vm.clone()); } - vm_list }