From 4761d233190ebf4818f9bcd93c399797c915cbbc Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Sun, 16 Mar 2025 20:39:32 +0800 Subject: [PATCH 1/6] [wip] on type15 boot, bug in jailhouse driver --- .gitignore | 1 + Cargo.lock | 38 ++++----- Cargo.toml | 11 ++- HOW-to.md | 49 +++++++++++ Makefile | 5 ++ build.rs | 6 +- configs/platforms/x86_64-qemu-linux.toml | 68 +++++++++++++++ scripts/lds/linker_linux.lds.S | 101 +++++++++++++++++++++++ scripts/vmm/guest/disable-arceos-vmm.sh | 6 ++ scripts/vmm/guest/enable-arceos-vmm.sh | 10 +++ scripts/vmm/guest/setup.sh | 25 ++++++ scripts/vmm/guest/test_hypercall.c | 40 +++++++++ scripts/vmm/host/.gitignore | 2 + scripts/vmm/host/Makefile | 58 +++++++++++++ scripts/vmm/scp.mk | 7 ++ src/main.rs | 6 +- 16 files changed, 408 insertions(+), 25 deletions(-) create mode 100644 HOW-to.md create mode 100644 configs/platforms/x86_64-qemu-linux.toml create mode 100644 scripts/lds/linker_linux.lds.S create mode 100755 scripts/vmm/guest/disable-arceos-vmm.sh create mode 100755 scripts/vmm/guest/enable-arceos-vmm.sh create mode 100755 scripts/vmm/guest/setup.sh create mode 100644 scripts/vmm/guest/test_hypercall.c create mode 100644 scripts/vmm/host/.gitignore create mode 100644 scripts/vmm/host/Makefile create mode 100644 scripts/vmm/scp.mk diff --git a/.gitignore b/.gitignore index ae6943b0..621e4c01 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ rusty-tags.vi # dev env /crates /Cargo.toml.bk +/deps # tools should be downloaded from github tools/* diff --git a/Cargo.lock b/Cargo.lock index de14e8d6..f37ca3d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,10 +117,9 @@ dependencies = [ [[package]] name = "arceos_api" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "axalloc", - "axconfig", + "axconfig 0.1.0", "axdriver", "axerrno", "axfeat", @@ -194,7 +193,6 @@ dependencies = [ [[package]] name = "axalloc" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "allocator", "axerrno", @@ -204,6 +202,13 @@ dependencies = [ "memory_addr", ] +[[package]] +name = "axconfig" +version = "0.1.0" +dependencies = [ + "axconfig-gen-macros", +] + [[package]] name = "axconfig" version = "0.1.0" @@ -263,10 +268,9 @@ dependencies = [ [[package]] name = "axdriver" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "axalloc", - "axconfig", + "axconfig 0.1.0", "axdriver_base", "axdriver_block", "axdriver_pci", @@ -320,7 +324,6 @@ dependencies = [ [[package]] name = "axfeat" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "axalloc", "axdriver", @@ -330,12 +333,12 @@ dependencies = [ "axruntime", "axsync", "axtask", + "kspin", ] [[package]] name = "axfs" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "axdriver", "axdriver_block", @@ -388,13 +391,12 @@ dependencies = [ [[package]] name = "axhal" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "aarch64-cpu 9.4.0", "arm_gicv2", "arm_pl011", "axalloc", - "axconfig", + "axconfig 0.1.0", "axlog", "bitflags 2.9.0", "cfg-if", @@ -415,11 +417,13 @@ dependencies = [ "raw-cpuid 11.5.0", "riscv 0.11.1", "sbi-rt", + "spin", "static_assertions", "tock-registers 0.8.1", "x2apic", "x86", "x86_64 0.15.2", + "x86_vcpu", ] [[package]] @@ -434,7 +438,6 @@ dependencies = [ [[package]] name = "axlog" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "cfg-if", "crate_interface", @@ -445,9 +448,8 @@ dependencies = [ [[package]] name = "axmm" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ - "axconfig", + "axconfig 0.1.0", "axerrno", "axhal", "kspin", @@ -459,10 +461,9 @@ dependencies = [ [[package]] name = "axruntime" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "axalloc", - "axconfig", + "axconfig 0.1.0", "axdriver", "axfs", "axhal", @@ -472,13 +473,13 @@ dependencies = [ "chrono", "crate_interface", "kernel_guard", + "lazyinit", "percpu", ] [[package]] name = "axstd" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "arceos_api", "axerrno", @@ -490,7 +491,6 @@ dependencies = [ [[package]] name = "axsync" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ "axtask", "kspin", @@ -499,9 +499,8 @@ dependencies = [ [[package]] name = "axtask" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm#39848356f3bb43c1c0a5b02a6188bd3235e789a5" dependencies = [ - "axconfig", + "axconfig 0.1.0", "axhal", "cfg-if", "cpumask", @@ -532,7 +531,7 @@ name = "axvisor" version = "0.1.0" dependencies = [ "axaddrspace", - "axconfig", + "axconfig 0.1.0 (git+https://github.com/arceos-hypervisor/arceos.git?branch=vmm)", "axerrno", "axstd", "axvcpu", @@ -1666,7 +1665,6 @@ dependencies = [ [[package]] name = "x86_vcpu" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/x86_vcpu.git#5f6eaf4157ee8385ba1a2bfe5060098be18b0245" dependencies = [ "axaddrspace", "axerrno", diff --git a/Cargo.toml b/Cargo.toml index d337d174..942b4412 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [profile.release] lto = true +debug = true [package] name = "axvisor" @@ -28,7 +29,8 @@ axstd = { git = "https://github.com/arceos-hypervisor/arceos.git", branch = "vmm "irq", "hv", "multitask", - # "sched_rr" + # "sched_rr", + "smp", ]} # System dependent modules provided by ArceOS-Hypervisor. @@ -50,3 +52,10 @@ axconfig = { git = "https://github.com/arceos-hypervisor/arceos.git", branch = " prettyplease = "0.2" quote = "1.0" syn = "2.0" + +[patch."https://github.com/arceos-hypervisor/arceos.git"] +axstd = { path = "./crates/arceos/ulib/axstd" } + +[patch."https://github.com/arceos-hypervisor/x86_vcpu.git"] +x86_vcpu = { path = "./crates/x86_vcpu" } + diff --git a/HOW-to.md b/HOW-to.md new file mode 100644 index 00000000..312fc713 --- /dev/null +++ b/HOW-to.md @@ -0,0 +1,49 @@ +``` +mkdir -p deps +cd deps +git clone git@github.com:EquationOS/jailhouse-equation.git + +``` + +## Getting Started + +### Build + +```bash +make PLATFORM=x86_64-qemu-linux defconfig +make PLATFORM=x86_64-qemu-linux SMP=4 LOG=debug scp_linux +``` + +### Test in QEMU (ubuntu as the guest OS) + +1. Download the guest image and run in QEMU: + + ```bash + cd scripts/host + make image # download image and configure for the first time + make qemu # execute this command only for subsequent runs + ``` + + You can login the guest OS via SSH. The default username and password is `ubuntu` and `123`. The default port is `2334` and can be changed by QEMU arguments. + +2. Copy helpful scripts into the guest OS: + + ```bash + scp -P 2334 scripts/guest/* ubuntu@localhost:/home/ubuntu # in host + ``` + +3. Setup in the guest OS: + + Here, you need to copy the [jailhouse-equation](https://github.com/EquationOS/jailhouse-equation) manually, because it is still WIP and not published. + + Copy jailhouse-equation dir: + ```bash + scp -P 2334 -r deps/jailhouse-equation ubuntu@localhost:~/ # in host + ``` + + Then run `setup.sh` in guest, (you only need to run it once see [`setup.sh`](scripts/guest/setup.sh) for details). + + ```bash + ssh -p 2334 ubuntu@localhost # in host + ./setup.sh # in guest + ``` \ No newline at end of file diff --git a/Makefile b/Makefile index dee0c9e1..51487ac6 100644 --- a/Makefile +++ b/Makefile @@ -155,6 +155,7 @@ else ifeq ($(PLAT_NAME), aarch64-bsta1000b-virt-hv) else ifeq ($(PLAT_NAME), aarch64-rk3588j) include scripts/make/rk3588.mk endif +include scripts/vmm/scp.mk defconfig: _axconfig-gen $(call defconfig) @@ -175,6 +176,10 @@ justrun: debug: build $(call run_qemu_debug) +.PHONY: scp_linux +scp_linux: $(OUT_DIR) $(OUT_BIN) + $(call scp_bin) + gdb: $(GDB) $(OUT_ELF) \ -ex 'target remote localhost:1234' \ diff --git a/build.rs b/build.rs index 0106e6ed..19dbe788 100644 --- a/build.rs +++ b/build.rs @@ -270,7 +270,11 @@ fn gen_linker_script(arch: &str, platform: &str) -> io::Result<()> { } else { arch }; - let ld_content = std::fs::read_to_string("scripts/lds/linker.lds.S")?; + let ld_content = if platform.contains("linux") { + std::fs::read_to_string("scripts/lds/linker_linux.lds.S") + } else { + std::fs::read_to_string("scripts/lds/linker.lds.S") + }?; let ld_content = ld_content.replace("%ARCH%", output_arch); let ld_content = ld_content.replace( "%KERNEL_BASE%", diff --git a/configs/platforms/x86_64-qemu-linux.toml b/configs/platforms/x86_64-qemu-linux.toml new file mode 100644 index 00000000..568ab77d --- /dev/null +++ b/configs/platforms/x86_64-qemu-linux.toml @@ -0,0 +1,68 @@ +# Architecture identifier. +arch = "x86_64" # str +# Platform identifier. +platform = "x86_64-qemu-linux" # str + +# +# Platform configs +# +[plat] +# Platform family. +family = "x86-linux" # str + +# Base address of the whole physical memory. +# phys-memory-base = "0x10000_0000" # uint +phys-memory-base = "0x3a00_0000" # uint +# Size of the whole physical memory. (# 256MB) +phys-memory-size = "0x1000_0000" # uint +# Base physical address of the kernel image. +# kernel-base-paddr = "0x10000_0000" # uint +kernel-base-paddr = "0x3a00_0000" # uint +# Base virtual address of the kernel image. +kernel-base-vaddr = "0xffff_8000_0000_0000" # uint +# Linear mapping offset, for quick conversions between physical and virtual +# addresses. (# kernel-base-vaddr - phys-memory-base) +# 0xffff_8000_0000_0000 - 0x3a00_0000 +phys-virt-offset = "0xffff_7fff_c600_0000" # uint + +# Offset of bus address and phys address. some boards, the bus address is +# different from the physical address. +phys-bus-offset = 0 # uint +# Kernel address space base. +# kernel-aspace-base = "0xffff_8000_0000_0000" # uint +# Note: this is extremely tricky, the kernel address space base should be 0xffff_8000_0000_0000 in fact. +# But, to map Linux's low memory address (below 0x3a00_0000) to the kernel address space, +# (for example, we need to map 0x1000 to 0xffff7fffc6001000), +# we just simply decreate the kernel address space base by 0x1000_0000_0000, to 0xffff700000000000. +# So, the kernel address space base is 0xffff700000000000, and the kernel address space size is 0x00008fffffff0000. +kernel-aspace-base = "0xffff_7000_0000_0000" # uint +# Kernel address space size. +# kernel-aspace-size = "0x0000_7fff_ffff_f000" # uint +kernel-aspace-size = "0x0000_8fff_ffff_f000" # uint + +# FFFFFEFFC6000000 + +# +# Device specifications +# +[devices] +# MMIO regions with format (`base_paddr`, `size`). +mmio-regions = [ + [0xb000_0000, 0x1000_0000], # PCI config space + [0xfe00_0000, 0xc0_0000], # PCI devices + [0xfec0_0000, 0x1000], # IO APIC + [0xfed0_0000, 0x1000], # HPET + [0xfee0_0000, 0x1000], # Local APIC + [0x380000000000, 0x4000] # PCI devices +] # [(uint, uint)] +# VirtIO MMIO regions with format (`base_paddr`, `size`). +virtio-mmio-regions = [] # [(uint, uint)] +# Base physical address of the PCIe ECAM space (should read from ACPI 'MCFG' table). +pci-ecam-base = 0xb000_0000 # uint +# End PCI bus number. +pci-bus-end = 0xff # uint +# PCI device memory ranges (not used on x86). +pci-ranges = [] # [(uint, uint)] + +# Timer interrupt frequencyin Hz. (4.0GHz) +timer-frequency = 4_000_000_000 # uint diff --git a/scripts/lds/linker_linux.lds.S b/scripts/lds/linker_linux.lds.S new file mode 100644 index 00000000..c3daa62a --- /dev/null +++ b/scripts/lds/linker_linux.lds.S @@ -0,0 +1,101 @@ +OUTPUT_ARCH(%ARCH%) + +BASE_ADDRESS = %KERNEL_BASE%; + +ENTRY(_start) +SECTIONS +{ + . = BASE_ADDRESS; + _skernel = .; + + .header : { + __header_start = .; + KEEP(*(.header)) + . = ALIGN(4K); + __header_end = .; + } + + .text : ALIGN(4K) { + _stext = .; + *(.text.boot) + *(.text .text.*) + . = ALIGN(4K); + _etext = .; + } + + .rodata : ALIGN(4K) { + _srodata = .; + *(.rodata .rodata.*) + *(.srodata .srodata.*) + *(.sdata2 .sdata2.*) + . = ALIGN(4K); + _erodata = .; + } + + .data : ALIGN(4K) { + _sdata = .; + *(.data.boot_page_table) + . = ALIGN(4K); + *(.data .data.*) + *(.sdata .sdata.*) + *(.got .got.*) + } + + .tdata : ALIGN(0x10) { + _stdata = .; + *(.tdata .tdata.*) + _etdata = .; + } + + .tbss : ALIGN(0x10) { + _stbss = .; + *(.tbss .tbss.*) + *(.tcommon) + _etbss = .; + } + + . = ALIGN(4K); + _percpu_start = .; + _percpu_end = _percpu_start + SIZEOF(.percpu); + .percpu 0x0 : AT(_percpu_start) { + _percpu_load_start = .; + *(.percpu .percpu.*) + _percpu_load_end = .; + . = _percpu_load_start + ALIGN(64) * %SMP%; + } + . = _percpu_end; + + . = ALIGN(4K); + _edata = .; + + .bss : ALIGN(4K) { + boot_stack = .; + *(.bss.stack) + . = ALIGN(4K); + boot_stack_top = .; + + _sbss = .; + *(.bss .bss.*) + *(.sbss .sbss.*) + *(COMMON) + . = ALIGN(4K); + _ebss = .; + } + + _ekernel = .; + + __entry_offset = _start - BASE_ADDRESS; + __kernel_size = _ekernel - BASE_ADDRESS; + + /DISCARD/ : { + *(.comment) *(.gnu*) *(.note*) *(.eh_frame*) + } +} + +SECTIONS { + linkme_IRQ : { *(linkme_IRQ) } + linkm2_IRQ : { *(linkm2_IRQ) } + linkme_PAGE_FAULT : { *(linkme_PAGE_FAULT) } + linkm2_PAGE_FAULT : { *(linkm2_PAGE_FAULT) } +} +INSERT AFTER .tbss; diff --git a/scripts/vmm/guest/disable-arceos-vmm.sh b/scripts/vmm/guest/disable-arceos-vmm.sh new file mode 100755 index 00000000..2b46fa44 --- /dev/null +++ b/scripts/vmm/guest/disable-arceos-vmm.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +JH_DIR=~/jailhouse-equation +JH=$JH_DIR/tools/jailhouse + +sudo $JH disable diff --git a/scripts/vmm/guest/enable-arceos-vmm.sh b/scripts/vmm/guest/enable-arceos-vmm.sh new file mode 100755 index 00000000..0b4c1a20 --- /dev/null +++ b/scripts/vmm/guest/enable-arceos-vmm.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +JH_DIR=~/jailhouse-equation +JH=$JH_DIR/tools/jailhouse + +sudo $JH disable +sudo rmmod jailhouse +sudo insmod $JH_DIR/driver/jailhouse.ko +sudo chown $(whoami) /dev/jailhouse +sudo $JH enable diff --git a/scripts/vmm/guest/setup.sh b/scripts/vmm/guest/setup.sh new file mode 100755 index 00000000..7d141f4c --- /dev/null +++ b/scripts/vmm/guest/setup.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Install packages +sudo sed -i "s/http:\/\/archive.ubuntu.com/http:\/\/mirrors.tuna.tsinghua.edu.cn/g" /etc/apt/sources.list +sudo apt-get update +sudo apt-get install -y build-essential python3-mako + +# Create a hypervisor image link to /lib/firmware/evm-intel.bin +sudo mkdir -p /lib/firmware +sudo ln -sf ~/evm-intel.bin /lib/firmware + +# Clone jailhouse-equation, apply patches and build +# git clone https://github.com/EquationOS/jailhouse-equation +cd jailhouse-equation +make + +# Update grub config file to update kernel cmdline +./update-cmdline.sh +sudo update-grub + +echo +echo "Setup OK!" +echo "Press ENTER to reboot..." +read +sudo reboot diff --git a/scripts/vmm/guest/test_hypercall.c b/scripts/vmm/guest/test_hypercall.c new file mode 100644 index 00000000..ceeec157 --- /dev/null +++ b/scripts/vmm/guest/test_hypercall.c @@ -0,0 +1,40 @@ +#include +#include +#include + +#define HYPERCALL "vmcall" + +static void in_guest() { + printf("Execute VMCALL OK.\n"); + printf("You are in the Guest mode.\n"); + exit(0); +} + +static void in_host() { + printf("Execute VMCALL failed.\n"); + printf("You are in the Host mode.\n"); + exit(1); +} + +static void sig_handler(int signum) { + printf("Caught signal %d\n", signum); + in_host(); +} + +static inline long hypercall(int num) { + long ret; + asm volatile(HYPERCALL : "=a"(ret) : "a"(num) : "memory"); + return ret; +} + +int main () { + signal(SIGSEGV, sig_handler); + signal(SIGILL, sig_handler); + int ret = hypercall(2333); + if (ret == 2333) { + in_guest(); + } else { + in_host(); + } + return 0; +} diff --git a/scripts/vmm/host/.gitignore b/scripts/vmm/host/.gitignore new file mode 100644 index 00000000..37c1c846 --- /dev/null +++ b/scripts/vmm/host/.gitignore @@ -0,0 +1,2 @@ +*.img +user-data \ No newline at end of file diff --git a/scripts/vmm/host/Makefile b/scripts/vmm/host/Makefile new file mode 100644 index 00000000..4b1caf2d --- /dev/null +++ b/scripts/vmm/host/Makefile @@ -0,0 +1,58 @@ +QEMU ?= qemu-system-x86_64 +PORT ?= 2334 +SMP ?= 4 +OUT_ELF ?= ../../../apps/vmm/vmm_x86_64-linux.elf + +TELNET_PORT := 4321 + +UV ?= 24.04 + +ifeq ($(UV),22.04) +RELEASE_NAME = focal +endif + +ifeq ($(UV),24.04) +RELEASE_NAME = noble +endif + +qemu_image := ubuntu-$(UV)-server-cloudimg-amd64.img +qemu_args := \ + -smp $(SMP) -m 4G -accel kvm -nographic \ + -machine q35,kernel_irqchip=split \ + -cpu host,-la57,-kvm-asyncpf,-kvm-pv-eoi,-kvm-pv-ipi,-kvm-pv-sched-yield,-kvm-pv-unhalt,-kvm-steal-time,-kvmclock \ + -drive file=$(qemu_image) \ + -net user,id=net,hostfwd=tcp::$(PORT)-:22 -net nic,model=e1000e \ + -serial mon:stdio \ + -serial telnet:localhost:$(TELNET_PORT),server#,nowait + +$(qemu_image): + wget -nc https://cloud-images.ubuntu.com/releases/$(RELEASE_NAME)/release/$(qemu_image) + +.ONESHELL: +image: $(qemu_image) + cat >user-data < Date: Tue, 18 Mar 2025 11:56:39 +0800 Subject: [PATCH 2/6] [wip] boot and enter main --- configs/platforms/x86_64-qemu-linux.toml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/configs/platforms/x86_64-qemu-linux.toml b/configs/platforms/x86_64-qemu-linux.toml index 568ab77d..9a7dac7c 100644 --- a/configs/platforms/x86_64-qemu-linux.toml +++ b/configs/platforms/x86_64-qemu-linux.toml @@ -19,11 +19,14 @@ phys-memory-size = "0x1000_0000" # uint # kernel-base-paddr = "0x10000_0000" # uint kernel-base-paddr = "0x3a00_0000" # uint # Base virtual address of the kernel image. -kernel-base-vaddr = "0xffff_8000_0000_0000" # uint +# kernel-base-vaddr = "0xffff_8000_0000_0000" # uint +kernel-base-vaddr = "0xffff_ff00_0000_0000" # uint # Linear mapping offset, for quick conversions between physical and virtual # addresses. (# kernel-base-vaddr - phys-memory-base) # 0xffff_8000_0000_0000 - 0x3a00_0000 -phys-virt-offset = "0xffff_7fff_c600_0000" # uint +# phys-virt-offset = "0xffff_7fff_c600_0000" # uint +# 0xffff_ff00_0000_0000 - 0x3a00_0000 +phys-virt-offset = "0xffff_feff_c600_0000" # uint # Offset of bus address and phys address. some boards, the bus address is # different from the physical address. @@ -35,10 +38,20 @@ phys-bus-offset = 0 # uint # (for example, we need to map 0x1000 to 0xffff7fffc6001000), # we just simply decreate the kernel address space base by 0x1000_0000_0000, to 0xffff700000000000. # So, the kernel address space base is 0xffff700000000000, and the kernel address space size is 0x00008fffffff0000. -kernel-aspace-base = "0xffff_7000_0000_0000" # uint -# Kernel address space size. +# kernel-aspace-base = "0xffff_7000_0000_0000" # uint # kernel-aspace-size = "0x0000_7fff_ffff_f000" # uint -kernel-aspace-size = "0x0000_8fff_ffff_f000" # uint +# kernel-aspace-size = "0x0000_8fff_ffff_f000" # uint + +# kernel-aspace-base = "0xffff_ff00_0000_0000" # uint +# Note: this is extremely tricky, the kernel address space base should be 0xffff_ff00_0000_0000 in fact. +# But, to map Linux's low memory address (below 0x3a00_0000) to the kernel address space, +# (for example, we need to map 0x1000 to 0xfffffeffc6001000), +# we just simply decreate the kernel address space base by 0x1_0000_0000, to 0xffff_feff_0000_0000. +# So, the kernel address space base is 0xffff_feff_0000_0000, and the kernel address space size is 0x0000_0100_ffff_f000. +kernel-aspace-base = "0xffff_feff_0000_0000" # uint +# Kernel address space size. +# kernel-aspace-size = "0x0000_00ff_ffff_f000" # uint +kernel-aspace-size = "0x0000_0100_ffff_f000" # uint # FFFFFEFFC6000000 @@ -53,7 +66,6 @@ mmio-regions = [ [0xfec0_0000, 0x1000], # IO APIC [0xfed0_0000, 0x1000], # HPET [0xfee0_0000, 0x1000], # Local APIC - [0x380000000000, 0x4000] # PCI devices ] # [(uint, uint)] # VirtIO MMIO regions with format (`base_paddr`, `size`). virtio-mmio-regions = [] # [(uint, uint)] From 2fe490ff287b35352dffa854bbcd524cb10adc2e Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Tue, 18 Mar 2025 22:00:29 +0800 Subject: [PATCH 3/6] [feat] bring host VM concept into axvisor, do compile --- Cargo.lock | 3 +-- Cargo.toml | 6 ++++++ src/main.rs | 6 +++--- src/vmm/config.rs | 34 ++++++++++++++++++++++++++++++++++ src/vmm/mod.rs | 43 ++++++++++++++++++++++++++++++++----------- src/vmm/timer.rs | 2 -- src/vmm/vcpus.rs | 31 +++++++++++++++++++++++++++++-- src/vmm/vm_list.rs | 15 ++++++++------- 8 files changed, 113 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f37ca3d5..4f6a6e6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -518,7 +518,6 @@ dependencies = [ [[package]] name = "axvcpu" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/axvcpu.git#8414a575723f929d5fd24010ce16446d90ecf268" dependencies = [ "axaddrspace", "axerrno", @@ -536,6 +535,7 @@ dependencies = [ "axstd", "axvcpu", "axvm", + "axvmconfig", "bitflags 2.9.0", "crate_interface", "kspin", @@ -556,7 +556,6 @@ dependencies = [ [[package]] name = "axvm" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/axvm.git#224fe307b2ba4c83f31bc3e9a6ef288422d1b5a1" dependencies = [ "arm_vcpu", "axaddrspace", diff --git a/Cargo.toml b/Cargo.toml index 942b4412..0d8f720b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ axstd = { git = "https://github.com/arceos-hypervisor/arceos.git", branch = "vmm axvm = { git = "https://github.com/arceos-hypervisor/axvm.git" } axvcpu = { git = "https://github.com/arceos-hypervisor/axvcpu.git" } axaddrspace = { git = "https://github.com/arceos-hypervisor/axaddrspace.git" } +axvmconfig = { git = "https://github.com/arceos-hypervisor/axvmconfig.git", default-features = false } # System independent crates provided by ArceOS, these crates could be imported by remote url. crate_interface = "0.1" @@ -59,3 +60,8 @@ axstd = { path = "./crates/arceos/ulib/axstd" } [patch."https://github.com/arceos-hypervisor/x86_vcpu.git"] x86_vcpu = { path = "./crates/x86_vcpu" } +[patch."https://github.com/arceos-hypervisor/axvm.git"] +axvm = { path = "./crates/axvm" } + +[patch."https://github.com/arceos-hypervisor/axvcpu.git"] +axvcpu = { path = "./crates/axvcpu" } \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 66cc337e..f694deed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,11 +18,11 @@ fn main() { info!("Hardware support: {:?}", axvm::has_hardware_support()); - // hal::enable_virtualization(); + hal::enable_virtualization(); - // vmm::init(); + vmm::init(); - // vmm::start(); + vmm::start(); info!("VMM shutdown"); } diff --git a/src/vmm/config.rs b/src/vmm/config.rs index 260d785d..82a19c3e 100644 --- a/src/vmm/config.rs +++ b/src/vmm/config.rs @@ -1,3 +1,5 @@ +use std::vec::Vec; + use axvm::config::{AxVMConfig, AxVMCrateConfig}; use crate::vmm::{VM, images::load_vm_images, vm_list::push_vm}; @@ -41,3 +43,35 @@ pub fn init_guest_vms() { load_vm_images(vm_create_config, vm.clone()).expect("Failed to load VM images"); } } + +pub fn init_host_vm() { + use crate::alloc::string::ToString; + + use std::os::arceos::modules::axhal::host_memory_regions; + use std::os::arceos::modules::{axconfig, axhal}; + + use axvm::config::AxVMConfig; + use axvmconfig::{VmMemConfig, VmMemMappingType}; + + let mut host_vm_cfg = AxVMConfig::new_host(0, "host".to_string(), axconfig::SMP); + + for region in host_memory_regions() { + host_vm_cfg.append_memory_region(VmMemConfig { + gpa: region.paddr.as_usize(), + size: region.size, + flags: region.flags.bits(), + map_type: VmMemMappingType::MapIentical, + }); + } + + let mut linux_cpus = Vec::new(); + + for cpu_id in 0..axconfig::SMP { + let linux_cpu_context = axhal::get_linux_context_by_cpu_id(cpu_id); + linux_cpus.push(linux_cpu_context); + } + + // Create VM. + let vm = VM::new(host_vm_cfg).expect("Failed to create VM"); + push_vm(vm.clone()); +} diff --git a/src/vmm/mod.rs b/src/vmm/mod.rs index a3d32041..b8229234 100644 --- a/src/vmm/mod.rs +++ b/src/vmm/mod.rs @@ -1,5 +1,7 @@ mod config; mod images; + +#[allow(dead_code)] mod timer; mod vcpus; mod vm_list; @@ -22,28 +24,47 @@ static VMM: AxWaitQueueHandle = AxWaitQueueHandle::new(); static RUNNING_VM_COUNT: AtomicUsize = AtomicUsize::new(0); pub fn init() { + config::init_host_vm(); // Initialize guest VM according to config file. config::init_guest_vms(); // Setup vcpus, spawn axtask for primary VCpu. info!("Setting up vcpus..."); - for vm in vm_list::get_vm_list() { - vcpus::setup_vm_primary_vcpu(vm); - } + + vm_list::manipulate_each_vm(|vm| { + if vm.is_host_vm() { + vcpus::setup_vm_all_cpus(vm); + } else { + vcpus::setup_vm_primary_vcpu(vm); + } + }); } pub fn start() { info!("VMM starting, booting VMs..."); - for vm in vm_list::get_vm_list() { - match vm.boot() { - Ok(_) => { + vm_list::manipulate_each_vm(|vm| { + let _ = vm + .boot() + .inspect(|_| { vcpus::notify_primary_vcpu(vm.id()); RUNNING_VM_COUNT.fetch_add(1, Ordering::Release); - info!("VM[{}] boot success", vm.id()) - } - Err(err) => warn!("VM[{}] boot failed, error {:?}", vm.id(), err), - } - } + info!("VM[{}] boot success", vm.id()); + }) + .inspect_err(|err| { + warn!("VM[{}] boot failed, error {:?}", vm.id(), err); + }); + }); + + // for vm in vm_list::get_vm_list() { + // match vm.boot() { + // Ok(_) => { + // vcpus::notify_primary_vcpu(vm.id()); + // RUNNING_VM_COUNT.fetch_add(1, Ordering::Release); + // info!("VM[{}] boot success", vm.id()) + // } + // Err(err) => warn!("VM[{}] boot failed, error {:?}", vm.id(), err), + // } + // } // Do not exit until all VMs are stopped. task::ax_wait_queue_wait_until(&VMM, || RUNNING_VM_COUNT.load(Ordering::Acquire) == 0, None); diff --git a/src/vmm/timer.rs b/src/vmm/timer.rs index 0d98704c..4708d2ee 100644 --- a/src/vmm/timer.rs +++ b/src/vmm/timer.rs @@ -1,5 +1,3 @@ -extern crate alloc; - use core::sync::atomic::AtomicUsize; use core::sync::atomic::Ordering; diff --git a/src/vmm/vcpus.rs b/src/vmm/vcpus.rs index 5db545f2..9aa8317b 100644 --- a/src/vmm/vcpus.rs +++ b/src/vmm/vcpus.rs @@ -43,7 +43,7 @@ impl VMVcpus { /// # Returns /// /// A new `VMVcpus` instance with an empty task list and a fresh wait queue. - fn new(vm: VMRef) -> Self { + fn new(vm: &VMRef) -> Self { Self { _vm_id: vm.id(), wait_queue: WaitQueue::new(), @@ -175,7 +175,7 @@ fn vcpu_on(vm: VMRef, vcpu_id: usize, entry_point: GuestPhysAddr, arg: usize) { pub fn setup_vm_primary_vcpu(vm: VMRef) { info!("Initializing VM[{}]'s {} vcpus", vm.id(), vm.vcpu_num()); let vm_id = vm.id(); - let mut vm_vcpus = VMVcpus::new(vm.clone()); + let mut vm_vcpus = VMVcpus::new(&vm); let primary_vcpu_id = 0; @@ -187,6 +187,33 @@ pub fn setup_vm_primary_vcpu(vm: VMRef) { } } +pub fn setup_vm_all_cpus(vm: VMRef) { + info!( + "Initializing VM[{}, {}]'s {} vcpus", + vm.id(), + vm.name(), + vm.vcpu_num() + ); + + if !vm.is_host_vm() { + warn!("setup_vm_all_cpus: not host vm"); + return; + } + + let vm_id = vm.id(); + let mut vm_vcpus = VMVcpus::new(&vm); + + for vcpu_id in 0..vm.vcpu_num() { + let vcpu = vm.vcpu_list()[vcpu_id].clone(); + let vcpu_task = alloc_vcpu_task(vm.clone(), vcpu); + vm_vcpus.add_vcpu_task(vcpu_task); + } + + unsafe { + VM_VCPU_TASK_WAIT_QUEUE.insert(vm_id, vm_vcpus); + } +} + /// Allocates arceos task for vcpu, set the task's entry function to [`vcpu_run()`], /// alse initializes the CPU mask if the vCPU has a dedicated physical CPU set. /// diff --git a/src/vmm/vm_list.rs b/src/vmm/vm_list.rs index 56a2f45e..3e7ec348 100644 --- a/src/vmm/vm_list.rs +++ b/src/vmm/vm_list.rs @@ -1,5 +1,4 @@ use alloc::collections::BTreeMap; -use alloc::vec::Vec; use spin::Mutex; @@ -106,11 +105,13 @@ pub fn get_vm_by_id(vm_id: usize) -> Option { GLOBAL_VM_LIST.lock().get_vm_by_id(vm_id) } -pub fn get_vm_list() -> Vec { - let global_vm_list = GLOBAL_VM_LIST.lock().vm_list.clone(); - let mut vm_list = Vec::with_capacity(global_vm_list.len()); - for (_id, vm) in global_vm_list { - vm_list.push(vm.clone()); +pub fn manipulate_each_vm(f: F) +where + F: Fn(VMRef), +{ + let vm_list_locked = GLOBAL_VM_LIST.lock(); + for vm in vm_list_locked.vm_list.values() { + debug!("Manipulating VM[{}]...", vm.id()); + f(vm.clone()); } - vm_list } From d565dbc6601b9f02d1bd4328a80a8c6a84c8c296 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Thu, 20 Mar 2025 22:11:40 +0800 Subject: [PATCH 4/6] [feat] support type15 in axvisor --- Cargo.toml | 3 ++- rust-toolchain.toml | 3 +++ src/hal.rs | 5 ++--- src/vmm/config.rs | 36 ++++++++++++++++++++++------------ src/vmm/mod.rs | 23 +++++----------------- src/vmm/vcpus.rs | 48 +++++++++++++++++++++++++++++++++++++-------- 6 files changed, 76 insertions(+), 42 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0d8f720b..3a3fbcff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ license = "GPL-3.0-or-later OR Apache-2.0 OR MulanPubL-2.0 OR MulanPSL2" [features] fs = ["axstd/fs"] +irq = ["axstd/irq"] [dependencies] log = "=0.4.21" @@ -26,7 +27,7 @@ axstd = { git = "https://github.com/arceos-hypervisor/arceos.git", branch = "vmm "alloc", "paging", # "fs", - "irq", + # "irq", "hv", "multitask", # "sched_rr", diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 7c587ea0..3080634a 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -3,3 +3,6 @@ profile = "minimal" channel = "nightly-2024-12-25" components = ["rust-src", "llvm-tools", "rustfmt", "clippy"] targets = ["x86_64-unknown-none", "riscv64gc-unknown-none-elf", "aarch64-unknown-none", "aarch64-unknown-none-softfloat"] + +[profile.release] +debug = true \ No newline at end of file diff --git a/src/hal.rs b/src/hal.rs index 3a73f51c..1c55d4e2 100644 --- a/src/hal.rs +++ b/src/hal.rs @@ -8,8 +8,6 @@ use axaddrspace::{HostPhysAddr, HostVirtAddr}; use axvcpu::AxVCpuHal; use axvm::{AxVMHal, AxVMPerCpu}; -use crate::vmm; - /// Implementation for `AxVMHal` trait. pub struct AxVMHalImpl; @@ -104,7 +102,8 @@ pub(crate) fn enable_virtualization() { "Initialize CPU affinity failed!" ); - vmm::init_timer_percpu(); + #[cfg(feature = "irq")] + crate::vmm::init_timer_percpu(); let percpu = unsafe { AXVM_PER_CPU.current_ref_mut_raw() }; percpu diff --git a/src/vmm/config.rs b/src/vmm/config.rs index 82a19c3e..e8b9a35d 100644 --- a/src/vmm/config.rs +++ b/src/vmm/config.rs @@ -12,12 +12,12 @@ pub mod config { #[allow(dead_code)] pub fn default_static_vm_configs() -> Vec<&'static str> { vec![ - #[cfg(target_arch = "x86_64")] - core::include_str!("../../configs/vms/nimbos-x86_64.toml"), - #[cfg(target_arch = "aarch64")] - core::include_str!("../../configs/vms/nimbos-aarch64.toml"), - #[cfg(target_arch = "riscv64")] - core::include_str!("../../configs/vms/nimbos-riscv64.toml"), + // #[cfg(target_arch = "x86_64")] + // core::include_str!("../../configs/vms/nimbos-x86_64.toml"), + // #[cfg(target_arch = "aarch64")] + // core::include_str!("../../configs/vms/nimbos-aarch64.toml"), + // #[cfg(target_arch = "riscv64")] + // core::include_str!("../../configs/vms/nimbos-riscv64.toml"), ] } @@ -53,6 +53,8 @@ pub fn init_host_vm() { use axvm::config::AxVMConfig; use axvmconfig::{VmMemConfig, VmMemMappingType}; + info!("Creating host VM..."); + let mut host_vm_cfg = AxVMConfig::new_host(0, "host".to_string(), axconfig::SMP); for region in host_memory_regions() { @@ -64,14 +66,24 @@ pub fn init_host_vm() { }); } - let mut linux_cpus = Vec::new(); + // let mut linux_cpus = Vec::new(); - for cpu_id in 0..axconfig::SMP { - let linux_cpu_context = axhal::get_linux_context_by_cpu_id(cpu_id); - linux_cpus.push(linux_cpu_context); - } + // for cpu_id in 0..axconfig::SMP { + // let linux_cpu_context = axhal::get_linux_context_by_cpu_id(cpu_id); + // debug!( + // "linux cpu {} at {:#x}", + // cpu_id, linux_cpu_context as *const _ as usize + // ); + + // linux_cpus.push(linux_cpu_context); + // } + + let linux_cpus = axhal::get_linux_context_list(); + + debug!("linux cpus at {:#x}", linux_cpus.as_ptr() as usize); // Create VM. - let vm = VM::new(host_vm_cfg).expect("Failed to create VM"); + let vm = + VM::new_host(host_vm_cfg, axhal::get_linux_context_list()).expect("Failed to create VM"); push_vm(vm.clone()); } diff --git a/src/vmm/mod.rs b/src/vmm/mod.rs index b8229234..7a043627 100644 --- a/src/vmm/mod.rs +++ b/src/vmm/mod.rs @@ -1,7 +1,7 @@ mod config; mod images; -#[allow(dead_code)] +#[cfg(feature = "irq")] mod timer; mod vcpus; mod vm_list; @@ -12,6 +12,8 @@ use core::sync::atomic::AtomicUsize; use core::sync::atomic::Ordering; use crate::hal::{AxVCpuHalImpl, AxVMHalImpl}; + +#[cfg(feature = "irq")] pub use timer::init_percpu as init_timer_percpu; pub type VM = axvm::AxVM; @@ -32,11 +34,7 @@ pub fn init() { info!("Setting up vcpus..."); vm_list::manipulate_each_vm(|vm| { - if vm.is_host_vm() { - vcpus::setup_vm_all_cpus(vm); - } else { - vcpus::setup_vm_primary_vcpu(vm); - } + vcpus::setup_vm_cpu(vm); }); } @@ -46,7 +44,7 @@ pub fn start() { let _ = vm .boot() .inspect(|_| { - vcpus::notify_primary_vcpu(vm.id()); + vcpus::boot_vm_cpu(&vm); RUNNING_VM_COUNT.fetch_add(1, Ordering::Release); info!("VM[{}] boot success", vm.id()); }) @@ -55,17 +53,6 @@ pub fn start() { }); }); - // for vm in vm_list::get_vm_list() { - // match vm.boot() { - // Ok(_) => { - // vcpus::notify_primary_vcpu(vm.id()); - // RUNNING_VM_COUNT.fetch_add(1, Ordering::Release); - // info!("VM[{}] boot success", vm.id()) - // } - // Err(err) => warn!("VM[{}] boot failed, error {:?}", vm.id(), err), - // } - // } - // Do not exit until all VMs are stopped. task::ax_wait_queue_wait_until(&VMM, || RUNNING_VM_COUNT.load(Ordering::Acquire) == 0, None); } diff --git a/src/vmm/vcpus.rs b/src/vmm/vcpus.rs index 9aa8317b..eb4b5cef 100644 --- a/src/vmm/vcpus.rs +++ b/src/vmm/vcpus.rs @@ -77,6 +77,10 @@ impl VMVcpus { fn notify_one(&mut self) { self.wait_queue.notify_one(false); } + + fn notify_all(&mut self) { + self.wait_queue.notify_all(false); + } } /// Blocks the current thread until it is explicitly woken up, using the wait queue @@ -116,13 +120,20 @@ where /// /// * `vm_id` - The ID of the VM whose vCPUs are to be notified. /// -pub(crate) fn notify_primary_vcpu(vm_id: usize) { +fn notify_primary_vcpu(vm_id: usize) { // Generally, the primary vCPU is the first and **only** vCPU in the list. unsafe { VM_VCPU_TASK_WAIT_QUEUE.get_mut(&vm_id) } .unwrap() .notify_one() } +/// Boots all vCPUs of the specified VM. +fn notify_all_vcpus(vm_id: usize) { + unsafe { VM_VCPU_TASK_WAIT_QUEUE.get_mut(&vm_id) } + .unwrap() + .notify_all() +} + /// Boot target vCPU on the specified VM. /// This function is used to boot a secondary vCPU on a VM, setting the entry point and argument for the vCPU. /// @@ -172,7 +183,7 @@ fn vcpu_on(vm: VMRef, vcpu_id: usize, entry_point: GuestPhysAddr, arg: usize) { /// # Arguments /// /// * `vm` - A reference to the VM for which the vCPUs are being set up. -pub fn setup_vm_primary_vcpu(vm: VMRef) { +fn setup_vm_primary_vcpu(vm: VMRef) { info!("Initializing VM[{}]'s {} vcpus", vm.id(), vm.vcpu_num()); let vm_id = vm.id(); let mut vm_vcpus = VMVcpus::new(&vm); @@ -187,7 +198,7 @@ pub fn setup_vm_primary_vcpu(vm: VMRef) { } } -pub fn setup_vm_all_cpus(vm: VMRef) { +fn setup_vm_all_cpus(vm: VMRef) { info!( "Initializing VM[{}, {}]'s {} vcpus", vm.id(), @@ -201,16 +212,36 @@ pub fn setup_vm_all_cpus(vm: VMRef) { } let vm_id = vm.id(); - let mut vm_vcpus = VMVcpus::new(&vm); + unsafe { + VM_VCPU_TASK_WAIT_QUEUE.insert(vm_id, VMVcpus::new(&vm)); + } for vcpu_id in 0..vm.vcpu_num() { let vcpu = vm.vcpu_list()[vcpu_id].clone(); let vcpu_task = alloc_vcpu_task(vm.clone(), vcpu); - vm_vcpus.add_vcpu_task(vcpu_task); + + unsafe { + VM_VCPU_TASK_WAIT_QUEUE + .get_mut(&vm_id) + .unwrap() + .add_vcpu_task(vcpu_task); + } } +} - unsafe { - VM_VCPU_TASK_WAIT_QUEUE.insert(vm_id, vm_vcpus); +pub fn setup_vm_cpu(vm: VMRef) { + if vm.is_host_vm() { + setup_vm_all_cpus(vm); + } else { + setup_vm_primary_vcpu(vm); + } +} + +pub fn boot_vm_cpu(vm: &VMRef) { + if vm.is_host_vm() { + notify_all_vcpus(vm.id()); + } else { + notify_primary_vcpu(vm.id()); } } @@ -232,7 +263,7 @@ pub fn setup_vm_all_cpus(vm: VMRef) { /// * The task is scheduled on the scheduler of arceos after it is spawned. fn alloc_vcpu_task(vm: VMRef, vcpu: VCpuRef) -> AxTaskRef { info!("Spawning task for VM[{}] Vcpu[{}]", vm.id(), vcpu.id()); - let mut vcpu_task = TaskInner::new( + let mut vcpu_task: TaskInner = TaskInner::new( vcpu_run, format!("VM[{}]-VCpu[{}]", vm.id(), vcpu.id()), KERNEL_STACK_SIZE, @@ -283,6 +314,7 @@ fn vcpu_run() { "VM[{}] VCpu[{}] run failed with exit code {}", vm_id, vcpu_id, hardware_entry_failure_reason ); + wait(vm_id) } AxVCpuExitReason::ExternalInterrupt { vector } => { debug!("VM[{}] run VCpu[{}] get irq {}", vm_id, vcpu_id, vector); From dfe4d1ece33038b49a2a9a61757c9357467118d7 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Wed, 26 Mar 2025 15:25:10 +0800 Subject: [PATCH 5/6] [feat] boot secondard cores belongs to arceos itself, bug after retuen to Linux --- Cargo.lock | 16 ++++++++- Cargo.toml | 7 ++-- Makefile | 4 +++ configs/platforms/x86_64-qemu-linux.toml | 42 ++++++++---------------- scripts/dev_deps.sh | 16 +++++++++ scripts/vmm/host/Makefile | 2 +- src/hal.rs | 24 +++++++------- src/vmm/config.rs | 27 ++++----------- 8 files changed, 73 insertions(+), 65 deletions(-) create mode 100644 scripts/dev_deps.sh diff --git a/Cargo.lock b/Cargo.lock index 4f6a6e6e..d048c03a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,7 +175,6 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "axaddrspace" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/axaddrspace.git#f1ab1108c1477f6f4f56035b74ea76c8932d6b8d" dependencies = [ "axerrno", "bit_field", @@ -522,6 +521,7 @@ dependencies = [ "axaddrspace", "axerrno", "memory_addr", + "page_table_multiarch", "percpu", ] @@ -852,6 +852,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "iced-x86" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c447cff8c7f384a7d4f741cfcff32f75f3ad02b406432e8d6c878d56b1edf6b" +dependencies = [ + "lazy_static", +] + [[package]] name = "indexmap" version = "2.8.0" @@ -899,6 +908,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "lazyinit" @@ -1672,10 +1684,12 @@ dependencies = [ "bitflags 2.9.0", "cfg-if", "crate_interface", + "iced-x86", "log", "memory_addr", "numeric-enum-macro", "page_table_entry", + "page_table_multiarch", "raw-cpuid 11.5.0", "x86", "x86_64 0.15.2", diff --git a/Cargo.toml b/Cargo.toml index 3a3fbcff..dfac87ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ axstd = { git = "https://github.com/arceos-hypervisor/arceos.git", branch = "vmm "multitask", # "sched_rr", "smp", -]} +] } # System dependent modules provided by ArceOS-Hypervisor. axvm = { git = "https://github.com/arceos-hypervisor/axvm.git" } @@ -65,4 +65,7 @@ x86_vcpu = { path = "./crates/x86_vcpu" } axvm = { path = "./crates/axvm" } [patch."https://github.com/arceos-hypervisor/axvcpu.git"] -axvcpu = { path = "./crates/axvcpu" } \ No newline at end of file +axvcpu = { path = "./crates/axvcpu" } + +[patch."https://github.com/arceos-hypervisor/axaddrspace.git"] +axaddrspace = { path = "./crates/axaddrspace" } diff --git a/Makefile b/Makefile index 51487ac6..575636ff 100644 --- a/Makefile +++ b/Makefile @@ -180,6 +180,10 @@ debug: build scp_linux: $(OUT_DIR) $(OUT_BIN) $(call scp_bin) +.PNONY: ssh +ssh: + ssh -p $(PORT) ubuntu@localhost + gdb: $(GDB) $(OUT_ELF) \ -ex 'target remote localhost:1234' \ diff --git a/configs/platforms/x86_64-qemu-linux.toml b/configs/platforms/x86_64-qemu-linux.toml index 9a7dac7c..88b144d2 100644 --- a/configs/platforms/x86_64-qemu-linux.toml +++ b/configs/platforms/x86_64-qemu-linux.toml @@ -11,49 +11,35 @@ platform = "x86_64-qemu-linux" # str family = "x86-linux" # str # Base address of the whole physical memory. -# phys-memory-base = "0x10000_0000" # uint -phys-memory-base = "0x3a00_0000" # uint +phys-memory-base = "0x4000_0000" # uint # Size of the whole physical memory. (# 256MB) phys-memory-size = "0x1000_0000" # uint # Base physical address of the kernel image. # kernel-base-paddr = "0x10000_0000" # uint -kernel-base-paddr = "0x3a00_0000" # uint +kernel-base-paddr = "0x4000_0000" # uint # Base virtual address of the kernel image. # kernel-base-vaddr = "0xffff_8000_0000_0000" # uint -kernel-base-vaddr = "0xffff_ff00_0000_0000" # uint +kernel-base-vaddr = "0xffff_ff80_0000_0000" # uint # Linear mapping offset, for quick conversions between physical and virtual # addresses. (# kernel-base-vaddr - phys-memory-base) -# 0xffff_8000_0000_0000 - 0x3a00_0000 -# phys-virt-offset = "0xffff_7fff_c600_0000" # uint -# 0xffff_ff00_0000_0000 - 0x3a00_0000 -phys-virt-offset = "0xffff_feff_c600_0000" # uint +# 0xffff_ff80_0000_0000 - 0x4000_0000 +phys-virt-offset = "0xffff_ff7f_c000_0000" # uint # Offset of bus address and phys address. some boards, the bus address is # different from the physical address. phys-bus-offset = 0 # uint # Kernel address space base. -# kernel-aspace-base = "0xffff_8000_0000_0000" # uint -# Note: this is extremely tricky, the kernel address space base should be 0xffff_8000_0000_0000 in fact. -# But, to map Linux's low memory address (below 0x3a00_0000) to the kernel address space, -# (for example, we need to map 0x1000 to 0xffff7fffc6001000), -# we just simply decreate the kernel address space base by 0x1000_0000_0000, to 0xffff700000000000. -# So, the kernel address space base is 0xffff700000000000, and the kernel address space size is 0x00008fffffff0000. -# kernel-aspace-base = "0xffff_7000_0000_0000" # uint -# kernel-aspace-size = "0x0000_7fff_ffff_f000" # uint -# kernel-aspace-size = "0x0000_8fff_ffff_f000" # uint -# kernel-aspace-base = "0xffff_ff00_0000_0000" # uint -# Note: this is extremely tricky, the kernel address space base should be 0xffff_ff00_0000_0000 in fact. -# But, to map Linux's low memory address (below 0x3a00_0000) to the kernel address space, -# (for example, we need to map 0x1000 to 0xfffffeffc6001000), -# we just simply decreate the kernel address space base by 0x1_0000_0000, to 0xffff_feff_0000_0000. -# So, the kernel address space base is 0xffff_feff_0000_0000, and the kernel address space size is 0x0000_0100_ffff_f000. -kernel-aspace-base = "0xffff_feff_0000_0000" # uint +# kernel-aspace-base = "0xffff_ff80_0000_0000" # uint +# Note: this is extremely tricky, the kernel address space base should be 0xffff_ff80_0000_0000 in fact. +# But, to map Linux's low memory address (below 0x4000_0000) to the kernel address space, +# (for example, we need to map 0x1000 to 0xffffff7fc0001000), +# we just simply decreate the kernel address space base by 0x10_0000_0000, to 0xffff_ff70_0000_0000. +# So, the kernel address space base is 0xffff_ff70_0000_0000, and the kernel address space size is 0x0000_008f_ffff_f000. +kernel-aspace-base = "0xffff_ff70_0000_0000" # uint # Kernel address space size. -# kernel-aspace-size = "0x0000_00ff_ffff_f000" # uint -kernel-aspace-size = "0x0000_0100_ffff_f000" # uint - -# FFFFFEFFC6000000 +# kernel-aspace-size = "0x0000_007f_ffff_f000" # uint +kernel-aspace-size = "0x0000_008f_ffff_f000" # uint # # Device specifications diff --git a/scripts/dev_deps.sh b/scripts/dev_deps.sh new file mode 100644 index 00000000..658be33f --- /dev/null +++ b/scripts/dev_deps.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Create directories in the parent directory +mkdir -p ../crates ../deps + +# Clone repositories into the crates directory +cd ../crates || exit +git clone git@github.com:EquationOS/arceos.git --branch vmm_type15 +git clone git@github.com:arceos-hypervisor/axaddrspace.git --branch type15 +git clone git@github.com:arceos-hypervisor/axvm.git --branch type15 +git clone git@github.com:arceos-hypervisor/axvcpu.git --branch type15 +git clone git@github.com:arceos-hypervisor/x86_vcpu.git --branch type15 + +# Clone repository into the deps directory +cd ../deps || exit +git clone git@github.com:EquationOS/jailhouse-equation.git --branch axvisor \ No newline at end of file diff --git a/scripts/vmm/host/Makefile b/scripts/vmm/host/Makefile index 4b1caf2d..5eedc515 100644 --- a/scripts/vmm/host/Makefile +++ b/scripts/vmm/host/Makefile @@ -23,7 +23,7 @@ qemu_args := \ -drive file=$(qemu_image) \ -net user,id=net,hostfwd=tcp::$(PORT)-:22 -net nic,model=e1000e \ -serial mon:stdio \ - -serial telnet:localhost:$(TELNET_PORT),server#,nowait + -serial telnet:localhost:$(TELNET_PORT),server,nowait $(qemu_image): wget -nc https://cloud-images.ubuntu.com/releases/$(RELEASE_NAME)/release/$(qemu_image) diff --git a/src/hal.rs b/src/hal.rs index 1c55d4e2..a3fab95a 100644 --- a/src/hal.rs +++ b/src/hal.rs @@ -1,7 +1,6 @@ use std::os::arceos; use memory_addr::{PAGE_SIZE_4K, align_up_4k}; -use page_table_multiarch::PagingHandler; use arceos::modules::{axalloc, axhal}; use axaddrspace::{HostPhysAddr, HostVirtAddr}; @@ -45,23 +44,22 @@ impl AxVMHal for AxVMHalImpl { } } -pub struct AxVCpuHalImpl; +pub struct EPTTranslatorImpl; -impl AxVCpuHal for AxVCpuHalImpl { - fn alloc_frame() -> Option { - ::PagingHandler::alloc_frame() +impl axaddrspace::EPTTranslator for EPTTranslatorImpl { + fn guest_phys_to_host_phys(gpa: axaddrspace::GuestPhysAddr) -> Option { + use std::os::arceos::modules::axtask::{self, TaskExtRef}; + axtask::current().task_ext().vm.guest_phys_to_host_phys(gpa) } +} - fn dealloc_frame(paddr: HostPhysAddr) { - ::PagingHandler::dealloc_frame(paddr) - } +pub struct AxVCpuHalImpl; - #[inline] - fn phys_to_virt(paddr: HostPhysAddr) -> HostVirtAddr { - ::PagingHandler::phys_to_virt(paddr) - } +impl AxVCpuHal for AxVCpuHalImpl { + type EPTTranslator = EPTTranslatorImpl; + type PagingHandler = axhal::paging::PagingHandlerImpl; - fn virt_to_phys(vaddr: axaddrspace::HostVirtAddr) -> axaddrspace::HostPhysAddr { + fn virt_to_phys(vaddr: HostVirtAddr) -> axaddrspace::HostPhysAddr { std::os::arceos::modules::axhal::mem::virt_to_phys(vaddr) } diff --git a/src/vmm/config.rs b/src/vmm/config.rs index e8b9a35d..3a40610e 100644 --- a/src/vmm/config.rs +++ b/src/vmm/config.rs @@ -47,17 +47,20 @@ pub fn init_guest_vms() { pub fn init_host_vm() { use crate::alloc::string::ToString; - use std::os::arceos::modules::axhal::host_memory_regions; - use std::os::arceos::modules::{axconfig, axhal}; + use std::os::arceos::modules::axhal; use axvm::config::AxVMConfig; use axvmconfig::{VmMemConfig, VmMemMappingType}; info!("Creating host VM..."); - let mut host_vm_cfg = AxVMConfig::new_host(0, "host".to_string(), axconfig::SMP); + let mut host_vm_cfg = AxVMConfig::new_host( + 0, + "host".to_string(), + axhal::hvheader::HvHeader::get().reserved_cpus() as usize, + ); - for region in host_memory_regions() { + for region in axhal::host_memory_regions() { host_vm_cfg.append_memory_region(VmMemConfig { gpa: region.paddr.as_usize(), size: region.size, @@ -66,22 +69,6 @@ pub fn init_host_vm() { }); } - // let mut linux_cpus = Vec::new(); - - // for cpu_id in 0..axconfig::SMP { - // let linux_cpu_context = axhal::get_linux_context_by_cpu_id(cpu_id); - // debug!( - // "linux cpu {} at {:#x}", - // cpu_id, linux_cpu_context as *const _ as usize - // ); - - // linux_cpus.push(linux_cpu_context); - // } - - let linux_cpus = axhal::get_linux_context_list(); - - debug!("linux cpus at {:#x}", linux_cpus.as_ptr() as usize); - // Create VM. let vm = VM::new_host(host_vm_cfg, axhal::get_linux_context_list()).expect("Failed to create VM"); From 46ef42c5ae44f37f0fcb5681292e9b1ea3a98172 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Thu, 27 Mar 2025 10:15:06 +0800 Subject: [PATCH 6/6] [fix] improve scripts, boot from Linux --- HOW-to.md | 25 ++++++++++++++++--- ...sable-arceos-vmm.sh => disable-axvisor.sh} | 0 ...enable-arceos-vmm.sh => enable-axvisor.sh} | 0 src/vmm/config.rs | 2 -- 4 files changed, 22 insertions(+), 5 deletions(-) rename scripts/vmm/guest/{disable-arceos-vmm.sh => disable-axvisor.sh} (100%) rename scripts/vmm/guest/{enable-arceos-vmm.sh => enable-axvisor.sh} (100%) diff --git a/HOW-to.md b/HOW-to.md index 312fc713..12ffcf1f 100644 --- a/HOW-to.md +++ b/HOW-to.md @@ -19,7 +19,7 @@ make PLATFORM=x86_64-qemu-linux SMP=4 LOG=debug scp_linux 1. Download the guest image and run in QEMU: ```bash - cd scripts/host + cd scripts/vmm/host make image # download image and configure for the first time make qemu # execute this command only for subsequent runs ``` @@ -29,7 +29,7 @@ make PLATFORM=x86_64-qemu-linux SMP=4 LOG=debug scp_linux 2. Copy helpful scripts into the guest OS: ```bash - scp -P 2334 scripts/guest/* ubuntu@localhost:/home/ubuntu # in host + scp -P 2334 scripts/vmm/guest/* ubuntu@localhost:/home/ubuntu # in host ``` 3. Setup in the guest OS: @@ -46,4 +46,23 @@ make PLATFORM=x86_64-qemu-linux SMP=4 LOG=debug scp_linux ```bash ssh -p 2334 ubuntu@localhost # in host ./setup.sh # in guest - ``` \ No newline at end of file + ``` + +4. Compile Jailhouse: + + You need to do this each time after modifing the jailhouse code. + + ``` + cd jailhouse-equation + make + ``` + + To change the CPU number reserved for ArceOS, modify the `RT_CPUS` macro in `jailhouse-equation/tools/jailhouse.c`. + +5. Enable AxVisor + + `./enable-axvisor.sh` + +## Development + + `cd scripts/ && ./dev_deps.sh` diff --git a/scripts/vmm/guest/disable-arceos-vmm.sh b/scripts/vmm/guest/disable-axvisor.sh similarity index 100% rename from scripts/vmm/guest/disable-arceos-vmm.sh rename to scripts/vmm/guest/disable-axvisor.sh diff --git a/scripts/vmm/guest/enable-arceos-vmm.sh b/scripts/vmm/guest/enable-axvisor.sh similarity index 100% rename from scripts/vmm/guest/enable-arceos-vmm.sh rename to scripts/vmm/guest/enable-axvisor.sh diff --git a/src/vmm/config.rs b/src/vmm/config.rs index 3a40610e..0f033441 100644 --- a/src/vmm/config.rs +++ b/src/vmm/config.rs @@ -1,5 +1,3 @@ -use std::vec::Vec; - use axvm::config::{AxVMConfig, AxVMCrateConfig}; use crate::vmm::{VM, images::load_vm_images, vm_list::push_vm};