chore: move vmm::vstate code under vmm::arch

Move the rest of vstate architecture-specific code under `vmm::arch`.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

src/vmm/src/arch/aarch64/vcpu.rs

@@ -13,7 +13,7 @@ use kvm_ioctls::VcpuFd;
 
 use super::get_fdt_addr;
 use super::regs::*;
-use crate::vstate::kvm::OptionalCapabilities;
+use crate::arch::OptionalCapabilities;
 use crate::vstate::memory::GuestMemoryMmap;
 
 /// Errors thrown while setting aarch64 registers.

src/vmm/src/arch/aarch64/vstate/mod.rs

@@ -3,3 +3,7 @@
 
 /// Architecture specific KVM-related code
 pub mod kvm;
+/// Architecture specific vCPU code
+pub mod vcpu;
+/// Architecture specific VM state code
+pub mod vm;

src/vmm/src/vstate/vcpu/aarch64.rs renamed to src/vmm/src/arch/aarch64/vstate/vcpu.rs

@@ -13,17 +13,16 @@ use kvm_bindings::{
 use kvm_ioctls::*;
 use serde::{Deserialize, Serialize};
 
-use crate::arch::EntryPoint;
 use crate::arch::aarch64::regs::{Aarch64RegisterVec, KVM_REG_ARM64_SVE_VLS};
 use crate::arch::aarch64::vcpu::{
     VcpuError as ArchError, get_all_registers, get_all_registers_ids, get_mpidr, get_mpstate,
     get_registers, set_mpstate, set_register, setup_boot_regs,
 };
+use crate::arch::{EntryPoint, OptionalCapabilities};
 use crate::cpu_config::aarch64::custom_cpu_template::VcpuFeatures;
 use crate::cpu_config::templates::CpuConfiguration;
 use crate::logger::{IncMetric, METRICS, error};
 use crate::vcpu::{VcpuConfig, VcpuError};
-use crate::vstate::kvm::OptionalCapabilities;
 use crate::vstate::memory::{Address, GuestMemoryMmap};
 use crate::vstate::vcpu::VcpuEmulation;
 use crate::vstate::vm::Vm;
@@ -60,14 +59,14 @@ pub struct KvmVcpu {
     /// KVM vcpu fd.
     pub fd: VcpuFd,
     /// Vcpu peripherals, such as buses
-    pub(super) peripherals: Peripherals,
+    pub peripherals: Peripherals,
     mpidr: u64,
     kvi: kvm_vcpu_init,
 }
 
 /// Vcpu peripherals
 #[derive(Default, Debug)]
-pub(super) struct Peripherals {
+pub struct Peripherals {
     /// mmio bus.
     pub mmio_bus: Option<crate::devices::Bus>,
 }

src/vmm/src/vstate/vm/aarch64.rs renamed to src/vmm/src/arch/aarch64/vstate/vm.rs

@@ -4,14 +4,15 @@
 use kvm_ioctls::VmFd;
 use serde::{Deserialize, Serialize};
 
-use super::VmError;
 use crate::Kvm;
 use crate::arch::aarch64::gic::GicState;
+use crate::vstate::vm::VmError;
 
 /// Structure representing the current architecture's understand of what a "virtual machine" is.
 #[derive(Debug)]
 pub struct ArchVm {
-    pub(super) fd: VmFd,
+    /// KVM file descriptor of microVM
+    pub fd: VmFd,
     // On aarch64 we need to keep around the fd obtained by creating the VGIC device.
     irqchip_handle: Option<crate::arch::aarch64::gic::GICDevice>,
 }
@@ -37,11 +38,13 @@ impl ArchVm {
         })
     }
 
-    pub(super) fn arch_pre_create_vcpus(&mut self, _: u8) -> Result<(), ArchVmError> {
+    /// Pre-vCPU creation setup.
+    pub fn arch_pre_create_vcpus(&mut self, _: u8) -> Result<(), ArchVmError> {
         Ok(())
     }
 
-    pub(super) fn arch_post_create_vcpus(&mut self, nr_vcpus: u8) -> Result<(), ArchVmError> {
+    /// Post-vCPU creation setup.
+    pub fn arch_post_create_vcpus(&mut self, nr_vcpus: u8) -> Result<(), ArchVmError> {
         // On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) before setting up the
         // IRQ chip because the `KVM_CREATE_VCPU` ioctl will return error if the IRQCHIP
         // was already initialized.

src/vmm/src/arch/mod.rs

@@ -15,6 +15,10 @@ pub mod aarch64;
 #[cfg(target_arch = "aarch64")]
 pub use aarch64::vstate::kvm::{Kvm, KvmArchError, OptionalCapabilities};
 #[cfg(target_arch = "aarch64")]
+pub use aarch64::vstate::vcpu::*;
+#[cfg(target_arch = "aarch64")]
+pub use aarch64::vstate::vm::{ArchVm, ArchVmError, VmState};
+#[cfg(target_arch = "aarch64")]
 pub use aarch64::{
     ConfigurationError, MMIO_MEM_SIZE, MMIO_MEM_START, arch_memory_regions, configure_system,
     get_kernel_start, initrd_load_addr, layout::CMDLINE_MAX_SIZE, layout::IRQ_BASE,
@@ -26,14 +30,18 @@ pub use aarch64::{
 pub mod x86_64;
 
 #[cfg(target_arch = "x86_64")]
-pub use crate::arch::x86_64::{
+pub use x86_64::vstate::kvm::{Kvm, KvmArchError};
+#[cfg(target_arch = "x86_64")]
+pub use x86_64::vstate::vcpu::*;
+#[cfg(target_arch = "x86_64")]
+pub use x86_64::vstate::vm::{ArchVm, ArchVmError, VmState};
+#[cfg(target_arch = "x86_64")]
+pub use x86_64::{
     ConfigurationError, MMIO_MEM_SIZE, MMIO_MEM_START, arch_memory_regions, configure_system,
     get_kernel_start, initrd_load_addr, layout::APIC_ADDR, layout::CMDLINE_MAX_SIZE,
     layout::IOAPIC_ADDR, layout::IRQ_BASE, layout::IRQ_MAX, layout::SYSTEM_MEM_SIZE,
     layout::SYSTEM_MEM_START,
 };
-#[cfg(target_arch = "x86_64")]
-pub use x86_64::vstate::kvm::{Kvm, KvmArchError};
 
 /// Types of devices that can get attached to this platform.
 #[derive(Clone, Debug, PartialEq, Eq, Hash, Copy, Serialize, Deserialize)]
@@ -83,7 +91,7 @@ impl fmt::Display for DeviceType {
     }
 }
 
-/// Suported boot protocols for
+/// Supported boot protocols for
 #[derive(Debug, Copy, Clone, PartialEq)]
 pub enum BootProtocol {
     /// Linux 64-bit boot protocol

src/vmm/src/arch/x86_64/vstate/mod.rs

@@ -3,3 +3,7 @@
 
 /// Architecture specific KVM-related code
 pub mod kvm;
+/// Architecture specific vCPU code
+pub mod vcpu;
+/// Architecture specific VM state code
+pub mod vm;

src/vmm/src/vstate/vcpu/x86_64.rs renamed to src/vmm/src/arch/x86_64/vstate/vcpu.rs

@@ -25,7 +25,7 @@ use crate::arch::x86_64::regs::{SetupFpuError, SetupRegistersError, SetupSpecial
 use crate::cpu_config::x86_64::{CpuConfiguration, cpuid};
 use crate::logger::{IncMetric, METRICS};
 use crate::vstate::memory::GuestMemoryMmap;
-use crate::vstate::vcpu::{VcpuConfig, VcpuEmulation};
+use crate::vstate::vcpu::{VcpuConfig, VcpuEmulation, VcpuError};
 use crate::vstate::vm::Vm;
 
 // Tolerance for TSC frequency expected variation.
@@ -145,7 +145,7 @@ pub struct KvmVcpu {
     /// KVM vcpu fd.
     pub fd: VcpuFd,
     /// Vcpu peripherals, such as buses
-    pub(super) peripherals: Peripherals,
+    pub peripherals: Peripherals,
     /// The list of MSRs to include in a VM snapshot, in the same order as KVM returned them
     /// from KVM_GET_MSR_INDEX_LIST
     msrs_to_save: Vec<u32>,
@@ -157,7 +157,7 @@ pub struct KvmVcpu {
 
 /// Vcpu peripherals
 #[derive(Default, Debug)]
-pub(super) struct Peripherals {
+pub struct Peripherals {
     /// Pio bus.
     pub pio_bus: Option<crate::devices::Bus>,
     /// Mmio bus.
@@ -334,7 +334,7 @@ impl KvmVcpu {
     ///
     /// # Errors
     ///
-    /// When [`kvm_ioctls::VcpuFd::get_tsc_khz`] errrors.
+    /// When [`kvm_ioctls::VcpuFd::get_tsc_khz`] errors.
     pub fn get_tsc_khz(&self) -> Result<u32, GetTscError> {
         let res = self.fd.get_tsc_khz()?;
         Ok(res)
@@ -477,7 +477,7 @@ impl KvmVcpu {
     /// # Arguments
     ///
     /// * `msr_index_iter`: Iterator over MSR indices.
-    /// * `chunk_size`: Lenght of a chunk.
+    /// * `chunk_size`: Length of a chunk.
     ///
     /// # Errors
     ///
@@ -504,7 +504,7 @@ impl KvmVcpu {
             .map_err(KvmVcpuError::VcpuGetMsrs)?;
         // GET_MSRS returns a number of successfully set msrs.
         // If number of set msrs is not equal to the length of
-        // `msrs`, then the value retuned by GET_MSRS can act
+        // `msrs`, then the value returned by GET_MSRS can act
         // as an index to the problematic msr.
         if nmsrs != chunk_size {
             Err(KvmVcpuError::VcpuGetMsr(msrs.as_slice()[nmsrs].index))
@@ -621,7 +621,7 @@ impl KvmVcpu {
         // in the state. If they are different, we need to
         // scale the TSC to the freq found in the state.
         // We accept values within a tolerance of 250 parts
-        // per million beacuse it is common for TSC frequency
+        // per million because it is common for TSC frequency
         // to differ due to calibration at boot time.
         let diff = (i64::from(self.get_tsc_khz()?) - i64::from(state_tsc_freq)).abs();
         // Cannot overflow since u32::MAX * 250 < i64::MAX
@@ -705,7 +705,7 @@ impl Peripherals {
     /// Runs the vCPU in KVM context and handles the kvm exit reason.
     ///
     /// Returns error or enum specifying whether emulation was handled or interrupted.
-    pub fn run_arch_emulation(&self, exit: VcpuExit) -> Result<VcpuEmulation, super::VcpuError> {
+    pub fn run_arch_emulation(&self, exit: VcpuExit) -> Result<VcpuEmulation, VcpuError> {
         match exit {
             VcpuExit::IoIn(addr, data) => {
                 if let Some(pio_bus) = &self.pio_bus {
@@ -728,7 +728,7 @@ impl Peripherals {
                 // TODO: Are we sure we want to finish running a vcpu upon
                 // receiving a vm exit that is not necessarily an error?
                 error!("Unexpected exit reason on vcpu run: {:?}", unexpected_exit);
-                Err(super::VcpuError::UnhandledKvmExit(format!(
+                Err(VcpuError::UnhandledKvmExit(format!(
                     "{:?}",
                     unexpected_exit
                 )))

src/vmm/src/vstate/vm/x86_64.rs renamed to src/vmm/src/arch/x86_64/vstate/vm.rs

@@ -48,7 +48,8 @@ pub enum ArchVmError {
 /// Structure representing the current architecture's understand of what a "virtual machine" is.
 #[derive(Debug)]
 pub struct ArchVm {
-    pub(super) fd: VmFd,
+    /// KVM file descriptor of microVM
+    pub fd: VmFd,
     msrs_to_save: MsrList,
     /// Size in bytes requiring to hold the dynamically-sized `kvm_xsave` struct.
     ///
@@ -93,12 +94,14 @@ impl ArchVm {
         })
     }
 
-    pub(super) fn arch_pre_create_vcpus(&mut self, _: u8) -> Result<(), ArchVmError> {
+    /// Pre-vCPU creation setup.
+    pub fn arch_pre_create_vcpus(&mut self, _: u8) -> Result<(), ArchVmError> {
         // For x86_64 we need to create the interrupt controller before calling `KVM_CREATE_VCPUS`
         self.setup_irqchip()
     }
 
-    pub(super) fn arch_post_create_vcpus(&mut self, _: u8) -> Result<(), ArchVmError> {
+    /// Post-vCPU creation setup.
+    pub fn arch_post_create_vcpus(&mut self, _: u8) -> Result<(), ArchVmError> {
         Ok(())
     }
 

src/vmm/src/builder.rs

@@ -420,10 +420,10 @@ pub enum BuildMicrovmFromSnapshotError {
     TscFrequencyNotPresent,
     #[cfg(target_arch = "x86_64")]
     /// Could not get TSC to check if TSC scaling was required with the snapshot: {0}
-    GetTsc(#[from] crate::vstate::vcpu::GetTscError),
+    GetTsc(#[from] crate::arch::GetTscError),
     #[cfg(target_arch = "x86_64")]
     /// Could not set TSC scaling within the snapshot: {0}
-    SetTsc(#[from] crate::vstate::vcpu::SetTscError),
+    SetTsc(#[from] crate::arch::SetTscError),
     /// Failed to restore microVM state: {0}
     RestoreState(#[from] crate::vstate::vm::ArchVmError),
     /// Failed to update microVM configuration: {0}

src/vmm/src/vstate/vcpu/mod.rs renamed to src/vmm/src/vstate/vcpu.rs

@@ -23,6 +23,7 @@ use vmm_sys_util::errno;
 use vmm_sys_util::eventfd::EventFd;
 
 use crate::FcExitCode;
+pub use crate::arch::{KvmVcpu, KvmVcpuConfigureError, KvmVcpuError, Peripherals, VcpuState};
 use crate::cpu_config::templates::{CpuConfiguration, GuestConfigError};
 #[cfg(feature = "gdb")]
 use crate::gdb::target::{GdbTargetError, get_raw_tid};
@@ -32,18 +33,6 @@ use crate::utils::signal::{Killable, register_signal_handler, sigrtmin};
 use crate::utils::sm::StateMachine;
 use crate::vstate::vm::Vm;
 
-/// Module with aarch64 vCPU implementation.
-#[cfg(target_arch = "aarch64")]
-pub mod aarch64;
-/// Module with x86_64 vCPU implementation.
-#[cfg(target_arch = "x86_64")]
-pub mod x86_64;
-
-#[cfg(target_arch = "aarch64")]
-pub use aarch64::{KvmVcpuError, *};
-#[cfg(target_arch = "x86_64")]
-pub use x86_64::{KvmVcpuError, *};
-
 /// Signal number (SIGRTMIN) used to kick Vcpus.
 pub const VCPU_RTSIG_OFFSET: i32 = 0;