WIP: Add PVTime device creation, boot-time setup, and logging

- Added PVTime struct that allocates and stores per-vCPU stolen_time regions
- Implemented register_vcpu() to register the IPA for each vCPU with KVM
- Invoked setup_pv_time in build_microvm_for_boot
- Added PVTime detection in setup_pv_time
- Started using debug/info logs while booting microvm for testing

Limitations/WIP:
- While manually booting a microvm for testing, error during set_device_attr in register_vcpu
- Snapshotting not yet supported

Signed-off-by: Dakshin Devanand <[email protected]>

Author: DakshinD

src/vmm/src/arch/aarch64/mod.rs

@@ -9,6 +9,8 @@ pub mod gic;
 pub mod kvm;
 /// Layout for this aarch64 system.
 pub mod layout;
+/// Module for the paravirtualized time device
+pub mod pvtime;
 /// Logic for configuring aarch64 registers.
 pub mod regs;
 /// Architecture specific vCPU code

src/vmm/src/arch/aarch64/pvtime/mod.rs

@@ -0,0 +1,117 @@
+// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+use std::collections::HashMap;
+
+use displaydoc::Display;
+use kvm_bindings::{KVM_ARM_VCPU_PVTIME_CTRL, KVM_ARM_VCPU_PVTIME_IPA};
+use log::{debug, info, warn};
+use thiserror::Error;
+
+use crate::device_manager::resources::ResourceAllocator;
+
+/// Size of the stolen_time struct in bytes, see 3.2.2 in DEN0057A
+pub const STEALTIME_STRUCT_MEM_SIZE: u64 = 16;
+
+/// Represent PVTime device for ARM
+#[derive(Debug)]
+pub struct PVTime {
+    /// Maps vCPU index to IPA location of stolen_time struct as defined in DEN0057A
+    steal_time_regions: HashMap<u8, u64>,
+}
+
+/// Errors associated with PVTime operations
+#[derive(Debug, Error, Display, PartialEq, Eq)]
+pub enum PVTimeError {
+    /// Failed to allocate memory region: {0}
+    AllocationFailed(String),
+    /// Invalid VCPU ID: {0}
+    InvalidVcpuIndex(u8),
+    /// Error while setting or getting device attributes for vCPU: {0}, {1}, {2}
+    DeviceAttribute(kvm_ioctls::Error, bool, u32),
+}
+
+impl PVTime {
+    /// Creates a new PVTime device by allocating system memory for all vCPUs
+    pub fn new(
+        resource_allocator: &mut ResourceAllocator,
+        vcpu_count: u8,
+    ) -> Result<Self, PVTimeError> {
+        info!("Creating PVTime for {vcpu_count} vCPUs...");
+        // This returns the IPA(?) of the start of our shared memory region for all vCPUs.
+        // Q: Confirm that allocate_system_memory returns an IPA?
+        let base_addr = resource_allocator
+            .allocate_system_memory(
+                STEALTIME_STRUCT_MEM_SIZE * vcpu_count as u64,
+                16, // Q: We believe this to be 16, need confirmation.
+                vm_allocator::AllocPolicy::LastMatch,
+            )
+            .map_err(|e| PVTimeError::AllocationFailed(e.to_string()))?;
+
+        debug!(
+            "Allocated base address for PVTime stolen_time region: 0x{:x}",
+            base_addr
+        );
+
+        // Now we need to store the base IPA for each vCPU's steal_time struct.
+        let mut steal_time_regions = HashMap::new();
+        for i in 0..vcpu_count {
+            let ipa = base_addr + (i as u64 * STEALTIME_STRUCT_MEM_SIZE);
+            debug!("Assigned vCPU {} to stolen_time IPA 0x{:x}", i, ipa);
+            steal_time_regions.insert(i, ipa);
+        }
+
+        // Return the PVTime device with the steal_time region IPAs mapped to vCPU indices.
+        Ok(PVTime { steal_time_regions })
+    }
+
+    /// Register a vCPU with its pre-allocated steal time region
+    pub fn register_vcpu(
+        &self,
+        vcpu_index: u8,
+        vcpu_fd: &kvm_ioctls::VcpuFd,
+    ) -> Result<(), PVTimeError> {
+        // Get IPA of the steal_time region for this vCPU
+        let ipa = self
+            .steal_time_regions
+            .get(&vcpu_index)
+            .ok_or(PVTimeError::InvalidVcpuIndex(vcpu_index))?;
+
+        debug!(
+            "Registering vCPU {} with stolen_time IPA = 0x{:x}",
+            vcpu_index, ipa
+        );
+
+        // IMPORTANT QUESTION: We need to confirm this is safe. We need to somehow
+        // ensure the ipa value is not dropped before we use it etc. since we
+        // are creating a raw pointer? Do we need a Box?
+        let ipa_val = *ipa;
+        let ipa_ptr = &ipa_val as *const u64;
+        debug!(
+            "Registering vCPU {} with stolen_time IPA = 0x{:x} (at userspace addr 0x{:x})",
+            vcpu_index, ipa_val, ipa_ptr as u64
+        );
+
+        // Use KVM syscall (kvm_set_device_attr) to register the vCPU with the steal_time region
+        let vcpu_device_attr = kvm_bindings::kvm_device_attr {
+            group: KVM_ARM_VCPU_PVTIME_CTRL,
+            attr: KVM_ARM_VCPU_PVTIME_IPA as u64,
+            addr: ipa_ptr as u64, // userspace address of attr data
+            flags: 0,
+        };
+
+        vcpu_fd.set_device_attr(&vcpu_device_attr).map_err(|err| {
+            warn!(
+                "Failed to set device attribute for vCPU {}: {:?}",
+                vcpu_index, err
+            );
+            PVTimeError::DeviceAttribute(err, true, KVM_ARM_VCPU_PVTIME_CTRL)
+        })?;
+
+        Ok(())
+    }
+}
+
+// TODO/Q: Would we be correct in implementing Persist for PVTime? Some sort of persistence is
+// needed for snapshot capability. We would only need to store base_addr IPA of shared memory region
+// assuming the # of vCPUs is constant across snapshots. Also assuming we are correct that
+// kvm_set_device_attr takes an IPA and we get an IPA back from resource_allocator.

src/vmm/src/builder.rs

@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: Apache-2.0
 
 //! Enables pre-boot setup, instantiation and booting of a Firecracker VMM.
-
 use std::fmt::Debug;
 use std::io;
 #[cfg(feature = "gdb")]
@@ -12,13 +11,15 @@ use std::sync::{Arc, Mutex};
 use event_manager::{MutEventSubscriber, SubscriberOps};
 use libc::EFD_NONBLOCK;
 use linux_loader::cmdline::Cmdline as LoaderKernelCmdline;
+use log::{info, warn};
 use userfaultfd::Uffd;
 use utils::time::TimestampUs;
 #[cfg(target_arch = "aarch64")]
 use vm_superio::Rtc;
 use vm_superio::Serial;
 use vmm_sys_util::eventfd::EventFd;
 
+use crate::arch::aarch64::pvtime::PVTimeError;
 use crate::arch::{ConfigurationError, configure_system_for_boot, load_kernel};
 #[cfg(target_arch = "aarch64")]
 use crate::construct_kvm_mpidrs;
@@ -69,6 +70,9 @@ pub enum StartMicrovmError {
     AttachBlockDevice(io::Error),
     /// Unable to attach the VMGenID device: {0}
     AttachVmgenidDevice(kvm_ioctls::Error),
+    /// PVTime not supported: {0}
+    #[cfg(target_arch = "aarch64")]
+    PVTimeNotSupported(kvm_ioctls::Error),
     /// System configuration error: {0}
     ConfigureSystem(#[from] ConfigurationError),
     /// Failed to create guest config: {0}
@@ -82,6 +86,8 @@ pub enum StartMicrovmError {
     CreateLegacyDevice(device_manager::legacy::LegacyDeviceError),
     /// Error creating VMGenID device: {0}
     CreateVMGenID(VmGenIdError),
+    /// Error creating PVTime device: {0}
+    CreatePVTime(PVTimeError),
     /// Invalid Memory Configuration: {0}
     GuestMemory(crate::vstate::memory::MemoryError),
     /// Error with initrd initialization: {0}.
@@ -289,6 +295,17 @@ pub fn build_microvm_for_boot(
 
     attach_vmgenid_device(&mut vmm)?;
 
+    // Attempt to setup PVTime, continue if not supported
+    #[cfg(target_arch = "aarch64")]
+    if let Err(e) = setup_pv_time(&mut vmm, vcpus.as_mut()) {
+        match e {
+            StartMicrovmError::PVTimeNotSupported(e) => {
+                warn!("PVTime not supported: {}", e);
+            }
+            other => return Err(other),
+        }
+    }
+
     configure_system_for_boot(
         &mut vmm,
         vcpus.as_mut(),
@@ -548,6 +565,49 @@ pub fn setup_serial_device(
     Ok(serial)
 }
 
+/// Sets up the pvtime device.
+#[cfg(target_arch = "aarch64")]
+fn setup_pv_time(vmm: &mut Vmm, vcpus: &mut Vec<Vcpu>) -> Result<(), StartMicrovmError> {
+    // Q: Is this bad practice, using crates in this scope?
+    use crate::arch::aarch64::pvtime::PVTime;
+
+    info!("PVTime setup started");
+
+    // Check if pvtime is enabled
+    let pvtime_device_attr = kvm_bindings::kvm_device_attr {
+        group: kvm_bindings::KVM_ARM_VCPU_PVTIME_CTRL,
+        attr: kvm_bindings::KVM_ARM_VCPU_PVTIME_IPA as u64,
+        addr: 0,
+        flags: 0,
+    };
+
+    // Use kvm_has_device_attr to check if PVTime is supported
+    // TODO: Flesh out feature detection. Either throw error and handle, or
+    // even return silently and just log "no support", or something else.
+    let vcpu_fd = &vcpus[0].kvm_vcpu.fd;
+    vcpu_fd
+        .has_device_attr(&pvtime_device_attr)
+        .map_err(StartMicrovmError::PVTimeNotSupported)?;
+
+    info!("PVTime is supported");
+
+    // Create the pvtime device
+    let pv_time = PVTime::new(&mut vmm.resource_allocator, vcpus.len() as u8)
+        .map_err(StartMicrovmError::CreatePVTime)?;
+
+    // Register the vcpu with the pvtime device to map its steal time region
+    for i in 0..vcpus.len() {
+        pv_time
+            .register_vcpu(i as u8, &vcpus[i].kvm_vcpu.fd)
+            .map_err(StartMicrovmError::CreatePVTime)?; // Change this to its own error
+    }
+
+    // TODO: Store pv_time somewhere (in Vmm ?) for snapshotting later instead of dropping it
+    info!("PVTime setup completed");
+
+    Ok(())
+}
+
 #[cfg(target_arch = "aarch64")]
 fn attach_legacy_devices_aarch64(
     event_manager: &mut EventManager,