macos/vstate: unify MPIDR generation

With HVF, we generate MPIDR by taking Vcpu.id and setting Aff1 to
that value by doing a left-shift, as this is what the GIC expects
to be found in the distributor.

But, in vstate::Vcpu::configure_aarch64, we were using the Vcpu.id
without left-shifting it.

For consistency, let's generate MPIDR for the Vcpu once and use it
everywhere.

Signed-off-by: Sergio Lopez <[email protected]>

Author: slp

src/hvf/src/lib.rs

@@ -231,7 +231,7 @@ pub struct HvfVcpu<'a> {
 }
 
 impl HvfVcpu<'_> {
-    pub fn new() -> Result<Self, Error> {
+    pub fn new(mpidr: u64) -> Result<Self, Error> {
         let mut vcpuid: hv_vcpu_t = 0;
         let vcpu_exit_ptr: *mut hv_vcpu_exit_t = std::ptr::null_mut();
 
@@ -257,8 +257,7 @@ impl HvfVcpu<'_> {
 
         // We write vcpuid to Aff1 as otherwise it won't match the redistributor ID
         // when using HVF in-kernel GICv3.
-        let ret =
-            unsafe { hv_vcpu_set_sys_reg(vcpuid, hv_sys_reg_t_HV_SYS_REG_MPIDR_EL1, vcpuid << 8) };
+        let ret = unsafe { hv_vcpu_set_sys_reg(vcpuid, hv_sys_reg_t_HV_SYS_REG_MPIDR_EL1, mpidr) };
         if ret != HV_SUCCESS {
             return Err(Error::VcpuCreate);
         }

src/vmm/src/builder.rs

@@ -6,6 +6,7 @@
 #[cfg(target_os = "macos")]
 use crossbeam_channel::{unbounded, Sender};
 use kernel::cmdline::Cmdline;
+use std::collections::HashMap;
 use std::fmt::{Display, Formatter};
 use std::fs::File;
 use std::io::{self, Read};
@@ -1460,15 +1461,14 @@ fn create_vcpus_aarch64(
     vcpu_list: Arc<VcpuList>,
 ) -> super::Result<Vec<Vcpu>> {
     let mut vcpus = Vec::with_capacity(vcpu_config.vcpu_count as usize);
-    let mut boot_senders = Vec::with_capacity(vcpu_config.vcpu_count as usize - 1);
+    let mut boot_senders: HashMap<u64, Sender<u64>> = HashMap::new();
 
     for cpu_index in 0..vcpu_config.vcpu_count {
-        let boot_receiver = if cpu_index != 0 {
+        let (boot_sender, boot_receiver) = if cpu_index != 0 {
             let (boot_sender, boot_receiver) = unbounded();
-            boot_senders.push(boot_sender);
-            Some(boot_receiver)
+            (Some(boot_sender), Some(boot_receiver))
         } else {
-            None
+            (None, None)
         };
 
         let mut vcpu = Vcpu::new_aarch64(
@@ -1482,6 +1482,10 @@ fn create_vcpus_aarch64(
 
         vcpu.configure_aarch64(guest_mem).map_err(Error::Vcpu)?;
 
+        if let Some(boot_sender) = boot_sender {
+            boot_senders.insert(vcpu.get_mpidr(), boot_sender);
+        }
+
         vcpus.push(vcpu);
     }
 

src/vmm/src/macos/vstate.rs

@@ -6,6 +6,7 @@
 // found in the THIRD-PARTY file.
 
 use std::cell::Cell;
+use std::collections::HashMap;
 use std::fmt::{Display, Formatter};
 use std::io;
 use std::result;
@@ -176,7 +177,7 @@ pub struct Vcpu {
     id: u8,
     boot_entry_addr: u64,
     boot_receiver: Option<Receiver<u64>>,
-    boot_senders: Option<Vec<Sender<u64>>>,
+    boot_senders: Option<HashMap<u64, Sender<u64>>>,
     fdt_addr: u64,
     mmio_bus: Option<devices::Bus>,
     #[cfg_attr(all(test, target_arch = "aarch64"), allow(unused))]
@@ -280,7 +281,7 @@ impl Vcpu {
             fdt_addr: 0,
             mmio_bus: None,
             exit_evt,
-            mpidr: 0,
+            mpidr: id as u64,
             event_receiver,
             event_sender: Some(event_sender),
             response_receiver: Some(response_receiver),
@@ -304,19 +305,16 @@ impl Vcpu {
         self.mmio_bus = Some(mmio_bus);
     }
 
-    pub fn set_boot_senders(&mut self, boot_senders: Vec<Sender<u64>>) {
+    pub fn set_boot_senders(&mut self, boot_senders: HashMap<u64, Sender<u64>>) {
         self.boot_senders = Some(boot_senders);
     }
 
     /// Configures an aarch64 specific vcpu.
     ///
     /// # Arguments
     ///
-    /// * `vm_fd` - The kvm `VmFd` for this microvm.
     /// * `guest_mem` - The guest memory used by this microvm.
-    /// * `kernel_load_addr` - Offset from `guest_mem` at which the kernel is loaded.
     pub fn configure_aarch64(&mut self, guest_mem: &GuestMemoryMmap) -> Result<()> {
-        self.mpidr = self.id as u64;
         self.fdt_addr = arch::aarch64::get_fdt_addr(guest_mem);
 
         Ok(())
@@ -369,13 +367,12 @@ impl Vcpu {
                         "CpuOn: mpidr=0x{:x} entry=0x{:x} context_id={}",
                         mpidr, entry, context_id
                     );
-                    // assuming a flat CPU hierarchy, only the bottom bits of mpidr should be used,
-                    // and cpuid == mpidr
-                    let cpuid: usize = mpidr as usize;
                     if let Some(boot_senders) = &self.boot_senders {
-                        if let Some(sender) = boot_senders.get(cpuid - 1) {
+                        if let Some(sender) = boot_senders.get(&mpidr) {
                             sender.send(entry).unwrap()
                         }
+                    } else {
+                        error!("CpuOn request coming from an unexpected vCPU={}", self.id);
                     }
                     Ok(VcpuEmulation::Handled)
                 }
@@ -432,7 +429,7 @@ impl Vcpu {
 
     /// Main loop of the vCPU thread.
     pub fn run(&mut self, init_tls_sender: Sender<bool>) {
-        let mut hvf_vcpu = HvfVcpu::new().expect("Can't create HVF vCPU");
+        let mut hvf_vcpu = HvfVcpu::new(self.mpidr).expect("Can't create HVF vCPU");
         let hvf_vcpuid = hvf_vcpu.id();
 
         init_tls_sender