Add "krun_set_nested_virt" API

After introducing Nested Virt (EL2) support on macOS, let's add a new
API to enable library users to request its enablement.

Signed-off-by: Sergio Lopez <[email protected]>

Author: slp

include/libkrun.h

@@ -536,6 +536,23 @@ int32_t krun_setuid(uint32_t ctx_id, uid_t uid);
  */
 int32_t krun_setgid(uint32_t ctx_id, gid_t gid);
 
+/**
+ * Configures the microVM to support Nested Virtualization
+ *
+ * Arguments:
+ *  "ctx_id"  - the configuration context ID.
+ *  "enabled" - true to enable Nested Virtualization in the microVM.
+ *
+ * Notes:
+ *  This feature is only supported on macOS.
+ *
+ * Returns:
+ *  Zero on success or a negative error number on failure. Success doesn't imply that
+ *  Nested Virtualization is supported on the system, only that it's going to be requested
+ *  when the microVM is created after calling "krun_start_enter".
+ */
+int32_t krun_set_nested_virt(uint32_t ctx_id, bool enabled);
+
 /**
  * Starts and enters the microVM with the configured parameters. The VMM will attempt to take over
  * stdin/stdout to manage them on behalf of the process running inside the isolated environment,

src/libkrun/src/lib.rs

@@ -1048,6 +1048,23 @@ pub unsafe extern "C" fn krun_set_console_output(ctx_id: u32, c_filepath: *const
     }
 }
 
+#[allow(clippy::missing_safety_doc)]
+#[no_mangle]
+pub unsafe extern "C" fn krun_set_nested_virt(ctx_id: u32, enabled: bool) -> i32 {
+    if enabled && !cfg!(target_os = "macos") {
+        return -libc::EINVAL;
+    }
+
+    match CTX_MAP.lock().unwrap().entry(ctx_id) {
+        Entry::Occupied(mut ctx_cfg) => {
+            let cfg = ctx_cfg.get_mut();
+            cfg.vmr.nested_enabled = enabled;
+            KRUN_SUCCESS
+        }
+        Entry::Vacant(_) => -libc::ENOENT,
+    }
+}
+
 #[allow(clippy::missing_safety_doc)]
 #[no_mangle]
 pub unsafe extern "C" fn krun_set_smbios_oem_strings(

src/vmm/src/builder.rs

@@ -533,7 +533,7 @@ pub fn build_microvm(
 
     #[cfg(not(feature = "tee"))]
     #[allow(unused_mut)]
-    let mut vm = setup_vm(&guest_memory)?;
+    let mut vm = setup_vm(&guest_memory, vm_resources.nested_enabled)?;
 
     #[cfg(feature = "tee")]
     let (kvm, vm) = {
@@ -723,6 +723,7 @@ pub fn build_microvm(
             payload_config.entry_addr,
             &exit_evt,
             vcpu_list.clone(),
+            vm_resources.nested_enabled,
         )
         .map_err(StartMicrovmError::Internal)?;
 
@@ -1260,6 +1261,7 @@ fn load_cmdline(vmm: &Vmm) -> std::result::Result<(), StartMicrovmError> {
 #[cfg(all(target_os = "linux", not(feature = "tee")))]
 pub(crate) fn setup_vm(
     guest_memory: &GuestMemoryMmap,
+    _nested_enabled: bool,
 ) -> std::result::Result<Vm, StartMicrovmError> {
     let kvm = KvmContext::new()
         .map_err(Error::KvmContext)
@@ -1289,8 +1291,9 @@ pub(crate) fn setup_vm(
 #[cfg(target_os = "macos")]
 pub(crate) fn setup_vm(
     guest_memory: &GuestMemoryMmap,
+    nested_enabled: bool,
 ) -> std::result::Result<Vm, StartMicrovmError> {
-    let mut vm = Vm::new(false)
+    let mut vm = Vm::new(nested_enabled)
         .map_err(Error::Vm)
         .map_err(StartMicrovmError::Internal)?;
     vm.memory_init(guest_memory)
@@ -1473,6 +1476,7 @@ fn create_vcpus_aarch64(
     entry_addr: GuestAddress,
     exit_evt: &EventFd,
     vcpu_list: Arc<VcpuList>,
+    nested_enabled: bool,
 ) -> super::Result<Vec<Vcpu>> {
     let mut vcpus = Vec::with_capacity(vcpu_config.vcpu_count as usize);
     let mut boot_senders: HashMap<u64, Sender<u64>> = HashMap::new();
@@ -1491,7 +1495,7 @@ fn create_vcpus_aarch64(
             boot_receiver,
             exit_evt.try_clone().map_err(Error::EventFd)?,
             vcpu_list.clone(),
-            false,
+            nested_enabled,
         )
         .map_err(Error::Vcpu)?;
 
@@ -1916,7 +1920,7 @@ pub mod tests {
 
         let (guest_memory, _arch_memory_info, _shm_manager, _payload_config) =
             default_guest_memory(128).unwrap();
-        let vm = setup_vm(&guest_memory).unwrap();
+        let vm = setup_vm(&guest_memory, false).unwrap();
         let _kvmioapic = KvmIoapic::new(&vm.fd()).unwrap();
 
         // Dummy entry_addr, vcpus will not boot.
@@ -1939,7 +1943,7 @@ pub mod tests {
     fn test_create_vcpus_aarch64() {
         let (guest_memory, _arch_memory_info) =
             create_guest_memory(128, None, Payload::Empty).unwrap();
-        let vm = setup_vm(&guest_memory).unwrap();
+        let vm = setup_vm(&guest_memory, false).unwrap();
         let vcpu_count = 2;
 
         let vcpu_config = VcpuConfig {

src/vmm/src/device_manager/kvm/mmio.rs

@@ -402,7 +402,7 @@ mod tests {
         let guest_mem =
             GuestMemoryMmap::from_ranges(&[(start_addr1, 0x1000), (start_addr2, 0x1000)]).unwrap();
         let vcpu_count: u8 = 1;
-        let vm = builder::setup_vm(&guest_mem).unwrap();
+        let vm = builder::setup_vm(&guest_mem, false).unwrap();
         let mut device_manager =
             MMIODeviceManager::new(&mut 0xd000_0000, (arch::IRQ_BASE, arch::IRQ_MAX));
         let _kvmioapic = KvmIoapic::new(vm.fd()).unwrap();
@@ -422,7 +422,7 @@ mod tests {
         let guest_mem =
             GuestMemoryMmap::from_ranges(&[(start_addr1, 0x1000), (start_addr2, 0x1000)]).unwrap();
         let vcpu_count: u8 = 1;
-        let vm = builder::setup_vm(&guest_mem).unwrap();
+        let vm = builder::setup_vm(&guest_mem, false).unwrap();
         let mut device_manager =
             MMIODeviceManager::new(&mut 0xd000_0000, (arch::IRQ_BASE, arch::IRQ_MAX));
         let _kvmioapic = KvmIoapic::new(vm.fd()).unwrap();
@@ -523,7 +523,7 @@ mod tests {
         let guest_mem =
             GuestMemoryMmap::from_ranges(&[(start_addr1, 0x1000), (start_addr2, 0x1000)]).unwrap();
         let vcpu_count = 1;
-        let vm = builder::setup_vm(&guest_mem).unwrap();
+        let vm = builder::setup_vm(&guest_mem, false).unwrap();
         let mut device_manager =
             MMIODeviceManager::new(&mut 0xd000_0000, (arch::IRQ_BASE, arch::IRQ_MAX));
         let mut cmdline = kernel_cmdline::Cmdline::new(4096);

src/vmm/src/resources.rs

@@ -118,6 +118,8 @@ pub struct VmResources {
     pub console_output: Option<PathBuf>,
     /// SMBIOS OEM Strings
     pub smbios_oem_strings: Option<Vec<String>>,
+    /// Whether to enable nested virtualization.
+    pub nested_enabled: bool,
 }
 
 impl VmResources {
@@ -341,6 +343,7 @@ mod tests {
             enable_snd: False,
             console_output: None,
             smbios_oem_strings: None,
+            nested_enabled: false,
         }
     }