refactor: remove TLS from vcpus

Instead of storing mmaped kvm_vcpu struct in the TLS,
we can have it in the VcpuHandle. This way we don't need
to deal with TLS and it's synchronization.

Signed-off-by: Egor Lazarchuk <[email protected]>

Author: ShadowCurse

src/vmm/src/builder.rs

@@ -281,7 +281,7 @@ pub fn build_microvm_for_boot(
         boot_cmdline,
     )?;
 
-    let vmm = Vmm {
+    let mut vmm = Vmm {
         instance_info: instance_info.clone(),
         shutdown_exit_code: None,
         kvm,
@@ -292,6 +292,16 @@ pub fn build_microvm_for_boot(
         device_manager,
     };
 
+    // Move vcpus to their own threads and start their state machine in the 'Paused' state.
+    vmm.start_vcpus(
+        vcpus,
+        seccomp_filters
+            .get("vcpu")
+            .ok_or_else(|| StartMicrovmError::MissingSeccompFilters("vcpu".to_string()))?
+            .clone(),
+    )
+    .map_err(VmmError::VcpuStart)?;
+
     let vmm = Arc::new(Mutex::new(vmm));
 
     #[cfg(feature = "gdb")]
@@ -308,18 +318,6 @@ pub fn build_microvm_for_boot(
         debug!("No GDB socket provided not starting gdb server.");
     }
 
-    // Move vcpus to their own threads and start their state machine in the 'Paused' state.
-    vmm.lock()
-        .unwrap()
-        .start_vcpus(
-            vcpus,
-            seccomp_filters
-                .get("vcpu")
-                .ok_or_else(|| StartMicrovmError::MissingSeccompFilters("vcpu".to_string()))?
-                .clone(),
-        )
-        .map_err(VmmError::VcpuStart)?;
-
     // Load seccomp filters for the VMM thread.
     // Execution panics if filters cannot be loaded, use --no-seccomp if skipping filters
     // altogether is the desired behaviour.

src/vmm/src/lib.rs

@@ -353,8 +353,11 @@ impl Vmm {
             #[cfg(target_arch = "x86_64")]
             vcpu.kvm_vcpu.set_pio_bus(self.vm.pio_bus.clone());
 
-            self.vcpus_handles
-                .push(vcpu.start_threaded(vcpu_seccomp_filter.clone(), barrier.clone())?);
+            self.vcpus_handles.push(vcpu.start_threaded(
+                &self.vm,
+                vcpu_seccomp_filter.clone(),
+                barrier.clone(),
+            )?);
         }
         self.instance_info.state = VmState::Paused;
         // Wait for vCPUs to initialize their TLS before moving forward.
@@ -369,7 +372,7 @@ impl Vmm {
 
         // Send the events.
         self.vcpus_handles
-            .iter()
+            .iter_mut()
             .try_for_each(|handle| handle.send_event(VcpuEvent::Resume))
             .map_err(|_| VmmError::VcpuMessage)?;
 
@@ -391,7 +394,7 @@ impl Vmm {
     pub fn pause_vm(&mut self) -> Result<(), VmmError> {
         // Send the events.
         self.vcpus_handles
-            .iter()
+            .iter_mut()
             .try_for_each(|handle| handle.send_event(VcpuEvent::Pause))
             .map_err(|_| VmmError::VcpuMessage)?;
 
@@ -450,7 +453,7 @@ impl Vmm {
     }
 
     fn save_vcpu_states(&mut self) -> Result<Vec<VcpuState>, MicrovmStateError> {
-        for handle in self.vcpus_handles.iter() {
+        for handle in self.vcpus_handles.iter_mut() {
             handle
                 .send_event(VcpuEvent::SaveState)
                 .map_err(MicrovmStateError::SignalVcpu)?;
@@ -479,7 +482,7 @@ impl Vmm {
 
     /// Dumps CPU configuration.
     pub fn dump_cpu_config(&mut self) -> Result<Vec<CpuConfiguration>, DumpCpuConfigError> {
-        for handle in self.vcpus_handles.iter() {
+        for handle in self.vcpus_handles.iter_mut() {
             handle
                 .send_event(VcpuEvent::DumpCpuConfig)
                 .map_err(DumpCpuConfigError::SendEvent)?;
@@ -615,7 +618,7 @@ impl Vmm {
         // We send a "Finish" event.  If a VCPU has already exited, this is the only
         // message it will accept... but running and paused will take it as well.
         // It breaks out of the state machine loop so that the thread can be joined.
-        for (idx, handle) in self.vcpus_handles.iter().enumerate() {
+        for (idx, handle) in self.vcpus_handles.iter_mut().enumerate() {
             if let Err(err) = handle.send_event(VcpuEvent::Finish) {
                 error!("Failed to send VcpuEvent::Finish to vCPU {}: {}", idx, err);
             }