From 0335b8a60052f4574c0bade1c7ddee8767e79f11 Mon Sep 17 00:00:00 2001 From: Wei-chung Hsu Date: Mon, 17 Mar 2025 09:53:26 -0700 Subject: [PATCH 1/3] Added snapshot feature for vhost-device-vsock. There is a hang issue during snapshot disable vring, my solution is to check vring enable/disable status but it requires a new function get_enabled from vring. Signed-off-by: Wei-Chung Hsu --- vhost-device-vsock/src/vhu_vsock.rs | 73 +++++++++++++++++++++- vhost-device-vsock/src/vhu_vsock_thread.rs | 16 ++++- 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/vhost-device-vsock/src/vhu_vsock.rs b/vhost-device-vsock/src/vhu_vsock.rs index d143c850c..aef752307 100644 --- a/vhost-device-vsock/src/vhu_vsock.rs +++ b/vhost-device-vsock/src/vhu_vsock.rs @@ -2,14 +2,18 @@ use std::{ collections::{HashMap, HashSet}, - io::{self, Result as IoResult}, + io::{self, Result as IoResult, BufReader, Read}, path::PathBuf, sync::{Arc, Mutex, RwLock}, + fs::File, + thread::{self, JoinHandle}, }; use log::warn; use thiserror::Error as ThisError; -use vhost::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures}; +use vhost::vhost_user::message::{ + VhostTransferStateDirection, VhostTransferStatePhase, VhostUserProtocolFeatures, VhostUserVirtioFeatures +}; use vhost_user_backend::{VhostUserBackend, VringRwLock}; use virtio_bindings::bindings::{ virtio_config::VIRTIO_F_NOTIFY_ON_EMPTY, virtio_config::VIRTIO_F_VERSION_1, @@ -252,12 +256,27 @@ struct VirtioVsockConfig { // reading its content from byte array. unsafe impl ByteValued for VirtioVsockConfig {} +pub(crate) struct HandlerThread { + pub handle: Option>, + pub created: bool, +} + +impl HandlerThread { + pub fn new(handle: Option>, created: bool) -> Result { + Ok(Self { + handle: handle, + created: created, + }) + } +} + pub(crate) struct VhostUserVsockBackend { config: VirtioVsockConfig, queue_size: usize, pub threads: Vec>, queues_per_thread: Vec, pub exit_event: EventFd, + pub handler_thread: Mutex, } impl VhostUserVsockBackend { @@ -279,6 +298,7 @@ impl VhostUserVsockBackend { threads: vec![thread], queues_per_thread, exit_event: EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?, + handler_thread: Mutex::new(HandlerThread::new(None, false)?), }) } } @@ -303,7 +323,54 @@ impl VhostUserBackend for VhostUserVsockBackend { } fn protocol_features(&self) -> VhostUserProtocolFeatures { - VhostUserProtocolFeatures::MQ | VhostUserProtocolFeatures::CONFIG + VhostUserProtocolFeatures::MQ | VhostUserProtocolFeatures::CONFIG | VhostUserProtocolFeatures::DEVICE_STATE + } + + fn set_device_state_fd( + &self, + direction: VhostTransferStateDirection, + phase: VhostTransferStatePhase, + file: File, + ) -> IoResult> { + self.handler_thread.lock().unwrap().created = true; + let handle = thread::spawn(move || { + match direction { + SAVE => { + // save + // No device state to save yet, just close the FD. + drop(file); + } + LOAD => { + // load + // No device state to load yet, just verify it is empty. + let mut data = Vec::new(); + let mut reader = BufReader::new(file); + if reader.read_to_end(&mut data).is_err() { + println!("vhost-device-vsock loaded device state read failed"); + return; + } + + if data.len() > 0 { + println!("vhost-device-vsock loaded device state is non-empty. BUG!"); + return; + } + } + _ => { + println!("invalid transfer_direction"); + return; + } + } + }); + self.handler_thread.lock().unwrap().handle = Some(handle); + return Ok(None); + } + + fn check_device_state(&self) -> IoResult<()> { + if (self.handler_thread.lock().unwrap().created) { + self.handler_thread.lock().unwrap().created = false; + self.handler_thread.lock().unwrap().handle.take().unwrap().join().unwrap(); + } + return Ok(()); } fn set_event_idx(&self, enabled: bool) { diff --git a/vhost-device-vsock/src/vhu_vsock_thread.rs b/vhost-device-vsock/src/vhu_vsock_thread.rs index 2cd9c4dbc..c951031fb 100644 --- a/vhost-device-vsock/src/vhu_vsock_thread.rs +++ b/vhost-device-vsock/src/vhu_vsock_thread.rs @@ -586,6 +586,10 @@ impl VhostUserVsockThread { let mut vring_mut = vring.get_mut(); + if !vring_mut.get_enabled() { + return Ok(true); + } + let queue = vring_mut.get_queue_mut(); while let Some(mut avail_desc) = queue @@ -657,6 +661,8 @@ impl VhostUserVsockThread { loop { if !self.thread_backend.pending_rx() { break; + } else if !vring.get_enabled() { + break; } vring.disable_notification().unwrap(); @@ -677,6 +683,8 @@ impl VhostUserVsockThread { loop { if !self.thread_backend.pending_raw_pkts() { break; + } else if !vring.get_enabled() { + break; } vring.disable_notification().unwrap(); @@ -722,8 +730,12 @@ impl VhostUserVsockThread { None => return Err(Error::NoMemoryConfigured), }; - while let Some(mut avail_desc) = vring - .get_mut() + let mut vring_mut = vring.get_mut(); + if !vring_mut.get_enabled() { + return Ok(true); + } + + while let Some(mut avail_desc) = vring_mut .get_queue_mut() .iter(atomic_mem.memory()) .map_err(|_| Error::IterateQueue)? From 96fd67982d7ea78cdac8970cb1b8fb7f80ec6cbc Mon Sep 17 00:00:00 2001 From: Wei-chung Hsu Date: Tue, 25 Mar 2025 11:58:22 -0700 Subject: [PATCH 2/3] Added snapshot feature for vhost-device-vsock. Fixed CI errors and reviewer's comments. Signed-off-by: Wei-Chung Hsu --- vhost-device-vsock/src/vhu_vsock.rs | 53 ++++++++++------------ vhost-device-vsock/src/vhu_vsock_thread.rs | 4 +- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/vhost-device-vsock/src/vhu_vsock.rs b/vhost-device-vsock/src/vhu_vsock.rs index aef752307..1c5ed0bb4 100644 --- a/vhost-device-vsock/src/vhu_vsock.rs +++ b/vhost-device-vsock/src/vhu_vsock.rs @@ -2,17 +2,18 @@ use std::{ collections::{HashMap, HashSet}, - io::{self, Result as IoResult, BufReader, Read}, + fs::File, + io::{self, BufReader, Read, Result as IoResult}, path::PathBuf, sync::{Arc, Mutex, RwLock}, - fs::File, thread::{self, JoinHandle}, }; use log::warn; use thiserror::Error as ThisError; use vhost::vhost_user::message::{ - VhostTransferStateDirection, VhostTransferStatePhase, VhostUserProtocolFeatures, VhostUserVirtioFeatures + VhostTransferStateDirection, VhostTransferStatePhase, VhostUserProtocolFeatures, + VhostUserVirtioFeatures }; use vhost_user_backend::{VhostUserBackend, VringRwLock}; use virtio_bindings::bindings::{ @@ -256,27 +257,13 @@ struct VirtioVsockConfig { // reading its content from byte array. unsafe impl ByteValued for VirtioVsockConfig {} -pub(crate) struct HandlerThread { - pub handle: Option>, - pub created: bool, -} - -impl HandlerThread { - pub fn new(handle: Option>, created: bool) -> Result { - Ok(Self { - handle: handle, - created: created, - }) - } -} - pub(crate) struct VhostUserVsockBackend { config: VirtioVsockConfig, queue_size: usize, pub threads: Vec>, queues_per_thread: Vec, pub exit_event: EventFd, - pub handler_thread: Mutex, + pub handler_thread: Mutex>>, } impl VhostUserVsockBackend { @@ -298,7 +285,7 @@ impl VhostUserVsockBackend { threads: vec![thread], queues_per_thread, exit_event: EventFd::new(EFD_NONBLOCK).map_err(Error::EventFdCreate)?, - handler_thread: Mutex::new(HandlerThread::new(None, false)?), + handler_thread: Mutex::new(None), }) } } @@ -323,24 +310,26 @@ impl VhostUserBackend for VhostUserVsockBackend { } fn protocol_features(&self) -> VhostUserProtocolFeatures { - VhostUserProtocolFeatures::MQ | VhostUserProtocolFeatures::CONFIG | VhostUserProtocolFeatures::DEVICE_STATE + VhostUserProtocolFeatures::MQ + | VhostUserProtocolFeatures::CONFIG + | VhostUserProtocolFeatures::DEVICE_STATE } + #[allow(unused_variables)] fn set_device_state_fd( &self, direction: VhostTransferStateDirection, phase: VhostTransferStatePhase, file: File, ) -> IoResult> { - self.handler_thread.lock().unwrap().created = true; let handle = thread::spawn(move || { match direction { - SAVE => { + VhostTransferStateDirection::SAVE => { // save // No device state to save yet, just close the FD. drop(file); } - LOAD => { + VhostTransferStateDirection::LOAD => { // load // No device state to load yet, just verify it is empty. let mut data = Vec::new(); @@ -361,16 +350,22 @@ impl VhostUserBackend for VhostUserVsockBackend { } } }); - self.handler_thread.lock().unwrap().handle = Some(handle); - return Ok(None); + *self.handler_thread.lock().unwrap() = Some(handle); + Ok(None) } fn check_device_state(&self) -> IoResult<()> { - if (self.handler_thread.lock().unwrap().created) { - self.handler_thread.lock().unwrap().created = false; - self.handler_thread.lock().unwrap().handle.take().unwrap().join().unwrap(); + if !self.handler_thread.lock().unwrap().is_none() { + self.handler_thread + .lock() + .unwrap() + .take() + .unwrap() + .join() + .unwrap(); + *self.handler_thread.lock().unwrap() = None; } - return Ok(()); + Ok(()) } fn set_event_idx(&self, enabled: bool) { diff --git a/vhost-device-vsock/src/vhu_vsock_thread.rs b/vhost-device-vsock/src/vhu_vsock_thread.rs index c951031fb..f56b65d49 100644 --- a/vhost-device-vsock/src/vhu_vsock_thread.rs +++ b/vhost-device-vsock/src/vhu_vsock_thread.rs @@ -587,7 +587,7 @@ impl VhostUserVsockThread { let mut vring_mut = vring.get_mut(); if !vring_mut.get_enabled() { - return Ok(true); + return Ok(()); } let queue = vring_mut.get_queue_mut(); @@ -732,7 +732,7 @@ impl VhostUserVsockThread { let mut vring_mut = vring.get_mut(); if !vring_mut.get_enabled() { - return Ok(true); + return Ok(()); } while let Some(mut avail_desc) = vring_mut From 8adcb79b3529e62fe232c2a5af57884e141cf2a6 Mon Sep 17 00:00:00 2001 From: Wei-chung Hsu Date: Tue, 15 Apr 2025 09:23:33 -0700 Subject: [PATCH 3/3] Update the Cargo file to point to my another change in vring functions. This PR needs a new vring function in another PR. Signed-off-by: Wei-chung Hsu --- vhost-device-vsock/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vhost-device-vsock/Cargo.toml b/vhost-device-vsock/Cargo.toml index 19179ca09..7bf4f1dd5 100644 --- a/vhost-device-vsock/Cargo.toml +++ b/vhost-device-vsock/Cargo.toml @@ -21,7 +21,7 @@ env_logger = "0.11" epoll = "4.3.2" log = "0.4" thiserror = "2.0" -vhost = { version = "0.13", features = ["vhost-user-backend"] } +vhost = { git = "https://github.com/WeiChungHsu/vhost.git", branch = "main", features = ["vhost-user-backend"] } vhost-user-backend = "0.17" virtio-bindings = "0.2.5" virtio-queue = "0.14"