From a8ac7f72c0ac0dd5b378ff12f3229bad3997f45c Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 6 Jul 2024 17:28:26 +0900 Subject: [PATCH 1/7] Add dynamic UNIX socket forwarding Unconditionally forward a range of ports to sockets named `$XDG_RUNTIME_DIR/krun/socket/port-$PORT`. This allows applications to dynamically bind to sockets at those paths and have them forwarded to the guest as vsock ports, after the krun VM is already running. The port range is currently hardcoded as 50000..50200. Signed-off-by: Asahi Lina Signed-off-by: Sasha Finkelstein --- crates/krun/src/bin/krun.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/krun/src/bin/krun.rs b/crates/krun/src/bin/krun.rs index fb3f1e5..1b5c0c3 100644 --- a/crates/krun/src/bin/krun.rs +++ b/crates/krun/src/bin/krun.rs @@ -179,6 +179,7 @@ fn main() -> Result<()> { return Err(err).context("Failed to configure vsock for pulse socket"); } } + let hidpipe_path = Path::new(&run_path).join("hidpipe"); if hidpipe_path.exists() { let hidpipe_path = CString::new( @@ -194,6 +195,25 @@ fn main() -> Result<()> { return Err(err).context("Failed to configure vsock for hidpipe socket"); } } + + let socket_dir = Path::new(&run_path).join("krun/socket"); + std::fs::create_dir_all(&socket_dir)?; + // Dynamic ports: Applications may listen on these sockets as neeeded. + for port in 50000..50200 { + let socket_path = socket_dir.join(format!("port-{}", port)); + let socket_path = CString::new( + socket_path + .to_str() + .expect("socket_path should not contain invalid UTF-8"), + ) + .context("Failed to process dynamic socket path as it contains NUL character")?; + // SAFETY: `socket_path` is a pointer to a `CString` with long enough lifetime. + let err = unsafe { krun_add_vsock_port(ctx_id, port, socket_path.as_ptr()) }; + if err < 0 { + let err = Errno::from_raw_os_error(-err); + return Err(err).context("Failed to configure vsock for dynamic socket"); + } + } } // Forward the native X11 display into the guest as a socket From 4375ec3609b55dfe35f887d6213d1b1ba0f5b097 Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Sat, 6 Jul 2024 19:11:24 +0200 Subject: [PATCH 2/7] Add interactive launch mode. After launching a process via the server, krun immediately closes and the launched process' std{in,out,err} go nowhere. Add a --interactive flag that allocates a pseudo-terminal, routes it to the host and connects to it. Somewhat like '-it' in docker/podman exec. Signed-off-by: Sasha Finkelstein --- crates/krun/src/bin/krun.rs | 5 ++- crates/krun/src/cli_options.rs | 6 +++ crates/krun/src/launch.rs | 69 +++++++++++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 4 deletions(-) diff --git a/crates/krun/src/bin/krun.rs b/crates/krun/src/bin/krun.rs index 1b5c0c3..c109b24 100644 --- a/crates/krun/src/bin/krun.rs +++ b/crates/krun/src/bin/krun.rs @@ -7,7 +7,7 @@ use anyhow::{anyhow, Context, Result}; use krun::cli_options::options; use krun::cpu::{get_fallback_cores, get_performance_cores}; use krun::env::{find_krun_exec, prepare_env_vars}; -use krun::launch::{launch_or_lock, LaunchResult}; +use krun::launch::{launch_or_lock, LaunchResult, DYNAMIC_PORT_RANGE}; use krun::net::{connect_to_passt, start_passt}; use krun::types::MiB; use krun_sys::{ @@ -39,6 +39,7 @@ fn main() -> Result<()> { options.command, options.command_args, options.env, + options.interactive, )? { LaunchResult::LaunchRequested => { // There was a krun instance already running and we've requested it @@ -199,7 +200,7 @@ fn main() -> Result<()> { let socket_dir = Path::new(&run_path).join("krun/socket"); std::fs::create_dir_all(&socket_dir)?; // Dynamic ports: Applications may listen on these sockets as neeeded. - for port in 50000..50200 { + for port in DYNAMIC_PORT_RANGE { let socket_path = socket_dir.join(format!("port-{}", port)); let socket_path = CString::new( socket_path diff --git a/crates/krun/src/cli_options.rs b/crates/krun/src/cli_options.rs index bd237da..ae96be7 100644 --- a/crates/krun/src/cli_options.rs +++ b/crates/krun/src/cli_options.rs @@ -13,6 +13,7 @@ pub struct Options { pub mem: Option, pub passt_socket: Option, pub server_port: u32, + pub interactive: bool, pub command: PathBuf, pub command_args: Vec, } @@ -80,6 +81,10 @@ pub fn options() -> OptionParser { .argument("SERVER_PORT") .fallback(3334) .display_fallback(); + let interactive = long("interactive") + .short('i') + .help("Allocate a tty guest-side and connect it to the current stdin/out") + .switch(); let command = positional("COMMAND").help("the command you want to execute in the vm"); let command_args = any::("COMMAND_ARGS", |arg| { (!["--help", "-h"].contains(&&*arg)).then_some(arg) @@ -93,6 +98,7 @@ pub fn options() -> OptionParser { mem, passt_socket, server_port, + interactive, // positionals command, command_args, diff --git a/crates/krun/src/launch.rs b/crates/krun/src/launch.rs index 65ed567..f4a9f6d 100644 --- a/crates/krun/src/launch.rs +++ b/crates/krun/src/launch.rs @@ -10,10 +10,15 @@ use std::path::{Path, PathBuf}; use anyhow::{anyhow, Context, Result}; use rustix::fs::{flock, FlockOperation}; use rustix::path::Arg; +use std::ops::Range; +use std::process::{Child, Command}; +use utils::env::find_in_path; use utils::launch::Launch; use crate::env::prepare_env_vars; +pub const DYNAMIC_PORT_RANGE: Range = 50000..50200; + pub enum LaunchResult { LaunchRequested, LockAcquired { @@ -49,17 +54,77 @@ impl Display for LaunchError { } } +fn start_socat() -> Result<(Child, u32)> { + let run_path = env::var("XDG_RUNTIME_DIR") + .map_err(|e| anyhow!("unable to get XDG_RUNTIME_DIR: {:?}", e))?; + let socket_dir = Path::new(&run_path).join("krun/socket"); + let socat_path = + find_in_path("socat")?.ok_or_else(|| anyhow!("Unable to find socat in PATH"))?; + for port in DYNAMIC_PORT_RANGE { + let path = socket_dir.join(&format!("port-{}", port)); + if path.exists() { + continue; + } + let child = Command::new(&socat_path) + .arg(format!("unix-l:{}", path.as_os_str().to_string_lossy())) + .arg("-,raw,echo=0") + .spawn()?; + return Ok((child, port)); + } + Err(anyhow!("Ran out of ports.")) +} + +fn escape_for_socat(s: String) -> String { + let mut ret = String::with_capacity(s.len()); + for c in s.chars() { + match c { + ':' | ',' | '!' | '"' | '\'' | '\\' | '(' | '[' | '{' => { + ret.push('\\'); + }, + _ => {}, + } + ret.push(c); + } + ret +} + +fn wrapped_launch( + server_port: u32, + mut command: PathBuf, + mut command_args: Vec, + env: HashMap, + interactive: bool, +) -> Result<()> { + if !interactive { + return request_launch(server_port, command, command_args, env); + } + let (mut socat, vsock_port) = start_socat()?; + command_args.insert(0, command.to_string_lossy().into_owned()); + command_args = vec![ + format!("vsock:2:{}", vsock_port), + format!( + "exec:{},pty,setsid,stderr", + escape_for_socat(command_args.join(" ")) + ), + ]; + command = "socat".into(); + request_launch(server_port, command, command_args, env)?; + socat.wait()?; + Ok(()) +} + pub fn launch_or_lock( server_port: u32, command: PathBuf, command_args: Vec, env: Vec<(String, Option)>, + interactive: bool, ) -> Result { let running_server_port = env::var("KRUN_SERVER_PORT").ok(); if let Some(port) = running_server_port { let port: u32 = port.parse()?; let env = prepare_env_vars(env)?; - if let Err(err) = request_launch(port, command, command_args, env) { + if let Err(err) = wrapped_launch(port, command, command_args, env, interactive) { return Err(anyhow!("could not request launch to server: {err}")); } return Ok(LaunchResult::LaunchRequested); @@ -78,7 +143,7 @@ pub fn launch_or_lock( let env = prepare_env_vars(env)?; let mut tries = 0; loop { - match request_launch(port, command.clone(), command_args.clone(), env.clone()) { + match wrapped_launch(port, command.clone(), command_args.clone(), env.clone(), interactive) { Err(err) => match err.downcast_ref::() { Some(&LaunchError::Connection(_)) => { if tries == 3 { From 9f25b1289b4c14c63c3b750618f9a84b6e36922d Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Sun, 1 Sep 2024 15:28:03 +0200 Subject: [PATCH 3/7] Pass through the current working directory. Aside from just being better UX, some commands are sensitive to it and will not work correctly without it. Signed-off-by: Sasha Finkelstein --- crates/krun-server/src/server.rs | 4 +++- crates/krun/src/launch.rs | 19 +++++++++++++++---- crates/utils/src/launch.rs | 1 + 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/crates/krun-server/src/server.rs b/crates/krun-server/src/server.rs index 5299f99..3506156 100644 --- a/crates/krun-server/src/server.rs +++ b/crates/krun-server/src/server.rs @@ -164,8 +164,9 @@ async fn handle_connection(mut stream: BufStream) -> Result<(PathBuf, command, command_args, env, + cwd, } = read_request(&mut stream).await?; - debug!(command:?, command_args:?, env:?; "received launch request"); + debug!(command:?, command_args:?, env:?, cwd:?; "received launch request"); envs.extend(env); let (stdout, stderr) = make_stdout_stderr(&command, &envs)?; @@ -176,6 +177,7 @@ async fn handle_connection(mut stream: BufStream) -> Result<(PathBuf, .stdin(Stdio::null()) .stdout(stdout) .stderr(stderr) + .current_dir(cwd) .spawn() .with_context(|| format!("Failed to execute {command:?} as child process")); if let Err(err) = &res { diff --git a/crates/krun/src/launch.rs b/crates/krun/src/launch.rs index f4a9f6d..15846dd 100644 --- a/crates/krun/src/launch.rs +++ b/crates/krun/src/launch.rs @@ -93,10 +93,11 @@ fn wrapped_launch( mut command: PathBuf, mut command_args: Vec, env: HashMap, + cwd: PathBuf, interactive: bool, ) -> Result<()> { if !interactive { - return request_launch(server_port, command, command_args, env); + return request_launch(server_port, command, command_args, env, cwd); } let (mut socat, vsock_port) = start_socat()?; command_args.insert(0, command.to_string_lossy().into_owned()); @@ -108,7 +109,7 @@ fn wrapped_launch( ), ]; command = "socat".into(); - request_launch(server_port, command, command_args, env)?; + request_launch(server_port, command, command_args, env, cwd)?; socat.wait()?; Ok(()) } @@ -121,10 +122,11 @@ pub fn launch_or_lock( interactive: bool, ) -> Result { let running_server_port = env::var("KRUN_SERVER_PORT").ok(); + let cwd = env::current_dir()?; if let Some(port) = running_server_port { let port: u32 = port.parse()?; let env = prepare_env_vars(env)?; - if let Err(err) = wrapped_launch(port, command, command_args, env, interactive) { + if let Err(err) = wrapped_launch(port, command, command_args, env, cwd, interactive) { return Err(anyhow!("could not request launch to server: {err}")); } return Ok(LaunchResult::LaunchRequested); @@ -143,7 +145,14 @@ pub fn launch_or_lock( let env = prepare_env_vars(env)?; let mut tries = 0; loop { - match wrapped_launch(port, command.clone(), command_args.clone(), env.clone(), interactive) { + match wrapped_launch( + port, + command.clone(), + command_args.clone(), + env.clone(), + cwd.clone(), + interactive, + ) { Err(err) => match err.downcast_ref::() { Some(&LaunchError::Connection(_)) => { if tries == 3 { @@ -215,6 +224,7 @@ fn request_launch( command: PathBuf, command_args: Vec, env: HashMap, + cwd: PathBuf, ) -> Result<()> { let mut stream = TcpStream::connect(format!("127.0.0.1:{server_port}")).map_err(LaunchError::Connection)?; @@ -223,6 +233,7 @@ fn request_launch( command, command_args, env, + cwd, }; stream diff --git a/crates/utils/src/launch.rs b/crates/utils/src/launch.rs index 78a3dcf..4754c6e 100644 --- a/crates/utils/src/launch.rs +++ b/crates/utils/src/launch.rs @@ -8,4 +8,5 @@ pub struct Launch { pub command: PathBuf, pub command_args: Vec, pub env: HashMap, + pub cwd: PathBuf, } From f8748a906f57d039722283a98d8c37b7f170493e Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Sun, 1 Sep 2024 15:53:12 +0200 Subject: [PATCH 4/7] Always use interactive mode. This is always what you want if running via binfmt Signed-off-by: Sasha Finkelstein --- crates/krun/src/bin/krun.rs | 1 - crates/krun/src/cli_options.rs | 6 ------ crates/krun/src/launch.rs | 8 +------- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/crates/krun/src/bin/krun.rs b/crates/krun/src/bin/krun.rs index c109b24..6e1ac23 100644 --- a/crates/krun/src/bin/krun.rs +++ b/crates/krun/src/bin/krun.rs @@ -39,7 +39,6 @@ fn main() -> Result<()> { options.command, options.command_args, options.env, - options.interactive, )? { LaunchResult::LaunchRequested => { // There was a krun instance already running and we've requested it diff --git a/crates/krun/src/cli_options.rs b/crates/krun/src/cli_options.rs index ae96be7..bd237da 100644 --- a/crates/krun/src/cli_options.rs +++ b/crates/krun/src/cli_options.rs @@ -13,7 +13,6 @@ pub struct Options { pub mem: Option, pub passt_socket: Option, pub server_port: u32, - pub interactive: bool, pub command: PathBuf, pub command_args: Vec, } @@ -81,10 +80,6 @@ pub fn options() -> OptionParser { .argument("SERVER_PORT") .fallback(3334) .display_fallback(); - let interactive = long("interactive") - .short('i') - .help("Allocate a tty guest-side and connect it to the current stdin/out") - .switch(); let command = positional("COMMAND").help("the command you want to execute in the vm"); let command_args = any::("COMMAND_ARGS", |arg| { (!["--help", "-h"].contains(&&*arg)).then_some(arg) @@ -98,7 +93,6 @@ pub fn options() -> OptionParser { mem, passt_socket, server_port, - interactive, // positionals command, command_args, diff --git a/crates/krun/src/launch.rs b/crates/krun/src/launch.rs index 15846dd..209a2ad 100644 --- a/crates/krun/src/launch.rs +++ b/crates/krun/src/launch.rs @@ -94,11 +94,7 @@ fn wrapped_launch( mut command_args: Vec, env: HashMap, cwd: PathBuf, - interactive: bool, ) -> Result<()> { - if !interactive { - return request_launch(server_port, command, command_args, env, cwd); - } let (mut socat, vsock_port) = start_socat()?; command_args.insert(0, command.to_string_lossy().into_owned()); command_args = vec![ @@ -119,14 +115,13 @@ pub fn launch_or_lock( command: PathBuf, command_args: Vec, env: Vec<(String, Option)>, - interactive: bool, ) -> Result { let running_server_port = env::var("KRUN_SERVER_PORT").ok(); let cwd = env::current_dir()?; if let Some(port) = running_server_port { let port: u32 = port.parse()?; let env = prepare_env_vars(env)?; - if let Err(err) = wrapped_launch(port, command, command_args, env, cwd, interactive) { + if let Err(err) = wrapped_launch(port, command, command_args, env, cwd) { return Err(anyhow!("could not request launch to server: {err}")); } return Ok(LaunchResult::LaunchRequested); @@ -151,7 +146,6 @@ pub fn launch_or_lock( command_args.clone(), env.clone(), cwd.clone(), - interactive, ) { Err(err) => match err.downcast_ref::() { Some(&LaunchError::Connection(_)) => { From ca28bde289f80136df74cf536d23e0e3e364211e Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Sun, 1 Sep 2024 22:43:44 +0200 Subject: [PATCH 5/7] Daemonize the vm process. Launches the initial command via the server too. Server is set to linger for 10 seconds after last command finishes. Signed-off-by: Sasha Finkelstein --- Cargo.lock | 1 + crates/krun-guest/src/bin/krun-guest.rs | 4 +- crates/krun-server/Cargo.toml | 2 +- crates/krun-server/src/bin/krun-server.rs | 75 +++++--------- crates/krun-server/src/cli_options.rs | 19 +--- crates/krun/Cargo.toml | 1 + crates/krun/src/bin/krun.rs | 98 ++++++++++++------ crates/krun/src/launch.rs | 117 ++++++++++++++++------ 8 files changed, 181 insertions(+), 136 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d8f1ac6..6ef1de8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -310,6 +310,7 @@ dependencies = [ "serde", "serde_json", "utils", + "uuid", ] [[package]] diff --git a/crates/krun-guest/src/bin/krun-guest.rs b/crates/krun-guest/src/bin/krun-guest.rs index 7201dfd..e144f8a 100644 --- a/crates/krun-guest/src/bin/krun-guest.rs +++ b/crates/krun-guest/src/bin/krun-guest.rs @@ -45,7 +45,9 @@ fn main() -> Result<()> { configure_network()?; if let Some(hidpipe_client_path) = find_in_path("hidpipe-client")? { - Command::new(hidpipe_client_path).arg(format!("{}", options.uid)).spawn()?; + Command::new(hidpipe_client_path) + .arg(format!("{}", options.uid)) + .spawn()?; } let run_path = match setup_user(options.username, options.uid, options.gid) { diff --git a/crates/krun-server/Cargo.toml b/crates/krun-server/Cargo.toml index 45eb74b..cc9f0e6 100644 --- a/crates/krun-server/Cargo.toml +++ b/crates/krun-server/Cargo.toml @@ -15,7 +15,7 @@ env_logger = { workspace = true, features = ["auto-color", "humantime", "unstabl log = { workspace = true, features = ["kv"] } serde = { workspace = true, features = [] } serde_json = { workspace = true, features = ["std"] } -tokio = { workspace = true, features = ["io-util", "macros", "net", "process", "rt-multi-thread", "sync"] } +tokio = { workspace = true, features = ["io-util", "macros", "net", "process", "rt-multi-thread", "sync", "time"] } tokio-stream = { workspace = true, features = ["net", "sync"] } utils = { workspace = true, features = [] } diff --git a/crates/krun-server/src/bin/krun-server.rs b/crates/krun-server/src/bin/krun-server.rs index a2b0bd4..ed47771 100644 --- a/crates/krun-server/src/bin/krun-server.rs +++ b/crates/krun-server/src/bin/krun-server.rs @@ -1,12 +1,12 @@ -use std::os::unix::process::ExitStatusExt as _; - use anyhow::Result; use krun_server::cli_options::options; use krun_server::server::{Server, State}; use log::error; +use std::time::Duration; use tokio::net::TcpListener; -use tokio::process::Command; use tokio::sync::watch; +use tokio::time; +use tokio::time::Instant; use tokio_stream::wrappers::WatchStream; use tokio_stream::StreamExt as _; @@ -23,18 +23,14 @@ async fn main() -> Result<()> { let mut server = Server::new(listener, state_tx); server.run().await; }); - let command_status = Command::new(&options.command) - .args(options.command_args) - .status(); - tokio::pin!(command_status); - let mut state_rx = WatchStream::new(state_rx); - - let mut server_died = false; - let mut command_exited = false; + let mut state_rx = WatchStream::from_changes(state_rx); + let far_future = Duration::from_secs(3600 * 24 * 365); + let linger_timer = time::sleep(far_future); + tokio::pin!(linger_timer); loop { tokio::select! { - res = &mut server_handle, if !server_died => { + res = &mut server_handle => { // If an error is received here, accepting connections from the // TCP listener failed due to non-transient errors and the // server is giving up and shutting down. @@ -43,51 +39,26 @@ async fn main() -> Result<()> { // not bubble up to this point. if let Err(err) = res { error!(err:% = err; "server task failed"); - server_died = true; - } - }, - res = &mut command_status, if !command_exited => { - match res { - Ok(status) => { - if !status.success() { - if let Some(code) = status.code() { - eprintln!( - "{:?} process exited with status code: {code}", - options.command - ); - } else { - eprintln!( - "{:?} process terminated by signal: {}", - options.command, - status - .signal() - .expect("either one of status code or signal should be set") - ); - } - } - }, - Err(err) => { - eprintln!( - "Failed to execute {:?} as child process: {err}", - options.command - ); - }, + return Ok(()); } - command_exited = true; }, - Some(state) = state_rx.next(), if command_exited => { + Some(state) = state_rx.next() => { if state.connection_idle() && state.child_processes() == 0 { - // Server is idle (not currently handling an accepted - // incoming connection) and no more child processes. - // We're done. - return Ok(()); + linger_timer.as_mut().reset(Instant::now() + Duration::from_secs(10)); + } else { + linger_timer.as_mut().reset(Instant::now() + far_future); + println!( + "Waiting for {} other commands launched through this krun server to exit...", + state.child_processes() + ); } - println!( - "Waiting for {} other commands launched through this krun server to exit...", - state.child_processes() - ); - println!("Press Ctrl+C to force quit"); }, + _tick = &mut linger_timer => { + // Server is idle (not currently handling an accepted + // incoming connection) and no more child processes. + // We're done. + return Ok(()); + } } } } diff --git a/crates/krun-server/src/cli_options.rs b/crates/krun-server/src/cli_options.rs index 62eee04..449938f 100644 --- a/crates/krun-server/src/cli_options.rs +++ b/crates/krun-server/src/cli_options.rs @@ -1,12 +1,8 @@ -use std::path::PathBuf; - -use bpaf::{any, construct, env, positional, OptionParser, Parser}; +use bpaf::{construct, env, OptionParser, Parser}; #[derive(Clone, Debug)] pub struct Options { pub server_port: u32, - pub command: PathBuf, - pub command_args: Vec, } pub fn options() -> OptionParser { @@ -16,19 +12,8 @@ pub fn options() -> OptionParser { .argument("SERVER_PORT") .fallback(3334) .display_fallback(); - let command = positional("COMMAND"); - let command_args = any::("COMMAND_ARGS", |arg| { - (!["--help", "-h"].contains(&&*arg)).then_some(arg) - }) - .many(); - construct!(Options { - server_port, - // positionals - command, - command_args, - }) - .to_options() + construct!(Options { server_port }).to_options() } #[cfg(test)] diff --git a/crates/krun/Cargo.toml b/crates/krun/Cargo.toml index 5095e06..88e2390 100644 --- a/crates/krun/Cargo.toml +++ b/crates/krun/Cargo.toml @@ -19,6 +19,7 @@ rustix = { workspace = true, features = ["process", "std", "use-libc-auxv"] } serde = { workspace = true, features = [] } serde_json = { workspace = true, features = ["std"] } utils = { workspace = true, features = [] } +uuid = { workspace = true, features = ["v4"] } [features] default = [] diff --git a/crates/krun/src/bin/krun.rs b/crates/krun/src/bin/krun.rs index 6e1ac23..7462745 100644 --- a/crates/krun/src/bin/krun.rs +++ b/crates/krun/src/bin/krun.rs @@ -1,28 +1,33 @@ -use std::ffi::{c_char, CString}; -use std::os::fd::{IntoRawFd, OwnedFd}; -use std::path::Path; -use std::{cmp, env}; - use anyhow::{anyhow, Context, Result}; -use krun::cli_options::options; +use krun::cli_options::{options, Options}; use krun::cpu::{get_fallback_cores, get_performance_cores}; use krun::env::{find_krun_exec, prepare_env_vars}; use krun::launch::{launch_or_lock, LaunchResult, DYNAMIC_PORT_RANGE}; use krun::net::{connect_to_passt, start_passt}; use krun::types::MiB; use krun_sys::{ - krun_add_vsock_port, krun_create_ctx, krun_set_exec, krun_set_gpu_options, krun_set_log_level, - krun_set_passt_fd, krun_set_root, krun_set_vm_config, krun_set_workdir, krun_start_enter, - VIRGLRENDERER_DRM, VIRGLRENDERER_THREAD_SYNC, VIRGLRENDERER_USE_ASYNC_FENCE_CB, - VIRGLRENDERER_USE_EGL, + krun_add_vsock_port, krun_create_ctx, krun_set_console_output, krun_set_exec, + krun_set_gpu_options, krun_set_log_level, krun_set_passt_fd, krun_set_root, krun_set_vm_config, + krun_set_workdir, krun_start_enter, VIRGLRENDERER_DRM, VIRGLRENDERER_THREAD_SYNC, + VIRGLRENDERER_USE_ASYNC_FENCE_CB, VIRGLRENDERER_USE_EGL, }; use log::debug; use nix::sys::sysinfo::sysinfo; use nix::unistd::User; -use rustix::io::Errno; +use rustix::io::{dup, Errno}; use rustix::process::{ geteuid, getgid, getrlimit, getuid, sched_setaffinity, setrlimit, CpuSet, Resource, }; +use std::ffi::{c_char, CString}; +use std::fs::File; +use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd}; +use std::path::Path; +use std::process::Command; +use std::{cmp, env}; +use uuid::Uuid; + +const LOCK_FD_ENV_VAR: &str = "__KRUN_DO_LAUNCH_VM_LOCK__"; +const NET_READY_FD_ENV_VAR: &str = "__KRUN_DO_LAUNCH_VM_NET_READY__"; fn main() -> Result<()> { env_logger::init(); @@ -33,8 +38,15 @@ fn main() -> Result<()> { } let options = options().fallback_to_usage().run(); + if let Ok(lock_fd) = env::var(LOCK_FD_ENV_VAR) { + // SAFETY: We are hoping that whoever launched us did indeed put fds there. + let _lock_file = unsafe { File::from_raw_fd(lock_fd.parse()?) }; + let net_ready = env::var(NET_READY_FD_ENV_VAR)?.parse()?; + let net_ready_file = unsafe { File::from_raw_fd(net_ready) }; + return launch_vm(options, net_ready_file); + } - let (_lock_file, command, command_args, env) = match launch_or_lock( + let (lock_file, net_ready, command, command_args, env) = match launch_or_lock( options.server_port, options.command, options.command_args, @@ -47,12 +59,31 @@ fn main() -> Result<()> { }, LaunchResult::LockAcquired { lock_file, + net_ready, command, command_args, env, - } => (lock_file, command, command_args, env), + } => (lock_file, net_ready, command, command_args, env), }; + // Make it lose CLOEXEC + let lock_fd = dup(lock_file)?; + let net_ready_fd = dup(net_ready)?; + Command::new(env::current_exe()?) + .args(env::args()) + .env(LOCK_FD_ENV_VAR, format!("{}", lock_fd.as_raw_fd())) + .env( + NET_READY_FD_ENV_VAR, + format!("{}", net_ready_fd.as_raw_fd()), + ) + .spawn()?; + drop(lock_fd); + drop(net_ready_fd); + let res = launch_or_lock(options.server_port, command, command_args, env)?; + assert!(matches!(res, LaunchResult::LaunchRequested)); + Ok(()) +} +fn launch_vm(options: Options, net_ready_file: File) -> Result<()> { { // Set the log level to "off". // @@ -162,7 +193,7 @@ fn main() -> Result<()> { return Err(err).context("Failed to configure net mode"); } } - + let console_base; if let Ok(run_path) = env::var("XDG_RUNTIME_DIR") { let pulse_path = Path::new(&run_path).join("pulse/native"); if pulse_path.exists() { @@ -214,6 +245,9 @@ fn main() -> Result<()> { return Err(err).context("Failed to configure vsock for dynamic socket"); } } + console_base = run_path; + } else { + console_base = "/tmp".to_string(); } // Forward the native X11 display into the guest as a socket @@ -278,28 +312,12 @@ fn main() -> Result<()> { CString::new(format!("{}", getgid().as_raw())) .expect("gid should not contain NUL character"), ]; - krun_guest_args.push(krun_server_path); - krun_guest_args.push( - CString::new( - command - .to_str() - .context("Failed to process command as it contains invalid UTF-8")?, - ) - .context("Failed to process command as it contains NUL character")?, - ); - let command_argc = command_args.len(); - for arg in command_args { - let s = CString::new(arg) - .context("Failed to process command arg as it contains NUL character")?; - krun_guest_args.push(s); - } let krun_guest_args: Vec<*const c_char> = { - const KRUN_GUEST_ARGS_FIXED: usize = 4; // SAFETY: All pointers must be stored in the same allocation. // See https://doc.rust-lang.org/std/slice/fn.from_raw_parts.html#safety - let mut vec = Vec::with_capacity(KRUN_GUEST_ARGS_FIXED + command_argc + 1); + let mut vec = Vec::with_capacity(krun_guest_args.len() + 1); for s in &krun_guest_args { vec.push(s.as_ptr()); } @@ -307,7 +325,8 @@ fn main() -> Result<()> { vec }; - let mut env = prepare_env_vars(env).context("Failed to prepare environment variables")?; + let mut env = + prepare_env_vars(Vec::new()).context("Failed to prepare environment variables")?; env.insert( "KRUN_SERVER_PORT".to_owned(), options.server_port.to_string(), @@ -358,6 +377,21 @@ fn main() -> Result<()> { } } + { + let uuid = Uuid::now_v7(); + let path = Path::new(&console_base).join(format!("krun-{}.console", uuid)); + let path = CString::new(path.to_str().expect("console_base contains invalid utf-8")) + .context("console_base contains NUL characters")?; + // SAFETY: path is a CString that outlives this call + let err = unsafe { krun_set_console_output(ctx_id, path.as_ptr()) }; + if err < 0 { + let err = Errno::from_raw_os_error(-err); + return Err(err).context("Failed to configure console"); + } + } + + drop(net_ready_file); + { // Start and enter the microVM. Unless there is some error while creating the // microVM this function never returns. diff --git a/crates/krun/src/launch.rs b/crates/krun/src/launch.rs index 209a2ad..c2e8b28 100644 --- a/crates/krun/src/launch.rs +++ b/crates/krun/src/launch.rs @@ -1,17 +1,22 @@ use std::collections::HashMap; -use std::env; use std::error::Error; use std::fmt::{Display, Formatter}; use std::fs::File; -use std::io::{BufRead, BufReader, Read, Write}; +use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; use std::net::TcpStream; use std::path::{Path, PathBuf}; +use std::{env, thread}; use anyhow::{anyhow, Context, Result}; -use rustix::fs::{flock, FlockOperation}; +use rustix::fs::{flock, unlink, FlockOperation}; +use rustix::io::{dup, Errno}; use rustix::path::Arg; use std::ops::Range; -use std::process::{Child, Command}; +use std::os::fd::IntoRawFd; +use std::os::unix::net::UnixListener; +use std::os::unix::process::CommandExt; +use std::process::Command; +use std::time::Duration; use utils::env::find_in_path; use utils::launch::Launch; @@ -23,6 +28,7 @@ pub enum LaunchResult { LaunchRequested, LockAcquired { lock_file: File, + net_ready: File, command: PathBuf, command_args: Vec, env: Vec<(String, Option)>, @@ -54,26 +60,6 @@ impl Display for LaunchError { } } -fn start_socat() -> Result<(Child, u32)> { - let run_path = env::var("XDG_RUNTIME_DIR") - .map_err(|e| anyhow!("unable to get XDG_RUNTIME_DIR: {:?}", e))?; - let socket_dir = Path::new(&run_path).join("krun/socket"); - let socat_path = - find_in_path("socat")?.ok_or_else(|| anyhow!("Unable to find socat in PATH"))?; - for port in DYNAMIC_PORT_RANGE { - let path = socket_dir.join(&format!("port-{}", port)); - if path.exists() { - continue; - } - let child = Command::new(&socat_path) - .arg(format!("unix-l:{}", path.as_os_str().to_string_lossy())) - .arg("-,raw,echo=0") - .spawn()?; - return Ok((child, port)); - } - Err(anyhow!("Ran out of ports.")) -} - fn escape_for_socat(s: String) -> String { let mut ret = String::with_capacity(s.len()); for c in s.chars() { @@ -88,6 +74,32 @@ fn escape_for_socat(s: String) -> String { ret } +fn listen_on_free_socket() -> Result<(UnixListener, File, u32)> { + let run_path = env::var("XDG_RUNTIME_DIR") + .map_err(|e| anyhow!("unable to get XDG_RUNTIME_DIR: {:?}", e))?; + let socket_dir = Path::new(&run_path).join("krun/socket"); + for port in DYNAMIC_PORT_RANGE { + let lock_path = socket_dir.join(&format!("port-{}.lock", port)); + let lock = File::options() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(lock_path)?; + match flock(&lock, FlockOperation::NonBlockingLockExclusive) { + Err(Errno::WOULDBLOCK) => continue, + r => r?, + } + let path = socket_dir.join(&format!("port-{}", port)); + match unlink(&path) { + Err(Errno::NOENT) => {}, + r => r?, + } + return Ok((UnixListener::bind(path)?, lock, port)); + } + Err(anyhow!("Ran out of ports.")) +} + fn wrapped_launch( server_port: u32, mut command: PathBuf, @@ -95,7 +107,9 @@ fn wrapped_launch( env: HashMap, cwd: PathBuf, ) -> Result<()> { - let (mut socat, vsock_port) = start_socat()?; + let socat_path = + find_in_path("socat")?.ok_or_else(|| anyhow!("Unable to find socat in PATH"))?; + let (listener, lock, vsock_port) = listen_on_free_socket()?; command_args.insert(0, command.to_string_lossy().into_owned()); command_args = vec![ format!("vsock:2:{}", vsock_port), @@ -106,8 +120,16 @@ fn wrapped_launch( ]; command = "socat".into(); request_launch(server_port, command, command_args, env, cwd)?; - socat.wait()?; - Ok(()) + + // Clear CLOEXEC + let listen_fd = dup(listener)?.into_raw_fd(); + // Leak the lock into socat, so it holds onto it. + dup(lock)?.into_raw_fd(); + Err(Command::new(&socat_path) + .arg(format!("accept-fd:{}", listen_fd)) + .arg("-,raw,echo=0") + .exec() + .into()) } pub fn launch_or_lock( @@ -127,16 +149,45 @@ pub fn launch_or_lock( return Ok(LaunchResult::LaunchRequested); } + let run_path = env::var("XDG_RUNTIME_DIR") + .context("Failed to read XDG_RUNTIME_DIR environment variable")?; + let net_ready_path = Path::new(&run_path).join("krun.ready"); + let (lock_file, running_server_port) = lock_file(server_port)?; match lock_file { - Some(lock_file) => Ok(LaunchResult::LockAcquired { - lock_file, - command, - command_args, - env, - }), + Some(lock_file) => { + let net_ready = File::options() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(net_ready_path)?; + flock(&net_ready, FlockOperation::LockExclusive)?; + + Ok(LaunchResult::LockAcquired { + lock_file, + net_ready, + command, + command_args, + env, + }) + }, None => { if let Some(port) = running_server_port { + let net_ready = loop { + let net_ready = File::options().read(true).write(true).open(&net_ready_path); + match net_ready { + Ok(f) => break f, + Err(e) => { + if e.kind() == ErrorKind::NotFound { + thread::sleep(Duration::from_millis(1)); + continue; + } + return Err(e.into()); + }, + } + }; + flock(net_ready, FlockOperation::LockShared)?; let env = prepare_env_vars(env)?; let mut tries = 0; loop { From 10eac3cc2caf6e3c94973bca4edc8033a690ae85 Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Mon, 2 Sep 2024 00:46:43 +0200 Subject: [PATCH 6/7] Pass through most environment variables. Change the approach to environment variables to delete ones that are likely to cause problems instead of only passing through a pre-approved list. This will be neccessary to correctly work as a binfmt handler Fixes: #52 Signed-off-by: Sasha Finkelstein --- crates/krun/src/bin/krun.rs | 4 ++-- crates/krun/src/env.rs | 37 ++++++++++++++++++++++++++++++++++++- crates/krun/src/launch.rs | 6 +++--- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/crates/krun/src/bin/krun.rs b/crates/krun/src/bin/krun.rs index 7462745..5330c13 100644 --- a/crates/krun/src/bin/krun.rs +++ b/crates/krun/src/bin/krun.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Context, Result}; use krun::cli_options::{options, Options}; use krun::cpu::{get_fallback_cores, get_performance_cores}; -use krun::env::{find_krun_exec, prepare_env_vars}; +use krun::env::{find_krun_exec, prepare_vm_env_vars}; use krun::launch::{launch_or_lock, LaunchResult, DYNAMIC_PORT_RANGE}; use krun::net::{connect_to_passt, start_passt}; use krun::types::MiB; @@ -326,7 +326,7 @@ fn launch_vm(options: Options, net_ready_file: File) -> Result<()> { }; let mut env = - prepare_env_vars(Vec::new()).context("Failed to prepare environment variables")?; + prepare_vm_env_vars(Vec::new()).context("Failed to prepare environment variables")?; env.insert( "KRUN_SERVER_PORT".to_owned(), options.server_port.to_string(), diff --git a/crates/krun/src/env.rs b/crates/krun/src/env.rs index 138003a..7219535 100644 --- a/crates/krun/src/env.rs +++ b/crates/krun/src/env.rs @@ -22,7 +22,7 @@ const WELL_KNOWN_ENV_VARS: [&str; 5] = [ /// See https://github.com/AsahiLinux/docs/wiki/Devices const ASAHI_SOC_COMPAT_IDS: [&str; 1] = ["apple,arm-platform"]; -pub fn prepare_env_vars(env: Vec<(String, Option)>) -> Result> { +pub fn prepare_vm_env_vars(env: Vec<(String, Option)>) -> Result> { let mut env_map = HashMap::new(); for key in WELL_KNOWN_ENV_VARS { @@ -83,6 +83,41 @@ pub fn prepare_env_vars(env: Vec<(String, Option)>) -> Result)>) -> HashMap { + let mut vars = HashMap::new(); + for (k, v) in env::vars() { + vars.insert(k, v); + } + for (k, v) in env { + if let Some(v) = v { + vars.insert(k, v); + } + } + for k in DROP_ENV_VARS { + vars.remove(k); + } + vars +} + pub fn find_krun_exec

(program: P) -> Result where P: AsRef, diff --git a/crates/krun/src/launch.rs b/crates/krun/src/launch.rs index c2e8b28..0876aa5 100644 --- a/crates/krun/src/launch.rs +++ b/crates/krun/src/launch.rs @@ -20,7 +20,7 @@ use std::time::Duration; use utils::env::find_in_path; use utils::launch::Launch; -use crate::env::prepare_env_vars; +use crate::env::prepare_proc_env_vars; pub const DYNAMIC_PORT_RANGE: Range = 50000..50200; @@ -142,7 +142,7 @@ pub fn launch_or_lock( let cwd = env::current_dir()?; if let Some(port) = running_server_port { let port: u32 = port.parse()?; - let env = prepare_env_vars(env)?; + let env = prepare_proc_env_vars(env); if let Err(err) = wrapped_launch(port, command, command_args, env, cwd) { return Err(anyhow!("could not request launch to server: {err}")); } @@ -188,7 +188,7 @@ pub fn launch_or_lock( } }; flock(net_ready, FlockOperation::LockShared)?; - let env = prepare_env_vars(env)?; + let env = prepare_proc_env_vars(env); let mut tries = 0; loop { match wrapped_launch( From 20c8dd45611dd7441b02cca7a6d5dae30088b012 Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Mon, 2 Sep 2024 01:20:01 +0200 Subject: [PATCH 7/7] Add binfmt_misc configuration files. Closes: #24 Signed-off-by: Sasha Finkelstein --- etc/binfmt.d/krun.conf | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 etc/binfmt.d/krun.conf diff --git a/etc/binfmt.d/krun.conf b/etc/binfmt.d/krun.conf new file mode 100644 index 0000000..b7b0950 --- /dev/null +++ b/etc/binfmt.d/krun.conf @@ -0,0 +1,3 @@ +:krun-i386:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/krun:F +:krun-i486:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x06\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/krun:F +:krun-x86_64:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/krun:F