diff --git a/Cargo.lock b/Cargo.lock index d8f1ac6..6ef1de8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -310,6 +310,7 @@ dependencies = [ "serde", "serde_json", "utils", + "uuid", ] [[package]] diff --git a/crates/krun-guest/src/bin/krun-guest.rs b/crates/krun-guest/src/bin/krun-guest.rs index 7201dfd..e144f8a 100644 --- a/crates/krun-guest/src/bin/krun-guest.rs +++ b/crates/krun-guest/src/bin/krun-guest.rs @@ -45,7 +45,9 @@ fn main() -> Result<()> { configure_network()?; if let Some(hidpipe_client_path) = find_in_path("hidpipe-client")? { - Command::new(hidpipe_client_path).arg(format!("{}", options.uid)).spawn()?; + Command::new(hidpipe_client_path) + .arg(format!("{}", options.uid)) + .spawn()?; } let run_path = match setup_user(options.username, options.uid, options.gid) { diff --git a/crates/krun-server/Cargo.toml b/crates/krun-server/Cargo.toml index 45eb74b..cc9f0e6 100644 --- a/crates/krun-server/Cargo.toml +++ b/crates/krun-server/Cargo.toml @@ -15,7 +15,7 @@ env_logger = { workspace = true, features = ["auto-color", "humantime", "unstabl log = { workspace = true, features = ["kv"] } serde = { workspace = true, features = [] } serde_json = { workspace = true, features = ["std"] } -tokio = { workspace = true, features = ["io-util", "macros", "net", "process", "rt-multi-thread", "sync"] } +tokio = { workspace = true, features = ["io-util", "macros", "net", "process", "rt-multi-thread", "sync", "time"] } tokio-stream = { workspace = true, features = ["net", "sync"] } utils = { workspace = true, features = [] } diff --git a/crates/krun-server/src/bin/krun-server.rs b/crates/krun-server/src/bin/krun-server.rs index a2b0bd4..ed47771 100644 --- a/crates/krun-server/src/bin/krun-server.rs +++ b/crates/krun-server/src/bin/krun-server.rs @@ -1,12 +1,12 @@ -use std::os::unix::process::ExitStatusExt as _; - use anyhow::Result; use krun_server::cli_options::options; use krun_server::server::{Server, State}; use log::error; +use std::time::Duration; use tokio::net::TcpListener; -use tokio::process::Command; use tokio::sync::watch; +use tokio::time; +use tokio::time::Instant; use tokio_stream::wrappers::WatchStream; use tokio_stream::StreamExt as _; @@ -23,18 +23,14 @@ async fn main() -> Result<()> { let mut server = Server::new(listener, state_tx); server.run().await; }); - let command_status = Command::new(&options.command) - .args(options.command_args) - .status(); - tokio::pin!(command_status); - let mut state_rx = WatchStream::new(state_rx); - - let mut server_died = false; - let mut command_exited = false; + let mut state_rx = WatchStream::from_changes(state_rx); + let far_future = Duration::from_secs(3600 * 24 * 365); + let linger_timer = time::sleep(far_future); + tokio::pin!(linger_timer); loop { tokio::select! { - res = &mut server_handle, if !server_died => { + res = &mut server_handle => { // If an error is received here, accepting connections from the // TCP listener failed due to non-transient errors and the // server is giving up and shutting down. @@ -43,51 +39,26 @@ async fn main() -> Result<()> { // not bubble up to this point. if let Err(err) = res { error!(err:% = err; "server task failed"); - server_died = true; - } - }, - res = &mut command_status, if !command_exited => { - match res { - Ok(status) => { - if !status.success() { - if let Some(code) = status.code() { - eprintln!( - "{:?} process exited with status code: {code}", - options.command - ); - } else { - eprintln!( - "{:?} process terminated by signal: {}", - options.command, - status - .signal() - .expect("either one of status code or signal should be set") - ); - } - } - }, - Err(err) => { - eprintln!( - "Failed to execute {:?} as child process: {err}", - options.command - ); - }, + return Ok(()); } - command_exited = true; }, - Some(state) = state_rx.next(), if command_exited => { + Some(state) = state_rx.next() => { if state.connection_idle() && state.child_processes() == 0 { - // Server is idle (not currently handling an accepted - // incoming connection) and no more child processes. - // We're done. - return Ok(()); + linger_timer.as_mut().reset(Instant::now() + Duration::from_secs(10)); + } else { + linger_timer.as_mut().reset(Instant::now() + far_future); + println!( + "Waiting for {} other commands launched through this krun server to exit...", + state.child_processes() + ); } - println!( - "Waiting for {} other commands launched through this krun server to exit...", - state.child_processes() - ); - println!("Press Ctrl+C to force quit"); }, + _tick = &mut linger_timer => { + // Server is idle (not currently handling an accepted + // incoming connection) and no more child processes. + // We're done. + return Ok(()); + } } } } diff --git a/crates/krun-server/src/cli_options.rs b/crates/krun-server/src/cli_options.rs index 62eee04..449938f 100644 --- a/crates/krun-server/src/cli_options.rs +++ b/crates/krun-server/src/cli_options.rs @@ -1,12 +1,8 @@ -use std::path::PathBuf; - -use bpaf::{any, construct, env, positional, OptionParser, Parser}; +use bpaf::{construct, env, OptionParser, Parser}; #[derive(Clone, Debug)] pub struct Options { pub server_port: u32, - pub command: PathBuf, - pub command_args: Vec, } pub fn options() -> OptionParser { @@ -16,19 +12,8 @@ pub fn options() -> OptionParser { .argument("SERVER_PORT") .fallback(3334) .display_fallback(); - let command = positional("COMMAND"); - let command_args = any::("COMMAND_ARGS", |arg| { - (!["--help", "-h"].contains(&&*arg)).then_some(arg) - }) - .many(); - construct!(Options { - server_port, - // positionals - command, - command_args, - }) - .to_options() + construct!(Options { server_port }).to_options() } #[cfg(test)] diff --git a/crates/krun-server/src/server.rs b/crates/krun-server/src/server.rs index 5299f99..3506156 100644 --- a/crates/krun-server/src/server.rs +++ b/crates/krun-server/src/server.rs @@ -164,8 +164,9 @@ async fn handle_connection(mut stream: BufStream) -> Result<(PathBuf, command, command_args, env, + cwd, } = read_request(&mut stream).await?; - debug!(command:?, command_args:?, env:?; "received launch request"); + debug!(command:?, command_args:?, env:?, cwd:?; "received launch request"); envs.extend(env); let (stdout, stderr) = make_stdout_stderr(&command, &envs)?; @@ -176,6 +177,7 @@ async fn handle_connection(mut stream: BufStream) -> Result<(PathBuf, .stdin(Stdio::null()) .stdout(stdout) .stderr(stderr) + .current_dir(cwd) .spawn() .with_context(|| format!("Failed to execute {command:?} as child process")); if let Err(err) = &res { diff --git a/crates/krun/Cargo.toml b/crates/krun/Cargo.toml index 5095e06..88e2390 100644 --- a/crates/krun/Cargo.toml +++ b/crates/krun/Cargo.toml @@ -19,6 +19,7 @@ rustix = { workspace = true, features = ["process", "std", "use-libc-auxv"] } serde = { workspace = true, features = [] } serde_json = { workspace = true, features = ["std"] } utils = { workspace = true, features = [] } +uuid = { workspace = true, features = ["v4"] } [features] default = [] diff --git a/crates/krun/src/bin/krun.rs b/crates/krun/src/bin/krun.rs index fb3f1e5..5330c13 100644 --- a/crates/krun/src/bin/krun.rs +++ b/crates/krun/src/bin/krun.rs @@ -1,28 +1,33 @@ -use std::ffi::{c_char, CString}; -use std::os::fd::{IntoRawFd, OwnedFd}; -use std::path::Path; -use std::{cmp, env}; - use anyhow::{anyhow, Context, Result}; -use krun::cli_options::options; +use krun::cli_options::{options, Options}; use krun::cpu::{get_fallback_cores, get_performance_cores}; -use krun::env::{find_krun_exec, prepare_env_vars}; -use krun::launch::{launch_or_lock, LaunchResult}; +use krun::env::{find_krun_exec, prepare_vm_env_vars}; +use krun::launch::{launch_or_lock, LaunchResult, DYNAMIC_PORT_RANGE}; use krun::net::{connect_to_passt, start_passt}; use krun::types::MiB; use krun_sys::{ - krun_add_vsock_port, krun_create_ctx, krun_set_exec, krun_set_gpu_options, krun_set_log_level, - krun_set_passt_fd, krun_set_root, krun_set_vm_config, krun_set_workdir, krun_start_enter, - VIRGLRENDERER_DRM, VIRGLRENDERER_THREAD_SYNC, VIRGLRENDERER_USE_ASYNC_FENCE_CB, - VIRGLRENDERER_USE_EGL, + krun_add_vsock_port, krun_create_ctx, krun_set_console_output, krun_set_exec, + krun_set_gpu_options, krun_set_log_level, krun_set_passt_fd, krun_set_root, krun_set_vm_config, + krun_set_workdir, krun_start_enter, VIRGLRENDERER_DRM, VIRGLRENDERER_THREAD_SYNC, + VIRGLRENDERER_USE_ASYNC_FENCE_CB, VIRGLRENDERER_USE_EGL, }; use log::debug; use nix::sys::sysinfo::sysinfo; use nix::unistd::User; -use rustix::io::Errno; +use rustix::io::{dup, Errno}; use rustix::process::{ geteuid, getgid, getrlimit, getuid, sched_setaffinity, setrlimit, CpuSet, Resource, }; +use std::ffi::{c_char, CString}; +use std::fs::File; +use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd}; +use std::path::Path; +use std::process::Command; +use std::{cmp, env}; +use uuid::Uuid; + +const LOCK_FD_ENV_VAR: &str = "__KRUN_DO_LAUNCH_VM_LOCK__"; +const NET_READY_FD_ENV_VAR: &str = "__KRUN_DO_LAUNCH_VM_NET_READY__"; fn main() -> Result<()> { env_logger::init(); @@ -33,8 +38,15 @@ fn main() -> Result<()> { } let options = options().fallback_to_usage().run(); + if let Ok(lock_fd) = env::var(LOCK_FD_ENV_VAR) { + // SAFETY: We are hoping that whoever launched us did indeed put fds there. + let _lock_file = unsafe { File::from_raw_fd(lock_fd.parse()?) }; + let net_ready = env::var(NET_READY_FD_ENV_VAR)?.parse()?; + let net_ready_file = unsafe { File::from_raw_fd(net_ready) }; + return launch_vm(options, net_ready_file); + } - let (_lock_file, command, command_args, env) = match launch_or_lock( + let (lock_file, net_ready, command, command_args, env) = match launch_or_lock( options.server_port, options.command, options.command_args, @@ -47,12 +59,31 @@ fn main() -> Result<()> { }, LaunchResult::LockAcquired { lock_file, + net_ready, command, command_args, env, - } => (lock_file, command, command_args, env), + } => (lock_file, net_ready, command, command_args, env), }; + // Make it lose CLOEXEC + let lock_fd = dup(lock_file)?; + let net_ready_fd = dup(net_ready)?; + Command::new(env::current_exe()?) + .args(env::args()) + .env(LOCK_FD_ENV_VAR, format!("{}", lock_fd.as_raw_fd())) + .env( + NET_READY_FD_ENV_VAR, + format!("{}", net_ready_fd.as_raw_fd()), + ) + .spawn()?; + drop(lock_fd); + drop(net_ready_fd); + let res = launch_or_lock(options.server_port, command, command_args, env)?; + assert!(matches!(res, LaunchResult::LaunchRequested)); + Ok(()) +} +fn launch_vm(options: Options, net_ready_file: File) -> Result<()> { { // Set the log level to "off". // @@ -162,7 +193,7 @@ fn main() -> Result<()> { return Err(err).context("Failed to configure net mode"); } } - + let console_base; if let Ok(run_path) = env::var("XDG_RUNTIME_DIR") { let pulse_path = Path::new(&run_path).join("pulse/native"); if pulse_path.exists() { @@ -179,6 +210,7 @@ fn main() -> Result<()> { return Err(err).context("Failed to configure vsock for pulse socket"); } } + let hidpipe_path = Path::new(&run_path).join("hidpipe"); if hidpipe_path.exists() { let hidpipe_path = CString::new( @@ -194,6 +226,28 @@ fn main() -> Result<()> { return Err(err).context("Failed to configure vsock for hidpipe socket"); } } + + let socket_dir = Path::new(&run_path).join("krun/socket"); + std::fs::create_dir_all(&socket_dir)?; + // Dynamic ports: Applications may listen on these sockets as neeeded. + for port in DYNAMIC_PORT_RANGE { + let socket_path = socket_dir.join(format!("port-{}", port)); + let socket_path = CString::new( + socket_path + .to_str() + .expect("socket_path should not contain invalid UTF-8"), + ) + .context("Failed to process dynamic socket path as it contains NUL character")?; + // SAFETY: `socket_path` is a pointer to a `CString` with long enough lifetime. + let err = unsafe { krun_add_vsock_port(ctx_id, port, socket_path.as_ptr()) }; + if err < 0 { + let err = Errno::from_raw_os_error(-err); + return Err(err).context("Failed to configure vsock for dynamic socket"); + } + } + console_base = run_path; + } else { + console_base = "/tmp".to_string(); } // Forward the native X11 display into the guest as a socket @@ -258,28 +312,12 @@ fn main() -> Result<()> { CString::new(format!("{}", getgid().as_raw())) .expect("gid should not contain NUL character"), ]; - krun_guest_args.push(krun_server_path); - krun_guest_args.push( - CString::new( - command - .to_str() - .context("Failed to process command as it contains invalid UTF-8")?, - ) - .context("Failed to process command as it contains NUL character")?, - ); - let command_argc = command_args.len(); - for arg in command_args { - let s = CString::new(arg) - .context("Failed to process command arg as it contains NUL character")?; - krun_guest_args.push(s); - } let krun_guest_args: Vec<*const c_char> = { - const KRUN_GUEST_ARGS_FIXED: usize = 4; // SAFETY: All pointers must be stored in the same allocation. // See https://doc.rust-lang.org/std/slice/fn.from_raw_parts.html#safety - let mut vec = Vec::with_capacity(KRUN_GUEST_ARGS_FIXED + command_argc + 1); + let mut vec = Vec::with_capacity(krun_guest_args.len() + 1); for s in &krun_guest_args { vec.push(s.as_ptr()); } @@ -287,7 +325,8 @@ fn main() -> Result<()> { vec }; - let mut env = prepare_env_vars(env).context("Failed to prepare environment variables")?; + let mut env = + prepare_vm_env_vars(Vec::new()).context("Failed to prepare environment variables")?; env.insert( "KRUN_SERVER_PORT".to_owned(), options.server_port.to_string(), @@ -338,6 +377,21 @@ fn main() -> Result<()> { } } + { + let uuid = Uuid::now_v7(); + let path = Path::new(&console_base).join(format!("krun-{}.console", uuid)); + let path = CString::new(path.to_str().expect("console_base contains invalid utf-8")) + .context("console_base contains NUL characters")?; + // SAFETY: path is a CString that outlives this call + let err = unsafe { krun_set_console_output(ctx_id, path.as_ptr()) }; + if err < 0 { + let err = Errno::from_raw_os_error(-err); + return Err(err).context("Failed to configure console"); + } + } + + drop(net_ready_file); + { // Start and enter the microVM. Unless there is some error while creating the // microVM this function never returns. diff --git a/crates/krun/src/env.rs b/crates/krun/src/env.rs index 138003a..7219535 100644 --- a/crates/krun/src/env.rs +++ b/crates/krun/src/env.rs @@ -22,7 +22,7 @@ const WELL_KNOWN_ENV_VARS: [&str; 5] = [ /// See https://github.com/AsahiLinux/docs/wiki/Devices const ASAHI_SOC_COMPAT_IDS: [&str; 1] = ["apple,arm-platform"]; -pub fn prepare_env_vars(env: Vec<(String, Option)>) -> Result> { +pub fn prepare_vm_env_vars(env: Vec<(String, Option)>) -> Result> { let mut env_map = HashMap::new(); for key in WELL_KNOWN_ENV_VARS { @@ -83,6 +83,41 @@ pub fn prepare_env_vars(env: Vec<(String, Option)>) -> Result)>) -> HashMap { + let mut vars = HashMap::new(); + for (k, v) in env::vars() { + vars.insert(k, v); + } + for (k, v) in env { + if let Some(v) = v { + vars.insert(k, v); + } + } + for k in DROP_ENV_VARS { + vars.remove(k); + } + vars +} + pub fn find_krun_exec

(program: P) -> Result where P: AsRef, diff --git a/crates/krun/src/launch.rs b/crates/krun/src/launch.rs index 65ed567..0876aa5 100644 --- a/crates/krun/src/launch.rs +++ b/crates/krun/src/launch.rs @@ -1,23 +1,34 @@ use std::collections::HashMap; -use std::env; use std::error::Error; use std::fmt::{Display, Formatter}; use std::fs::File; -use std::io::{BufRead, BufReader, Read, Write}; +use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; use std::net::TcpStream; use std::path::{Path, PathBuf}; +use std::{env, thread}; use anyhow::{anyhow, Context, Result}; -use rustix::fs::{flock, FlockOperation}; +use rustix::fs::{flock, unlink, FlockOperation}; +use rustix::io::{dup, Errno}; use rustix::path::Arg; +use std::ops::Range; +use std::os::fd::IntoRawFd; +use std::os::unix::net::UnixListener; +use std::os::unix::process::CommandExt; +use std::process::Command; +use std::time::Duration; +use utils::env::find_in_path; use utils::launch::Launch; -use crate::env::prepare_env_vars; +use crate::env::prepare_proc_env_vars; + +pub const DYNAMIC_PORT_RANGE: Range = 50000..50200; pub enum LaunchResult { LaunchRequested, LockAcquired { lock_file: File, + net_ready: File, command: PathBuf, command_args: Vec, env: Vec<(String, Option)>, @@ -49,6 +60,78 @@ impl Display for LaunchError { } } +fn escape_for_socat(s: String) -> String { + let mut ret = String::with_capacity(s.len()); + for c in s.chars() { + match c { + ':' | ',' | '!' | '"' | '\'' | '\\' | '(' | '[' | '{' => { + ret.push('\\'); + }, + _ => {}, + } + ret.push(c); + } + ret +} + +fn listen_on_free_socket() -> Result<(UnixListener, File, u32)> { + let run_path = env::var("XDG_RUNTIME_DIR") + .map_err(|e| anyhow!("unable to get XDG_RUNTIME_DIR: {:?}", e))?; + let socket_dir = Path::new(&run_path).join("krun/socket"); + for port in DYNAMIC_PORT_RANGE { + let lock_path = socket_dir.join(&format!("port-{}.lock", port)); + let lock = File::options() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(lock_path)?; + match flock(&lock, FlockOperation::NonBlockingLockExclusive) { + Err(Errno::WOULDBLOCK) => continue, + r => r?, + } + let path = socket_dir.join(&format!("port-{}", port)); + match unlink(&path) { + Err(Errno::NOENT) => {}, + r => r?, + } + return Ok((UnixListener::bind(path)?, lock, port)); + } + Err(anyhow!("Ran out of ports.")) +} + +fn wrapped_launch( + server_port: u32, + mut command: PathBuf, + mut command_args: Vec, + env: HashMap, + cwd: PathBuf, +) -> Result<()> { + let socat_path = + find_in_path("socat")?.ok_or_else(|| anyhow!("Unable to find socat in PATH"))?; + let (listener, lock, vsock_port) = listen_on_free_socket()?; + command_args.insert(0, command.to_string_lossy().into_owned()); + command_args = vec![ + format!("vsock:2:{}", vsock_port), + format!( + "exec:{},pty,setsid,stderr", + escape_for_socat(command_args.join(" ")) + ), + ]; + command = "socat".into(); + request_launch(server_port, command, command_args, env, cwd)?; + + // Clear CLOEXEC + let listen_fd = dup(listener)?.into_raw_fd(); + // Leak the lock into socat, so it holds onto it. + dup(lock)?.into_raw_fd(); + Err(Command::new(&socat_path) + .arg(format!("accept-fd:{}", listen_fd)) + .arg("-,raw,echo=0") + .exec() + .into()) +} + pub fn launch_or_lock( server_port: u32, command: PathBuf, @@ -56,29 +139,65 @@ pub fn launch_or_lock( env: Vec<(String, Option)>, ) -> Result { let running_server_port = env::var("KRUN_SERVER_PORT").ok(); + let cwd = env::current_dir()?; if let Some(port) = running_server_port { let port: u32 = port.parse()?; - let env = prepare_env_vars(env)?; - if let Err(err) = request_launch(port, command, command_args, env) { + let env = prepare_proc_env_vars(env); + if let Err(err) = wrapped_launch(port, command, command_args, env, cwd) { return Err(anyhow!("could not request launch to server: {err}")); } return Ok(LaunchResult::LaunchRequested); } + let run_path = env::var("XDG_RUNTIME_DIR") + .context("Failed to read XDG_RUNTIME_DIR environment variable")?; + let net_ready_path = Path::new(&run_path).join("krun.ready"); + let (lock_file, running_server_port) = lock_file(server_port)?; match lock_file { - Some(lock_file) => Ok(LaunchResult::LockAcquired { - lock_file, - command, - command_args, - env, - }), + Some(lock_file) => { + let net_ready = File::options() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(net_ready_path)?; + flock(&net_ready, FlockOperation::LockExclusive)?; + + Ok(LaunchResult::LockAcquired { + lock_file, + net_ready, + command, + command_args, + env, + }) + }, None => { if let Some(port) = running_server_port { - let env = prepare_env_vars(env)?; + let net_ready = loop { + let net_ready = File::options().read(true).write(true).open(&net_ready_path); + match net_ready { + Ok(f) => break f, + Err(e) => { + if e.kind() == ErrorKind::NotFound { + thread::sleep(Duration::from_millis(1)); + continue; + } + return Err(e.into()); + }, + } + }; + flock(net_ready, FlockOperation::LockShared)?; + let env = prepare_proc_env_vars(env); let mut tries = 0; loop { - match request_launch(port, command.clone(), command_args.clone(), env.clone()) { + match wrapped_launch( + port, + command.clone(), + command_args.clone(), + env.clone(), + cwd.clone(), + ) { Err(err) => match err.downcast_ref::() { Some(&LaunchError::Connection(_)) => { if tries == 3 { @@ -150,6 +269,7 @@ fn request_launch( command: PathBuf, command_args: Vec, env: HashMap, + cwd: PathBuf, ) -> Result<()> { let mut stream = TcpStream::connect(format!("127.0.0.1:{server_port}")).map_err(LaunchError::Connection)?; @@ -158,6 +278,7 @@ fn request_launch( command, command_args, env, + cwd, }; stream diff --git a/crates/utils/src/launch.rs b/crates/utils/src/launch.rs index 78a3dcf..4754c6e 100644 --- a/crates/utils/src/launch.rs +++ b/crates/utils/src/launch.rs @@ -8,4 +8,5 @@ pub struct Launch { pub command: PathBuf, pub command_args: Vec, pub env: HashMap, + pub cwd: PathBuf, } diff --git a/etc/binfmt.d/krun.conf b/etc/binfmt.d/krun.conf new file mode 100644 index 0000000..b7b0950 --- /dev/null +++ b/etc/binfmt.d/krun.conf @@ -0,0 +1,3 @@ +:krun-i386:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/krun:F +:krun-i486:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x06\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/krun:F +:krun-x86_64:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/krun:F