From a95d2bfa67ef69636bf04fc761c3cbc02079f261 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Mon, 27 Jul 2026 22:40:54 +0000 Subject: [PATCH 01/91] Bundle the Firecracker jailer alongside the VMM The jailer ships in the same release archive as Firecracker, so extracting both entries from a single download keeps the existing SHA256 pin as the only trust anchor and adds no new URL. Bundling them together also guarantees the VMM and its jailer cannot drift apart across a runner self-update, since both are re-extracted from the same binary. Adds a BENCHER_JAILER_PATH build-time override to match the existing BENCHER_FIRECRACKER_PATH, so a debug build that supplies its own Firecracker is not left without a jailer. --- plus/bencher_runner/build.rs | 197 ++++++++++++++++---------- plus/bencher_runner/src/jailer_bin.rs | 52 +++++++ plus/bencher_runner/src/lib.rs | 2 + 3 files changed, 174 insertions(+), 77 deletions(-) create mode 100644 plus/bencher_runner/src/jailer_bin.rs diff --git a/plus/bencher_runner/build.rs b/plus/bencher_runner/build.rs index b56816002..2f103aa4b 100644 --- a/plus/bencher_runner/build.rs +++ b/plus/bencher_runner/build.rs @@ -1,7 +1,7 @@ //! Build script for `bencher_runner`. //! -//! Bundles the `bencher-init`, `firecracker`, and `vmlinux` binaries -//! for distribution as a single binary. +//! Bundles the `bencher-init`, `firecracker`, `jailer`, and `vmlinux` +//! binaries for distribution as a single binary. //! //! In release builds, binaries are embedded via `include_bytes!`. //! In debug builds, they are downloaded/cached locally and loaded from disk at runtime. @@ -22,6 +22,7 @@ //! //! - `BENCHER_INIT_PATH` — path to a pre-built bencher-init binary //! - `BENCHER_FIRECRACKER_PATH` — path to a pre-built firecracker binary +//! - `BENCHER_JAILER_PATH` — path to a pre-built jailer binary //! - `BENCHER_KERNEL_PATH` — path to a pre-built vmlinux kernel #![expect( @@ -103,16 +104,26 @@ fn main() { generate_stub_module("init", &out_dir); } - // --- firecracker --- - let firecracker_path = find_or_download_firecracker(&out_dir); - if is_release { - let firecracker_path = firecracker_path.unwrap_or_else(|| panic!("firecracker binary not found. Set BENCHER_FIRECRACKER_PATH or ensure download succeeds.")); - generate_binary_module("firecracker", &firecracker_path, is_release, &out_dir); - } else if let Some(firecracker_path) = firecracker_path { - generate_binary_module("firecracker", &firecracker_path, is_release, &out_dir); - } else { - eprintln!("WARNING: firecracker not found, generating stub module for debug build"); - generate_stub_module("firecracker", &out_dir); + // --- firecracker and jailer --- + // Both ship in the same release archive, so a single download under a + // single hash check yields both. Bundling them together also keeps the + // VMM and its jailer at the same version across a runner self-update. + let (firecracker_path, jailer_path) = find_or_download_firecracker_release(&out_dir); + for (name, path) in [("firecracker", firecracker_path), ("jailer", jailer_path)] { + if is_release { + let path = path.unwrap_or_else(|| { + panic!( + "{name} binary not found. Set BENCHER_{}_PATH or ensure download succeeds.", + name.to_uppercase() + ) + }); + generate_binary_module(name, &path, is_release, &out_dir); + } else if let Some(path) = path { + generate_binary_module(name, &path, is_release, &out_dir); + } else { + eprintln!("WARNING: {name} not found, generating stub module for debug build"); + generate_stub_module(name, &out_dir); + } } // --- kernel (vmlinux) --- @@ -132,6 +143,7 @@ fn main() { println!("cargo:rerun-if-changed=build.rs"); println!("cargo:rerun-if-env-changed=BENCHER_INIT_PATH"); println!("cargo:rerun-if-env-changed=BENCHER_FIRECRACKER_PATH"); + println!("cargo:rerun-if-env-changed=BENCHER_JAILER_PATH"); println!("cargo:rerun-if-env-changed=BENCHER_KERNEL_PATH"); println!("cargo:rerun-if-env-changed=PROFILE"); } @@ -200,70 +212,88 @@ fn find_init_binary() -> Option { None } -/// Find or download the Firecracker binary. +/// Find or download the `firecracker` and `jailer` binaries. /// -/// Checks `BENCHER_FIRECRACKER_PATH` env var first, then tries to download -/// the `.tgz` release archive from GitHub and extract the binary to `OUT_DIR`. -fn find_or_download_firecracker(out_dir: &Path) -> Option { - // 1. Check explicit env var - if let Ok(path) = env::var("BENCHER_FIRECRACKER_PATH") { - let path = PathBuf::from(path); - if path.exists() { - eprintln!( - "Using firecracker from BENCHER_FIRECRACKER_PATH: {}", - path.display() - ); - return Some(path); - } - eprintln!( - "WARNING: BENCHER_FIRECRACKER_PATH set but file not found: {}", - path.display() - ); - } +/// Checks the `BENCHER_FIRECRACKER_PATH` and `BENCHER_JAILER_PATH` env vars +/// first, then downloads the `.tgz` release archive from GitHub once and +/// extracts whichever binaries are still missing into `OUT_DIR`. +/// +/// Returns `(firecracker, jailer)`. +fn find_or_download_firecracker_release(out_dir: &Path) -> (Option, Option) { + let firecracker_override = binary_path_override("firecracker", "BENCHER_FIRECRACKER_PATH"); + let jailer_override = binary_path_override("jailer", "BENCHER_JAILER_PATH"); - // 2. Download from GitHub releases let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); let arch = match target_arch.as_str() { "x86_64" => "x86_64", "aarch64" => "aarch64", _ => { - eprintln!("Unsupported architecture for firecracker: {target_arch}"); - return None; + eprintln!("Unsupported architecture for the Firecracker release: {target_arch}"); + return (firecracker_override, jailer_override); }, }; - let dest = out_dir.join("firecracker"); - if dest.exists() { - eprintln!("Using cached firecracker at: {}", dest.display()); - return Some(dest); - } - - let url = format!( - "https://github.com/firecracker-microvm/firecracker/releases/download/{DEFAULT_FIRECRACKER_VERSION}/firecracker-{DEFAULT_FIRECRACKER_VERSION}-{arch}.tgz", - ); + // The binaries inside the tgz are at: + // release-{version}-{arch}/{name}-{version}-{arch} + let wanted: Vec<(String, PathBuf)> = ["firecracker", "jailer"] + .into_iter() + .zip([&firecracker_override, &jailer_override]) + .filter(|&(_, overridden)| overridden.is_none()) + .map(|(name, _)| { + ( + format!( + "release-{DEFAULT_FIRECRACKER_VERSION}-{arch}/{name}-{DEFAULT_FIRECRACKER_VERSION}-{arch}", + ), + out_dir.join(name), + ) + }) + .filter(|(_, dest)| { + let cached = dest.exists(); + if cached { + eprintln!("Using cached binary at: {}", dest.display()); + } + !cached + }) + .collect(); - // The binary inside the tgz is at: - // release-{version}-{arch}/firecracker-{version}-{arch} - let entry_name = format!( - "release-{DEFAULT_FIRECRACKER_VERSION}-{arch}/firecracker-{DEFAULT_FIRECRACKER_VERSION}-{arch}", - ); + if !wanted.is_empty() { + let url = format!( + "https://github.com/firecracker-microvm/firecracker/releases/download/{DEFAULT_FIRECRACKER_VERSION}/firecracker-{DEFAULT_FIRECRACKER_VERSION}-{arch}.tgz", + ); + let expected_hash = match arch { + "x86_64" => FIRECRACKER_TGZ_SHA256_X86_64, + "aarch64" => FIRECRACKER_TGZ_SHA256_AARCH64, + _ => unreachable!(), + }; + + eprintln!("Downloading the Firecracker release from: {url}"); + if let Err(e) = download_and_extract_tgz(&url, &wanted, Some(expected_hash)) { + eprintln!("WARNING: Failed to download/extract the Firecracker release: {e}"); + } + } - let expected_hash = match arch { - "x86_64" => FIRECRACKER_TGZ_SHA256_X86_64, - "aarch64" => FIRECRACKER_TGZ_SHA256_AARCH64, - _ => unreachable!(), + let resolved = |overridden: Option, name: &str| { + overridden.or_else(|| { + let dest = out_dir.join(name); + dest.exists().then_some(dest) + }) }; - eprintln!("Downloading firecracker from: {url}"); - match download_and_extract_tgz(&url, &entry_name, &dest, Some(expected_hash)) { - Ok(()) => { - eprintln!("Extracted firecracker to: {}", dest.display()); - Some(dest) - }, - Err(e) => { - eprintln!("WARNING: Failed to download/extract firecracker: {e}"); - None - }, + ( + resolved(firecracker_override, "firecracker"), + resolved(jailer_override, "jailer"), + ) +} + +/// Resolve a build-time binary path override from an env var. +fn binary_path_override(name: &str, var: &str) -> Option { + let path = PathBuf::from(env::var(var).ok()?); + if path.exists() { + eprintln!("Using {name} from {var}: {}", path.display()); + Some(path) + } else { + eprintln!("WARNING: {var} set but file not found: {}", path.display()); + None } } @@ -351,18 +381,17 @@ fn download_file(url: &str, dest: &Path, expected_sha256: Option<&str>) -> Resul Ok(()) } -/// Download a `.tgz` archive and extract a single file from it. +/// Download a `.tgz` archive and extract the requested files from it. /// /// # Arguments /// /// * `url` - URL of the `.tgz` archive -/// * `entry_name` - Path of the entry to extract (e.g., `release-v1.15.1-x86_64/firecracker-v1.15.1-x86_64`) -/// * `dest` - Destination path for the extracted file +/// * `wanted` - `(entry_name, dest)` pairs, where `entry_name` is the path of +/// the entry inside the archive (e.g., `release-v1.15.1-x86_64/firecracker-v1.15.1-x86_64`) /// * `expected_sha256` - If `Some`, verify the archive's SHA256 before extracting fn download_and_extract_tgz( url: &str, - entry_name: &str, - dest: &Path, + wanted: &[(String, PathBuf)], expected_sha256: Option<&str>, ) -> Result<(), String> { let response = ureq::get(url) @@ -392,6 +421,7 @@ fn download_and_extract_tgz( let gz = flate2::read::GzDecoder::new(archive_bytes.as_slice()); let mut archive = tar::Archive::new(gz); + let mut remaining = wanted.len(); for entry in archive .entries() .map_err(|e| format!("Failed to read tar entries: {e}"))? @@ -400,19 +430,32 @@ fn download_and_extract_tgz( let path = entry .path() .map_err(|e| format!("Failed to read entry path: {e}"))?; - - if path.to_string_lossy() == entry_name { - let mut bytes = Vec::new(); - entry - .read_to_end(&mut bytes) - .map_err(|e| format!("Failed to read entry data: {e}"))?; - fs::write(dest, &bytes) - .map_err(|e| format!("Failed to write to {}: {e}", dest.display()))?; + let path = path.to_string_lossy().into_owned(); + + let Some((_, dest)) = wanted.iter().find(|(name, _)| *name == path) else { + continue; + }; + + let mut bytes = Vec::new(); + entry + .read_to_end(&mut bytes) + .map_err(|e| format!("Failed to read entry data: {e}"))?; + fs::write(dest, &bytes) + .map_err(|e| format!("Failed to write to {}: {e}", dest.display()))?; + eprintln!("Extracted '{path}' to: {}", dest.display()); + remaining -= 1; + if remaining == 0 { return Ok(()); } } - Err(format!("Entry '{entry_name}' not found in archive")) + let missing = wanted + .iter() + .filter(|(_, dest)| !dest.exists()) + .map(|(name, _)| name.as_str()) + .collect::>() + .join(", "); + Err(format!("Entries not found in archive: {missing}")) } // --------------------------------------------------------------------------- @@ -489,7 +532,7 @@ pub const {name_upper}_BUNDLED: bool = true; fn generate_stub_modules() { let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set")); - for name in &["init", "firecracker", "kernel"] { + for name in &["init", "firecracker", "jailer", "kernel"] { generate_stub_module(name, &out_dir); } } diff --git a/plus/bencher_runner/src/jailer_bin.rs b/plus/bencher_runner/src/jailer_bin.rs new file mode 100644 index 000000000..9044c684d --- /dev/null +++ b/plus/bencher_runner/src/jailer_bin.rs @@ -0,0 +1,52 @@ +//! Bundled Firecracker jailer binary support. +//! +//! This module provides access to the bundled `jailer` binary, which confines +//! the Firecracker VMM to a chroot under an unprivileged uid. +//! +//! - In **release** builds: The binary is embedded directly in bencher-runner. +//! - In **debug** builds: The binary is loaded from disk (downloaded by build.rs). +//! +//! The jailer ships in the same release archive as Firecracker, so both are +//! bundled from a single download under a single hash. Extracting both per job +//! keeps the VMM and its jailer at the same version across a runner self-update. +//! +//! # Example +//! +//! ```ignore +//! use bencher_runner::jailer_bin::{write_jailer_to_file, JAILER_BUNDLED}; +//! +//! if JAILER_BUNDLED { +//! write_jailer_to_file("/tmp/jailer".as_ref())?; +//! } +//! ``` + +use std::io; + +use camino::Utf8Path; + +// Include the generated jailer module +include!(concat!(env!("OUT_DIR"), "/jailer_generated.rs")); + +/// Write the bundled jailer binary to a file. +/// +/// The file is written with executable permissions (0o755). +/// +/// # Arguments +/// +/// * `path` - The destination path for the jailer binary +/// +/// # Errors +/// +/// Returns an error if the file cannot be written. +pub fn write_jailer_to_file(path: &Utf8Path) -> io::Result<()> { + use std::os::unix::fs::PermissionsExt as _; + + std::fs::write(path, jailer_bytes())?; + + // Make it executable + let mut perms = std::fs::metadata(path)?.permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(path, perms)?; + + Ok(()) +} diff --git a/plus/bencher_runner/src/lib.rs b/plus/bencher_runner/src/lib.rs index c9a95a2c5..7b70265fc 100644 --- a/plus/bencher_runner/src/lib.rs +++ b/plus/bencher_runner/src/lib.rs @@ -33,6 +33,8 @@ pub mod init; #[cfg(feature = "plus")] pub mod jail; #[cfg(all(feature = "plus", target_os = "linux"))] +pub mod jailer_bin; +#[cfg(all(feature = "plus", target_os = "linux"))] pub mod kernel; #[cfg(feature = "plus")] mod local; From a8b80ce963aa1b4c1f73ae561154350df11bcb68 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Mon, 27 Jul 2026 22:48:26 +0000 Subject: [PATCH 02/91] Add a runner state directory, jail sweep, and empty network namespace The jail needs somewhere to live that outlives a single job: every per-job directory today is a tempfile::TempDir, and the chroot base, the sweep, and the network namespace handle all need a persistent location. --state-dir defaults to /var/lib/bencher-runner and is created at mode 0700 owned by root, since it holds every job's chroot and therefore the guest rootfs. Both entry points that can reach the VM executor call one idempotent prepare_host(): the daemon has a startup hook and the one-shot CLI does not, so the work lives in the shared function rather than in daemon startup. The sweep reclaims chroots left by a runner that exited without unwinding. Jobs run serially, so anything found is stale by construction, and the runner disappears without unwinding in several ordinary ways: SIGKILL, a crash, and the exec in a self-update. Drop runs in none of them, and each leftover chroot holds a copy of the VMM binary and a full rootfs image. The network namespace is for the VMM process, not the guest. A compromised VMM with host network access can exfiltrate; an empty namespace removes that reach. vsock is unaffected, since its host side is filesystem-scoped Unix domain sockets. The namespace is unshared on a dedicated thread rather than in the runner: namespaces are per-task, so only that thread moves, and the bind mount pins the namespace once the thread exits. /proc/thread-self is required there, because /proc/self resolves through the thread group leader and would pin the host network instead. --- plus/bencher_runner/Cargo.toml | 8 +- plus/bencher_runner/src/config.rs | 19 ++ plus/bencher_runner/src/error.rs | 35 +++ plus/bencher_runner/src/jail/mod.rs | 59 ++++- plus/bencher_runner/src/jail/netns.rs | 157 ++++++++++++ plus/bencher_runner/src/jail/state.rs | 225 ++++++++++++++++++ plus/bencher_runner/src/lib.rs | 2 +- plus/bencher_runner/src/run.rs | 10 + plus/bencher_runner/src/up/job.rs | 1 + plus/bencher_runner/src/up/mod.rs | 9 + .../docs-reference/runner/de/runner-run.mdx | 8 + .../docs-reference/runner/de/runner-up.mdx | 8 + .../docs-reference/runner/en/runner-run.mdx | 8 + .../docs-reference/runner/en/runner-up.mdx | 8 + .../docs-reference/runner/es/runner-run.mdx | 8 + .../docs-reference/runner/es/runner-up.mdx | 8 + .../docs-reference/runner/fr/runner-run.mdx | 8 + .../docs-reference/runner/fr/runner-up.mdx | 8 + .../docs-reference/runner/ja/runner-run.mdx | 8 + .../docs-reference/runner/ja/runner-up.mdx | 8 + .../docs-reference/runner/ko/runner-run.mdx | 8 + .../docs-reference/runner/ko/runner-up.mdx | 8 + .../docs-reference/runner/pt/runner-run.mdx | 8 + .../docs-reference/runner/pt/runner-up.mdx | 8 + .../docs-reference/runner/ru/runner-run.mdx | 8 + .../docs-reference/runner/ru/runner-up.mdx | 8 + .../docs-reference/runner/zh/runner-run.mdx | 8 + .../docs-reference/runner/zh/runner-up.mdx | 8 + services/runner/src/parser/mod.rs | 4 + services/runner/src/parser/up.rs | 5 + services/runner/src/runner/run.rs | 1 + services/runner/src/runner/up.rs | 1 + 32 files changed, 675 insertions(+), 5 deletions(-) create mode 100644 plus/bencher_runner/src/jail/netns.rs create mode 100644 plus/bencher_runner/src/jail/state.rs diff --git a/plus/bencher_runner/Cargo.toml b/plus/bencher_runner/Cargo.toml index 271fe86f5..7be174554 100644 --- a/plus/bencher_runner/Cargo.toml +++ b/plus/bencher_runner/Cargo.toml @@ -60,7 +60,13 @@ libc = { workspace = true } # Linux-only dependencies [target.'cfg(target_os = "linux")'.dependencies] -nix = { workspace = true, features = ["feature", "poll", "sched", "signal"] } +nix = { workspace = true, features = [ + "feature", + "mount", + "poll", + "sched", + "signal", +] } [build-dependencies] flate2 = { workspace = true } diff --git a/plus/bencher_runner/src/config.rs b/plus/bencher_runner/src/config.rs index 9a445747a..ab5396e17 100644 --- a/plus/bencher_runner/src/config.rs +++ b/plus/bencher_runner/src/config.rs @@ -151,6 +151,13 @@ pub struct Config { /// directly on the host (non-sandboxed mode, any platform). #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox: Option, + + /// The runner's persistent state directory. + /// + /// The jail chroot for each sandboxed run is built under this directory. + /// This field is not serialized. + #[serde(skip, default = "default_state_dir")] + pub state_dir: Utf8PathBuf, } fn default_vcpus() -> Cpu { @@ -222,6 +229,10 @@ fn default_grace_period() -> GracePeriod { GracePeriod::MIN } +fn default_state_dir() -> Utf8PathBuf { + Utf8PathBuf::from(crate::jail::DEFAULT_STATE_DIR) +} + impl Config { /// Create a new configuration with the bundled kernel. /// @@ -255,6 +266,7 @@ impl Config { cpu_layout: None, sandbox_log_level: SandboxLogLevel::default(), sandbox: None, + state_dir: default_state_dir(), } } @@ -445,6 +457,13 @@ impl Config { self } + /// Set the runner's persistent state directory. + #[must_use] + pub fn with_state_dir(mut self, state_dir: Utf8PathBuf) -> Self { + self.state_dir = state_dir; + self + } + /// Set the CPU layout for core isolation. /// /// When set, the Firecracker process will be pinned to benchmark cores diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index 94072dc32..07726cf5e 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -58,6 +58,41 @@ pub enum JailError { #[error("Cpuset partition mode '{mode}' rejected by the kernel: {state}")] PartitionInvalid { mode: String, state: String }, + + #[error("Failed to create runner state directory {path}: {source}")] + CreateStateDir { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to create network namespace directory {path}: {source}")] + NetnsDir { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to create network namespace handle {path}: {source}")] + NetnsHandle { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to unshare the network namespace: {0}")] + Unshare(#[source] nix::Error), + + #[cfg(target_os = "linux")] + #[error("Failed to bind the network namespace handle {path}: {source}")] + BindNetns { + path: Utf8PathBuf, + source: nix::Error, + }, + + #[cfg(target_os = "linux")] + #[error("The network namespace thread panicked")] + NetnsThread, } #[derive(Debug, Error)] diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 558c03a84..4902a80ab 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -1,18 +1,71 @@ -//! Resource management for Firecracker microVMs. +//! Confinement for Firecracker microVMs. //! -//! This module provides cgroup-based resource limits -//! for controlling Firecracker microVM processes. +//! Managed runners execute arbitrary code submitted by anyone, so the VMM must +//! not inherit the runner's root. This module owns everything that confines it: +//! the persistent state directory the chroots are built under, the empty +//! network namespace the VMM joins, and the cgroup that both places it on the +//! benchmark cores and bounds its resources. #[cfg(target_os = "linux")] mod cgroup; +#[cfg(target_os = "linux")] +pub mod netns; +#[cfg(target_os = "linux")] +pub mod state; #[cfg(target_os = "linux")] pub use cgroup::CgroupManager; #[cfg(target_os = "linux")] pub(crate) use cgroup::{BENCHER_CGROUP_BASE, effective_mems}; +#[cfg(target_os = "linux")] +pub use state::StateDir; use serde::{Deserialize, Serialize}; +/// Default location of the runner's persistent state directory. +pub const DEFAULT_STATE_DIR: &str = "/var/lib/bencher-runner"; + +/// The unprivileged uid the jailed Firecracker VMM runs as. +/// +/// One dedicated id, not one per job: jobs run serially and each gets a fresh +/// chroot that is swept, so a per-job allocator adds a scheme without closing +/// a live vector. The value sits in the gap between the ids `systemd-homed` +/// claims and the `DynamicUser` range (61184-65519), and well clear of both +/// the regular user range and `nobody` (65534), so it is unlikely to collide +/// with an account that owns anything on the host. No passwd entry is needed: +/// the jailer sets the numeric id directly. +pub const JAIL_UID: u32 = 60613; + +/// The unprivileged gid the jailed Firecracker VMM runs as. +/// +/// See [`JAIL_UID`]. +pub const JAIL_GID: u32 = 60613; + +/// Prepare the host for jailed execution. +/// +/// Idempotent, and called from every entry point that can reach the VM +/// executor: the `up` daemon has a startup hook, the one-shot `run` CLI does +/// not, so the work lives here rather than in daemon startup. +/// +/// Failure is fatal. Untrusted code never runs with silently degraded +/// confinement, so a host that cannot be prepared does not execute a job. +#[cfg(target_os = "linux")] +pub fn prepare_host(state_dir: &camino::Utf8Path) -> Result<(), crate::error::JailError> { + let state = StateDir::new(state_dir.to_owned()); + state.create()?; + state::sweep_jails(&state.jail_parent()); + netns::ensure()?; + Ok(()) +} + +/// Prepare the host for jailed execution. +/// +/// The jail is Linux-only, as is the VM executor it protects. +#[cfg(not(target_os = "linux"))] +pub fn prepare_host(_state_dir: &camino::Utf8Path) -> Result<(), crate::error::JailError> { + Ok(()) +} + /// Resource limits for the Firecracker microVM process. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ResourceLimits { diff --git a/plus/bencher_runner/src/jail/netns.rs b/plus/bencher_runner/src/jail/netns.rs new file mode 100644 index 000000000..c98aa4ded --- /dev/null +++ b/plus/bencher_runner/src/jail/netns.rs @@ -0,0 +1,157 @@ +//! The empty network namespace the jailed VMM joins. +//! +//! The guest has no network device and never will; this namespace is for the +//! VMM process itself. A compromised Firecracker with host network access can +//! exfiltrate, and an empty namespace removes that reach. The vsock transport +//! is unaffected: its host side is filesystem-scoped Unix domain sockets, not +//! network-namespace-scoped. + +use std::fs; +use std::os::unix::fs::MetadataExt as _; + +use camino::{Utf8Path, Utf8PathBuf}; +use nix::mount::{MntFlags, MsFlags, mount, umount2}; +use nix::sched::{CloneFlags, unshare}; + +use crate::error::JailError; + +/// Directory holding named network namespace handles. +/// +/// This follows the `ip netns` convention (iproute2's `NETNS_RUN_DIR`), so the +/// runner's namespace shows up in `ip netns list` for operators. `/run` is a +/// tmpfs, so handles do not survive a reboot, which is exactly right for a +/// handle onto a kernel object. +const NETNS_DIR: &str = "/run/netns"; + +/// Name of the empty network namespace the jailed VMM joins. +const NETNS_NAME: &str = "bencher-jail"; + +/// The runner's own network namespace, used as the reference for deciding +/// whether a handle is a live namespace distinct from the host's. +const SELF_NETNS: &str = "/proc/self/ns/net"; + +/// The calling *thread's* network namespace. +/// +/// `/proc/self` resolves through the thread group leader, so it must not be +/// used from the namespace-creating thread: it would name the runner's own +/// namespace and pin the host network instead of the new one. +const THREAD_NETNS: &str = "/proc/thread-self/ns/net"; + +/// The path of the network namespace handle. +#[must_use] +pub fn handle_path() -> Utf8PathBuf { + Utf8Path::new(NETNS_DIR).join(NETNS_NAME) +} + +/// Ensure the empty network namespace exists, returning its handle path. +/// +/// Idempotent: a handle that is already a live namespace distinct from the +/// runner's own is reused. Anything else at the path (a leftover placeholder +/// file, or a handle whose mount is gone) is cleared and recreated. +pub fn ensure() -> Result { + let handle = handle_path(); + + fs::create_dir_all(NETNS_DIR).map_err(|e| JailError::NetnsDir { + path: Utf8PathBuf::from(NETNS_DIR), + source: e, + })?; + + if is_live_netns(&handle) { + return Ok(handle); + } + + // Clear whatever is at the path. A bind mount over a file does not + // report EBUSY, so mounts would otherwise stack up silently. + let _detached = umount2(handle.as_std_path(), MntFlags::MNT_DETACH); + drop(fs::remove_file(&handle)); + + // The bind mount needs a regular file to land on. + fs::File::create(&handle).map_err(|e| JailError::NetnsHandle { + path: handle.clone(), + source: e, + })?; + + if let Err(e) = create(&handle) { + drop(fs::remove_file(&handle)); + return Err(e); + } + + Ok(handle) +} + +/// Whether `handle` is a live network namespace other than the runner's own. +/// +/// Every namespace inode lives on the single kernel `nsfs`, so sharing a +/// device with a known namespace proves the handle is one, and a differing +/// inode proves it is not the host namespace the runner itself is in. A +/// leftover placeholder file sits on the `/run` tmpfs and fails the device +/// check. +fn is_live_netns(handle: &Utf8Path) -> bool { + let (Ok(own), Ok(candidate)) = (fs::metadata(SELF_NETNS), fs::metadata(handle)) else { + return false; + }; + own.dev() == candidate.dev() && own.ino() != candidate.ino() +} + +/// Create the namespace and bind its handle into place. +/// +/// The namespace is unshared on a dedicated thread rather than in the runner +/// itself. Network namespaces are per-task, so only this thread moves and the +/// runner stays on the host network; the bind mount then holds a reference +/// that keeps the namespace alive once the thread exits. The thread is not +/// reused for anything else, precisely because it never returns to the host +/// namespace. +fn create(handle: &Utf8Path) -> Result<(), JailError> { + let target = handle.to_owned(); + std::thread::spawn(move || -> Result<(), JailError> { + unshare(CloneFlags::CLONE_NEWNET).map_err(JailError::Unshare)?; + mount( + Some(THREAD_NETNS), + target.as_std_path(), + None::<&str>, + MsFlags::MS_BIND, + None::<&str>, + ) + .map_err(|e| JailError::BindNetns { + path: target.clone(), + source: e, + }) + }) + .join() + .map_err(|_panic| JailError::NetnsThread)? +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn handle_follows_the_ip_netns_convention() { + assert_eq!(handle_path(), "/run/netns/bencher-jail"); + } + + #[test] + fn a_plain_file_is_not_a_live_netns() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let path = root.join("net"); + fs::write(&path, b"").unwrap(); + + assert!(!is_live_netns(&path)); + } + + #[test] + fn a_missing_handle_is_not_a_live_netns() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + assert!(!is_live_netns(&root.join("absent"))); + } + + #[test] + fn the_runners_own_namespace_is_not_a_distinct_netns() { + // The handle must be a namespace *other* than the one the runner is + // in, or the VMM would keep host network reach. + assert!(!is_live_netns(Utf8Path::new(SELF_NETNS))); + } +} diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs new file mode 100644 index 000000000..9f02479ef --- /dev/null +++ b/plus/bencher_runner/src/jail/state.rs @@ -0,0 +1,225 @@ +//! The runner's persistent state directory. +//! +//! Everything the jail needs that must outlive a single job hangs off one +//! directory: the chroot base the jailer builds under, and the sweep that +//! reclaims chroots left behind by a runner that exited without unwinding. + +#![expect(clippy::print_stderr, reason = "host preparation prints diagnostics")] + +use std::fs; +use std::os::unix::fs::PermissionsExt as _; + +use camino::{Utf8Path, Utf8PathBuf}; + +use crate::error::JailError; + +/// Subdirectory of the state directory used as the jailer's chroot base. +const CHROOT_BASE: &str = "jail"; + +/// The `--exec-file` base name the jailer derives the chroot layout from. +/// +/// The jailer builds `///root`, so the +/// staged Firecracker binary must be named exactly this for the runner and +/// the jailer to agree on where the chroot lives. +pub(crate) const EXEC_FILE_NAME: &str = "firecracker"; + +/// The runner's persistent state directory. +/// +/// Created at mode 0700 owned by root: it holds every job's chroot, which +/// contains the guest rootfs and the copied VMM binary. +#[derive(Debug, Clone)] +pub struct StateDir { + root: Utf8PathBuf, +} + +impl StateDir { + /// Create a handle for the state directory rooted at `root`. + #[must_use] + pub fn new(root: Utf8PathBuf) -> Self { + Self { root } + } + + /// The state directory itself. + #[must_use] + pub fn path(&self) -> &Utf8Path { + &self.root + } + + /// The jailer's `--chroot-base-dir`. + #[must_use] + pub fn chroot_base(&self) -> Utf8PathBuf { + self.root.join(CHROOT_BASE) + } + + /// The directory holding one subdirectory per jailed VMM. + /// + /// This is the level the sweep operates on. + #[must_use] + pub fn jail_parent(&self) -> Utf8PathBuf { + self.chroot_base().join(EXEC_FILE_NAME) + } + + /// The jail directory for a VM, the tree teardown removes. + #[must_use] + pub fn jail_dir(&self, vm_id: &str) -> Utf8PathBuf { + self.jail_parent().join(vm_id) + } + + /// The chroot root for a VM, which becomes `/` inside the jail. + #[must_use] + pub fn jail_root(&self, vm_id: &str) -> Utf8PathBuf { + self.jail_dir(vm_id).join("root") + } + + /// Create the state directory tree at mode 0700. + /// + /// Idempotent. The mode is applied on every call so a directory created + /// with a laxer mode by an older runner is tightened on upgrade. + pub fn create(&self) -> Result<(), JailError> { + for dir in [&self.root, &self.chroot_base(), &self.jail_parent()] { + fs::create_dir_all(dir).map_err(|e| JailError::CreateStateDir { + path: dir.clone(), + source: e, + })?; + fs::set_permissions(dir, fs::Permissions::from_mode(0o700)).map_err(|e| { + JailError::CreateStateDir { + path: dir.clone(), + source: e, + } + })?; + } + Ok(()) + } +} + +/// Remove every jail directory under `jail_parent`, returning how many were +/// reclaimed. +/// +/// Jobs run serially, so anything found here is stale by construction. The +/// runner disappears without unwinding in several ordinary ways, including +/// SIGKILL, a crash, and the `exec` in a self-update, and `Drop` runs in +/// none of them. Each leftover chroot holds a copy of the VMM binary and a +/// full guest rootfs image, so leaving them is not an option. +/// +/// Non-directory entries are left alone: the jailer only ever creates +/// directories here, so anything else was put there by someone else. +pub fn sweep_jails(jail_parent: &Utf8Path) -> usize { + let Ok(entries) = fs::read_dir(jail_parent) else { + return 0; + }; + + let mut swept = 0; + for entry in entries.flatten() { + let path = entry.path(); + if !entry.file_type().is_ok_and(|file_type| file_type.is_dir()) { + continue; + } + match fs::remove_dir_all(&path) { + Ok(()) => swept += 1, + Err(e) => eprintln!( + "Warning: failed to sweep stale jail {}: {e}", + path.display() + ), + } + } + swept +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_root() -> (tempfile::TempDir, Utf8PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + (dir, root) + } + + #[test] + fn jail_layout_matches_jailer_template() { + let state = StateDir::new(Utf8PathBuf::from("/var/lib/bencher-runner")); + assert_eq!(state.chroot_base(), "/var/lib/bencher-runner/jail"); + assert_eq!( + state.jail_parent(), + "/var/lib/bencher-runner/jail/firecracker" + ); + assert_eq!( + state.jail_dir("abc"), + "/var/lib/bencher-runner/jail/firecracker/abc" + ); + // ///root + assert_eq!( + state.jail_root("abc"), + state + .chroot_base() + .join(EXEC_FILE_NAME) + .join("abc") + .join("root") + ); + } + + #[test] + fn create_is_idempotent_and_private() { + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")); + + state.create().unwrap(); + state.create().unwrap(); + + for dir in [state.path(), &state.chroot_base(), &state.jail_parent()] { + let mode = fs::metadata(dir).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o700, "{dir} should be private"); + } + } + + #[test] + fn create_tightens_a_lax_directory() { + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")); + fs::create_dir_all(state.path()).unwrap(); + fs::set_permissions(state.path(), fs::Permissions::from_mode(0o755)).unwrap(); + + state.create().unwrap(); + + let mode = fs::metadata(state.path()).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o700); + } + + #[test] + fn sweep_removes_stale_jails() { + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")); + state.create().unwrap(); + + // Two stale jails, one with a nested chroot tree. + fs::create_dir_all(state.jail_root("one")).unwrap(); + fs::write(state.jail_root("one").join("rootfs.ext4"), b"stale").unwrap(); + fs::create_dir_all(state.jail_dir("two")).unwrap(); + + assert_eq!(sweep_jails(&state.jail_parent()), 2); + assert!(!state.jail_dir("one").exists()); + assert!(!state.jail_dir("two").exists()); + assert!(state.jail_parent().exists()); + } + + #[test] + fn sweep_leaves_unrelated_entries_alone() { + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")); + state.create().unwrap(); + + let note = state.jail_parent().join("NOTES.txt"); + fs::write(¬e, b"not a jail").unwrap(); + fs::create_dir_all(state.jail_dir("stale")).unwrap(); + + assert_eq!(sweep_jails(&state.jail_parent()), 1); + assert!(!state.jail_dir("stale").exists()); + assert!(note.exists(), "non-directory entries are not the sweep's"); + } + + #[test] + fn sweep_missing_parent_is_zero() { + let (_dir, root) = temp_root(); + assert_eq!(sweep_jails(&root.join("nope")), 0); + } +} diff --git a/plus/bencher_runner/src/lib.rs b/plus/bencher_runner/src/lib.rs index 7b70265fc..6c96d3232 100644 --- a/plus/bencher_runner/src/lib.rs +++ b/plus/bencher_runner/src/lib.rs @@ -62,7 +62,7 @@ pub use config::Config; #[cfg(feature = "plus")] pub use error::{ConfigError, ExecutionError, JailError, RunnerError}; #[cfg(feature = "plus")] -pub use jail::ResourceLimits; +pub use jail::{DEFAULT_STATE_DIR, ResourceLimits}; #[cfg(feature = "plus")] pub use log_level::SandboxLogLevel; #[cfg(feature = "plus")] diff --git a/plus/bencher_runner/src/run.rs b/plus/bencher_runner/src/run.rs index a9405749e..f50f204d1 100644 --- a/plus/bencher_runner/src/run.rs +++ b/plus/bencher_runner/src/run.rs @@ -71,6 +71,8 @@ pub struct RunArgs { pub sandbox_log_level: crate::SandboxLogLevel, /// Sandbox mode for benchmark execution. pub sandbox: Option, + /// The runner's persistent state directory. + pub state_dir: Utf8PathBuf, } /// Build a `Config` from CLI `RunArgs`. @@ -121,6 +123,7 @@ fn build_config_from_run_args(args: &RunArgs) -> Result Result<(), RunnerError> { let mut config = build_config_from_run_args(args)?; + // Prepare the host only for the sandboxed path: the one-shot CLI can + // execute on the host without a jail, and that path has no business + // requiring the runner's state directory. + if config.sandbox.is_some() { + crate::jail::prepare_host(&config.state_dir)?; + } + // Detect the CPU layout after tuning (disabling SMT changes the core // count), steer kernel work off the benchmark cores, and pin the run // to them. Mirrors the `runner up` path. Core pinning is core runner diff --git a/plus/bencher_runner/src/up/job.rs b/plus/bencher_runner/src/up/job.rs index 2f49dae2e..eafac983f 100644 --- a/plus/bencher_runner/src/up/job.rs +++ b/plus/bencher_runner/src/up/job.rs @@ -543,6 +543,7 @@ mod tests { no_auto_update: false, update_channel: bencher_valid::UpdateChannel::default(), max_download_size: None, + state_dir: Utf8PathBuf::from(crate::jail::DEFAULT_STATE_DIR), } } diff --git a/plus/bencher_runner/src/up/mod.rs b/plus/bencher_runner/src/up/mod.rs index 212f97227..0915ac203 100644 --- a/plus/bencher_runner/src/up/mod.rs +++ b/plus/bencher_runner/src/up/mod.rs @@ -67,6 +67,8 @@ pub struct UpConfig { pub update_channel: bencher_valid::UpdateChannel, /// Maximum download size in bytes for self-update binaries. pub max_download_size: Option, + /// The runner's persistent state directory. + pub state_dir: camino::Utf8PathBuf, } pub struct Up { @@ -104,6 +106,13 @@ impl Up { // Warn about host conditions that limit benchmark accuracy (Linux only) preflight::print_host_warnings(); + // Create the state directory, sweep chroots left behind by a runner + // that exited without unwinding, and ensure the empty network + // namespace. The daemon claims sandboxed jobs, so this is required + // before the first one arrives, not on demand. + crate::jail::prepare_host(&self.config.state_dir).map_err(crate::RunnerError::from)?; + println!(" State directory: {}", self.config.state_dir); + // Serialize host-global tuning across runner processes. Declared // before the guard so the lock releases only after restore completes. let host_lock = crate::tuning::HostTuningLock::acquire(); diff --git a/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx index e8ce92046..3a9928cfc 100644 --- a/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx @@ -32,6 +32,14 @@ Lassen Sie es für die Ausführung auf dem Host ohne Sandbox weg. Das Ausführungs-Timeout in Sekunden. Standardmäßig wird `300` verwendet. +### `--state-dir ` + +Das persistente Zustandsverzeichnis für den Runner. +Das Jail, das die Sandbox einschließt, wird unter diesem Verzeichnis erstellt, +und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor der Ausführung eines Jobs daraus entfernt. +Standardmäßig wird `/var/lib/bencher-runner` verwendet. +Kann auch mit der Umgebungsvariable `BENCHER_STATE_DIR` gesetzt werden. + ### `--iter ` Die Anzahl der auszuführenden Benchmark-Iterationen. diff --git a/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx index a27169f34..a1cbe4e07 100644 --- a/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx @@ -35,6 +35,14 @@ Kann auch mit der Umgebungsvariable `BENCHER_RUNNER_KEY` gesetzt werden. Das Long-Poll-Timeout in Sekunden beim Warten auf einen Job, zwischen `1` und `900`. Standardmäßig wird `55` verwendet. +### `--state-dir ` + +Das persistente Zustandsverzeichnis für den Runner. +Das Jail, das die Sandbox einschließt, wird unter diesem Verzeichnis erstellt, +und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor der Ausführung eines Jobs daraus entfernt. +Standardmäßig wird `/var/lib/bencher-runner` verwendet. +Kann auch mit der Umgebungsvariable `BENCHER_STATE_DIR` gesetzt werden. + ### `--danger-allow-no-sandbox` Erlaubt das Ausführen von Jobs ohne [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx index 403d8c060..51bdf1b4f 100644 --- a/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx @@ -32,6 +32,14 @@ Omit for non-sandboxed host execution. The execution timeout in seconds. By default, `300` is used. +### `--state-dir ` + +The persistent state directory for the Runner. +The jail that confines the sandbox is built under this directory, +and jails left behind by an unclean exit are swept from it before a Job runs. +By default, `/var/lib/bencher-runner` is used. +Can also be set with the `BENCHER_STATE_DIR` environment variable. + ### `--iter ` The number of benchmark iterations to execute. diff --git a/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx index 21455118c..68bd2cf86 100644 --- a/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx @@ -35,6 +35,14 @@ Can also be set with the `BENCHER_RUNNER_KEY` environment variable. The long-poll timeout in seconds while waiting for a Job, between `1` and `900`. By default, `55` is used. +### `--state-dir ` + +The persistent state directory for the Runner. +The jail that confines the sandbox is built under this directory, +and jails left behind by an unclean exit are swept from it before a Job runs. +By default, `/var/lib/bencher-runner` is used. +Can also be set with the `BENCHER_STATE_DIR` environment variable. + ### `--danger-allow-no-sandbox` Allow executing Jobs without a [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx index 757bf8378..ea020e287 100644 --- a/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx @@ -32,6 +32,14 @@ Omítelo para ejecución en el host sin sandbox. El tiempo de espera de ejecución en segundos. Por defecto, se usa `300`. +### `--state-dir ` + +El directorio de estado persistente del Runner. +La jaula que confina el sandbox se crea dentro de este directorio, +y las jaulas que quedaron tras una salida no limpia se eliminan de él antes de ejecutar un Job. +Por defecto, se usa `/var/lib/bencher-runner`. +También se puede establecer con la variable de entorno `BENCHER_STATE_DIR`. + ### `--iter ` El número de iteraciones de benchmark a ejecutar. diff --git a/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx index 503e46964..b95cc5edb 100644 --- a/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx @@ -35,6 +35,14 @@ También se puede establecer con la variable de entorno `BENCHER_RUNNER_KEY`. El tiempo de espera del long-poll en segundos mientras se espera un Job, entre `1` y `900`. Por defecto, se usa `55`. +### `--state-dir ` + +El directorio de estado persistente del Runner. +La jaula que confina el sandbox se crea dentro de este directorio, +y las jaulas que quedaron tras una salida no limpia se eliminan de él antes de ejecutar un Job. +Por defecto, se usa `/var/lib/bencher-runner`. +También se puede establecer con la variable de entorno `BENCHER_STATE_DIR`. + ### `--danger-allow-no-sandbox` Permite ejecutar Jobs sin un [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx index d0d4e1c6f..51a4a912a 100644 --- a/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx @@ -32,6 +32,14 @@ Omettez pour une exécution sur l'hôte sans sandbox. Le délai d'attente d'exécution en secondes. Par défaut, `300` est utilisé. +### `--state-dir ` + +Le répertoire d'état persistant du Runner. +La prison qui confine le bac à sable est créée dans ce répertoire, +et les prisons laissées par un arrêt brutal en sont supprimées avant l'exécution d'un Job. +Par défaut, `/var/lib/bencher-runner` est utilisé. +Peut également être défini avec la variable d'environnement `BENCHER_STATE_DIR`. + ### `--iter ` Le nombre d'itérations de benchmark à exécuter. diff --git a/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx index bd5e6b4e3..3247b3f5c 100644 --- a/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx @@ -35,6 +35,14 @@ Peut aussi être défini avec la variable d'environnement `BENCHER_RUNNER_KEY`. Le délai d'attente du long-poll en secondes pendant l'attente d'un Job, entre `1` et `900`. Par défaut, `55` est utilisé. +### `--state-dir ` + +Le répertoire d'état persistant du Runner. +La prison qui confine le bac à sable est créée dans ce répertoire, +et les prisons laissées par un arrêt brutal en sont supprimées avant l'exécution d'un Job. +Par défaut, `/var/lib/bencher-runner` est utilisé. +Peut également être défini avec la variable d'environnement `BENCHER_STATE_DIR`. + ### `--danger-allow-no-sandbox` Autorise l'exécution de Jobs sans [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx index dbc8cc51e..99f2e23f2 100644 --- a/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx @@ -32,6 +32,14 @@ Firecracker microVM には `firecracker` を使用します (Linux のみ)。 実行タイムアウト (秒)。 デフォルトでは `300` が使用されます。 +### `--state-dir ` + +Runner の永続的な状態ディレクトリ。 +サンドボックスを閉じ込める jail はこのディレクトリの下に作成され、 +異常終了で残った jail は Job の実行前にここから一掃されます。 +デフォルトでは `/var/lib/bencher-runner` が使用されます。 +環境変数 `BENCHER_STATE_DIR` でも設定できます。 + ### `--iter ` 実行するベンチマークの反復回数。 diff --git a/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx index 37619064b..f37039cd1 100644 --- a/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx @@ -34,6 +34,14 @@ runner up [OPTIONS] Job を待機する際のロングポーリングのタイムアウト (秒)。`1` から `900` の範囲です。 デフォルトでは `55` が使用されます。 +### `--state-dir ` + +Runner の永続的な状態ディレクトリ。 +サンドボックスを閉じ込める jail はこのディレクトリの下に作成され、 +異常終了で残った jail は Job の実行前にここから一掃されます。 +デフォルトでは `/var/lib/bencher-runner` が使用されます。 +環境変数 `BENCHER_STATE_DIR` でも設定できます。 + ### `--danger-allow-no-sandbox` [Sandbox][sandbox] なしでの Job の実行を許可します。 diff --git a/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx index edd05be9c..c95316de5 100644 --- a/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx @@ -32,6 +32,14 @@ Firecracker microVM(Linux 전용)을 사용하려면 `firecracker`를 사용하 실행 타임아웃(초)입니다. 기본적으로 `300`이 사용됩니다. +### `--state-dir ` + +Runner의 영구 상태 디렉터리입니다. +샌드박스를 가두는 jail이 이 디렉터리 아래에 생성되며, +비정상 종료로 남겨진 jail은 Job 실행 전에 이곳에서 정리됩니다. +기본적으로 `/var/lib/bencher-runner`가 사용됩니다. +`BENCHER_STATE_DIR` 환경 변수로도 설정할 수 있습니다. + ### `--iter ` 실행할 벤치마크 반복 횟수입니다. diff --git a/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx index 1b6635605..ad18677b1 100644 --- a/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx @@ -35,6 +35,14 @@ Runner 인증 키(`bencher_runner_...`)입니다. Job을 기다리는 동안의 롱 폴(long-poll) 타임아웃(초)으로, `1`에서 `900` 사이입니다. 기본적으로 `55`가 사용됩니다. +### `--state-dir ` + +Runner의 영구 상태 디렉터리입니다. +샌드박스를 가두는 jail이 이 디렉터리 아래에 생성되며, +비정상 종료로 남겨진 jail은 Job 실행 전에 이곳에서 정리됩니다. +기본적으로 `/var/lib/bencher-runner`가 사용됩니다. +`BENCHER_STATE_DIR` 환경 변수로도 설정할 수 있습니다. + ### `--danger-allow-no-sandbox` [Sandbox][sandbox] 없이 Job 실행을 허용합니다. diff --git a/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx index adb578127..269798242 100644 --- a/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx @@ -32,6 +32,14 @@ Omita para execução no host sem sandbox. O timeout de execução em segundos. Por padrão, `300` é usado. +### `--state-dir ` + +O diretório de estado persistente do Runner. +A jail que confina o sandbox é criada sob este diretório, +e as jails deixadas por um encerramento não limpo são removidas dele antes de um Job ser executado. +Por padrão, `/var/lib/bencher-runner` é usado. +Também pode ser definido com a variável de ambiente `BENCHER_STATE_DIR`. + ### `--iter ` O número de iterações de benchmark a executar. diff --git a/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx index df5d1fe17..9ef519f8a 100644 --- a/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx @@ -35,6 +35,14 @@ Também pode ser definida com a variável de ambiente `BENCHER_RUNNER_KEY`. O timeout de long-poll em segundos enquanto aguarda um Job, entre `1` e `900`. Por padrão, `55` é usado. +### `--state-dir ` + +O diretório de estado persistente do Runner. +A jail que confina o sandbox é criada sob este diretório, +e as jails deixadas por um encerramento não limpo são removidas dele antes de um Job ser executado. +Por padrão, `/var/lib/bencher-runner` é usado. +Também pode ser definido com a variável de ambiente `BENCHER_STATE_DIR`. + ### `--danger-allow-no-sandbox` Permite executar Jobs sem um [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx index 6caceb1ba..936714ca3 100644 --- a/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx @@ -32,6 +32,14 @@ JWT-токен для аутентификации в реестре при за Тайм-аут выполнения в секундах. По умолчанию используется `300`. +### `--state-dir ` + +Постоянный каталог состояния Runner. +Изолирующая среда (jail), ограничивающая песочницу, создаётся внутри этого каталога, +а среды, оставшиеся после аварийного завершения, удаляются из него перед запуском Job. +По умолчанию используется `/var/lib/bencher-runner`. +Также может быть задан переменной окружения `BENCHER_STATE_DIR`. + ### `--iter ` Количество итераций бенчмарка для выполнения. diff --git a/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx index 90f62beb0..62f66e1e2 100644 --- a/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx @@ -35,6 +35,14 @@ UUID или slug Runner, от имени которого работать. Тайм-аут длинного опроса в секундах при ожидании Job, от `1` до `900`. По умолчанию используется `55`. +### `--state-dir ` + +Постоянный каталог состояния Runner. +Изолирующая среда (jail), ограничивающая песочницу, создаётся внутри этого каталога, +а среды, оставшиеся после аварийного завершения, удаляются из него перед запуском Job. +По умолчанию используется `/var/lib/bencher-runner`. +Также может быть задан переменной окружения `BENCHER_STATE_DIR`. + ### `--danger-allow-no-sandbox` Разрешить выполнение Job без [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx index 445d703f2..f54a7b90f 100644 --- a/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx @@ -32,6 +32,14 @@ runner run --image [OPTIONS] 执行超时（秒）。 默认使用 `300`。 +### `--state-dir ` + +Runner 的持久状态目录。 +限制沙箱的 jail 在该目录下创建， +非正常退出遗留的 jail 会在 Job 运行前从中清除。 +默认使用 `/var/lib/bencher-runner`。 +也可以通过 `BENCHER_STATE_DIR` 环境变量设置。 + ### `--iter ` 要执行的基准测试迭代次数。 diff --git a/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx index cfb283b8f..e16b8acb1 100644 --- a/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx @@ -34,6 +34,14 @@ runner up [OPTIONS] 等待 Job 时的长轮询超时（秒），介于 `1` 和 `900` 之间。 默认使用 `55`。 +### `--state-dir ` + +Runner 的持久状态目录。 +限制沙箱的 jail 在该目录下创建， +非正常退出遗留的 jail 会在 Job 运行前从中清除。 +默认使用 `/var/lib/bencher-runner`。 +也可以通过 `BENCHER_STATE_DIR` 环境变量设置。 + ### `--danger-allow-no-sandbox` 允许在没有 [Sandbox][sandbox] 的情况下执行 Job。 diff --git a/services/runner/src/parser/mod.rs b/services/runner/src/parser/mod.rs index 4641f88c9..6deecb3df 100644 --- a/services/runner/src/parser/mod.rs +++ b/services/runner/src/parser/mod.rs @@ -62,6 +62,10 @@ pub struct CliRun { #[arg(long, default_value = "300")] pub timeout: u64, + /// Persistent state directory for the runner. + #[arg(long, env = "BENCHER_STATE_DIR", default_value = bencher_runner::DEFAULT_STATE_DIR)] + pub state_dir: Utf8PathBuf, + /// Output file paths inside guest (may be repeated). #[arg(long)] pub output: Vec, diff --git a/services/runner/src/parser/up.rs b/services/runner/src/parser/up.rs index 8422adf47..e6d5f076d 100644 --- a/services/runner/src/parser/up.rs +++ b/services/runner/src/parser/up.rs @@ -1,4 +1,5 @@ use bencher_json::RunnerResourceId; +use camino::Utf8PathBuf; use clap::Parser; use super::CliTuning; @@ -22,6 +23,10 @@ pub struct CliUp { #[arg(long, default_value = "55", value_parser = clap::value_parser!(u32).range(1..=900))] pub poll_timeout: u32, + /// Persistent state directory for the runner. + #[arg(long, env = "BENCHER_STATE_DIR", default_value = bencher_runner::DEFAULT_STATE_DIR)] + pub state_dir: Utf8PathBuf, + #[command(flatten)] pub tuning: CliTuning, diff --git a/services/runner/src/runner/run.rs b/services/runner/src/runner/run.rs index 509865c6a..764bf4d09 100644 --- a/services/runner/src/runner/run.rs +++ b/services/runner/src/runner/run.rs @@ -54,6 +54,7 @@ impl TryFrom for Run { grace_period: task.grace_period, sandbox_log_level: task.sandbox_log_level, sandbox: task.sandbox, + state_dir: task.state_dir, }, }) } diff --git a/services/runner/src/runner/up.rs b/services/runner/src/runner/up.rs index 0407dfd51..55a655163 100644 --- a/services/runner/src/runner/up.rs +++ b/services/runner/src/runner/up.rs @@ -31,6 +31,7 @@ impl TryFrom for Up { no_auto_update: task.no_auto_update, update_channel: task.update_channel, max_download_size: task.max_download_size, + state_dir: task.state_dir, }, }) } From b3758538302060a8272483c16d6b5a9edc710c7f Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Mon, 27 Jul 2026 22:59:30 +0000 Subject: [PATCH 03/91] Confine the Firecracker VMM with the jailer The runner spawns the jailer instead of Firecracker. The jailer builds a chroot, creates /dev/kvm, drops to a dedicated unprivileged uid and gid, joins the empty network namespace, and execs Firecracker in place. Managed runners execute arbitrary code submitted by anyone, and until now a VMM escape landed as root on the runner host holding the runner key, every prior job's work directory, and write access to the self-updating runner binary. The VM id is minted before the job's artifacts, because the jail root is a function of it and rootfs.ext4 and vmlinux are now built directly inside the chroot rather than copied in. That is legal because the jailer uses create_dir_all for the chroot and does nothing if the path already exists. The artifacts leaving the workspace temp directory forfeits its RAII cleanup, so the jail guard takes over that responsibility and removes the tree on completion, timeout, cancellation, and every error return. Paths handed to Firecracker now resolve inside the chroot while the runner reaches the same files from outside, so the two views are separate types. Passing a host path where the API expects a chroot path is a compile error rather than a boot that hangs on a socket that never appears. Cgroup placement moves before exec. Membership is inherited across fork and survives execve, so a pre-opened cgroup.procs written from pre_exec places the child before it execs the jailer, and Firecracker inherits it through the jailer's own exec. This also fixes a second defect: the cpuset used to be applied after the VMM was already running, so Firecracker booted its API and touched memory on the wrong cores before being moved. No cgroup flags are passed to the jailer. The runner has to create, verify, read metrics from, and remove the cgroup, and the cpuset partition needs read-back verification because the kernel accepts the write and reports rejection inline. The jailer's write-once interface cannot provide that. Neither --daemonize nor --new-pid-ns is passed either: both make the jailer fork, which would break the pid identity the process management relies on. Confinement failures are fatal, so untrusted code never runs with silently degraded confinement. The cgroup keeps its existing degrade behavior at the edges, since a host that cannot isolate is a declared limitation, but when the cgroup does exist placement and verification are hard requirements: a cgroup that does not contain the VMM is a silent lie about where the benchmark ran. --- plus/bencher_runner/src/error.rs | 35 +++ plus/bencher_runner/src/firecracker/client.rs | 10 +- plus/bencher_runner/src/firecracker/config.rs | 14 +- plus/bencher_runner/src/firecracker/error.rs | 20 ++ plus/bencher_runner/src/firecracker/mod.rs | 117 +++++++--- .../bencher_runner/src/firecracker/process.rs | 150 ++++++++++--- plus/bencher_runner/src/firecracker/vsock.rs | 82 +++++-- plus/bencher_runner/src/jail/cgroup.rs | 76 ++++++- plus/bencher_runner/src/jail/chroot.rs | 143 ++++++++++++ plus/bencher_runner/src/jail/mod.rs | 8 + plus/bencher_runner/src/jail/paths.rs | 208 ++++++++++++++++++ plus/bencher_runner/src/vm.rs | 144 +++++++++--- 12 files changed, 880 insertions(+), 127 deletions(-) create mode 100644 plus/bencher_runner/src/jail/chroot.rs create mode 100644 plus/bencher_runner/src/jail/paths.rs diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index 07726cf5e..520143dfc 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -93,6 +93,34 @@ pub enum JailError { #[cfg(target_os = "linux")] #[error("The network namespace thread panicked")] NetnsThread, + + #[cfg(target_os = "linux")] + #[error("Failed to create jail chroot {path}: {source}")] + CreateJail { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to hand {path} to the jail uid and gid: {source}")] + ChownJail { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to open {path} for cgroup placement: {source}")] + OpenCgroupProcs { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to read cgroup file {path}: {source}")] + ReadCgroup { + path: Utf8PathBuf, + source: std::io::Error, + }, } #[derive(Debug, Error)] @@ -112,6 +140,13 @@ pub enum ConfigError { #[error("Binary not found: {name}. {hint}")] BinaryNotFound { name: String, hint: String }, + #[error("Failed to copy {src} to {dest}: {source}")] + CopyFile { + src: Utf8PathBuf, + dest: Utf8PathBuf, + source: std::io::Error, + }, + #[error("Failed to copy init binary from {src} to {dest}: {source}")] CopyInit { src: Utf8PathBuf, diff --git a/plus/bencher_runner/src/firecracker/client.rs b/plus/bencher_runner/src/firecracker/client.rs index e98590e83..a9f04c838 100644 --- a/plus/bencher_runner/src/firecracker/client.rs +++ b/plus/bencher_runner/src/firecracker/client.rs @@ -13,6 +13,7 @@ use camino::Utf8Path; use crate::firecracker::config::{Action, BootSource, Drive, MachineConfig, VsockConfig}; use crate::firecracker::error::FirecrackerError; +use crate::jail::HostPath; /// Client for the Firecracker REST API. pub struct FirecrackerClient { @@ -20,10 +21,13 @@ pub struct FirecrackerClient { } impl FirecrackerClient { - /// Create a new client for the given API socket path. - pub fn new(socket_path: &str) -> Self { + /// Create a new client for the API socket. + /// + /// The runner reaches the socket from outside the chroot, so this is the + /// host view; the jailed VMM binds the chroot view of the same file. + pub fn new(socket_path: &HostPath) -> Self { Self { - socket_path: socket_path.to_owned(), + socket_path: socket_path.as_str().to_owned(), } } diff --git a/plus/bencher_runner/src/firecracker/config.rs b/plus/bencher_runner/src/firecracker/config.rs index 8e5a92f00..cfa6c75e9 100644 --- a/plus/bencher_runner/src/firecracker/config.rs +++ b/plus/bencher_runner/src/firecracker/config.rs @@ -2,6 +2,8 @@ use serde::Serialize; +use crate::jail::ChrootPath; + /// Machine configuration for Firecracker. #[derive(Debug, Serialize)] pub struct MachineConfig { @@ -16,8 +18,8 @@ pub struct MachineConfig { /// Boot source configuration. #[derive(Debug, Serialize)] pub struct BootSource { - /// Path to the kernel image on the host. - pub kernel_image_path: String, + /// Path to the kernel image, as the jailed VMM sees it. + pub kernel_image_path: ChrootPath, /// Kernel boot arguments. pub boot_args: String, } @@ -27,8 +29,8 @@ pub struct BootSource { pub struct Drive { /// Unique drive identifier. pub drive_id: String, - /// Path to the disk image on the host. - pub path_on_host: String, + /// Path to the disk image, as the jailed VMM sees it. + pub path_on_host: ChrootPath, /// Whether this is the root device. pub is_root_device: bool, /// Whether the drive is read-only. @@ -40,8 +42,8 @@ pub struct Drive { pub struct VsockConfig { /// Guest CID (must be >= 3 for Firecracker). pub guest_cid: u32, - /// Path to the Unix domain socket on the host. - pub uds_path: String, + /// Path to the Unix domain socket, as the jailed VMM sees it. + pub uds_path: ChrootPath, } /// VM action request. diff --git a/plus/bencher_runner/src/firecracker/error.rs b/plus/bencher_runner/src/firecracker/error.rs index 8801884a5..63282c411 100644 --- a/plus/bencher_runner/src/firecracker/error.rs +++ b/plus/bencher_runner/src/firecracker/error.rs @@ -37,4 +37,24 @@ pub enum FirecrackerError { /// Job was cancelled. #[error("Job cancelled")] Cancelled, + + /// The VMM could not be placed in, or verified against, its cgroup. + #[error("Cgroup placement failed: {0}")] + CgroupPlacement(#[source] crate::error::JailError), + + /// The cgroup exists but the VMM is not in it. + /// + /// Fatal: a cgroup that does not contain the VMM is a silent lie about + /// which cores the benchmark ran on. + #[error("Firecracker (pid {pid}) is not in its cgroup {cgroup}")] + CgroupMissingPid { + /// PID of the Firecracker process. + pid: u32, + /// The cgroup it should be in. + cgroup: camino::Utf8PathBuf, + }, + + /// A jail artifact could not be handed to the jail uid and gid. + #[error("Jail ownership failed: {0}")] + Chown(#[source] crate::error::JailError), } diff --git a/plus/bencher_runner/src/firecracker/mod.rs b/plus/bencher_runner/src/firecracker/mod.rs index fdbdee111..2d5bd85a6 100644 --- a/plus/bencher_runner/src/firecracker/mod.rs +++ b/plus/bencher_runner/src/firecracker/mod.rs @@ -28,6 +28,7 @@ use std::time::{Duration, Instant}; use camino::Utf8PathBuf; use crate::cpu::CpuLayout; +use crate::jail::{CgroupManager, JailPaths}; use crate::metrics::{self, RunMetrics}; pub use error::FirecrackerError; @@ -46,18 +47,26 @@ const GUEST_CID: u32 = 3; use crate::run::RunOutput; use config::{Action, ActionType, BootSource, Drive, MachineConfig, VsockConfig}; -use process::FirecrackerProcess; +use process::{FirecrackerProcess, JailedSpawn}; use vsock::VsockListener; /// Configuration for a Firecracker-based benchmark run. #[derive(Debug)] pub struct FirecrackerJobConfig { - /// Path to the Firecracker binary. + /// Path to the staged Firecracker binary, outside the jail. pub firecracker_bin: Utf8PathBuf, - /// Path to the kernel image. - pub kernel_path: Utf8PathBuf, - /// Path to the ext4 rootfs image. - pub rootfs_path: Utf8PathBuf, + /// Path to the jailer binary. + pub jailer_bin: Utf8PathBuf, + /// Identity of this microVM: the jailer id, the chroot name, and the + /// cgroup name. Minted before the job's artifacts, because the jail root + /// they are built in is a function of it. + pub vm_id: String, + /// Both views of every file inside the jail chroot. + pub jail: JailPaths, + /// The jailer's `--chroot-base-dir`. + pub chroot_base_dir: Utf8PathBuf, + /// Handle of the empty network namespace the VMM joins. + pub netns: Utf8PathBuf, /// Number of vCPUs. pub vcpus: u8, /// Memory size in MiB. @@ -66,8 +75,6 @@ pub struct FirecrackerJobConfig { pub boot_args: String, /// Execution timeout in seconds. pub timeout_secs: u64, - /// Working directory for temporary files (API socket, vsock UDS). - pub work_dir: Utf8PathBuf, /// Optional CPU layout for core isolation via cpuset. pub cpu_layout: Option, /// Firecracker process log level. @@ -82,16 +89,24 @@ pub struct FirecrackerJobConfig { pub grace_period: bencher_json::GracePeriod, } -/// Run a benchmark inside a Firecracker microVM. +/// Run a benchmark inside a jailed Firecracker microVM. /// /// This function: /// 1. Optionally creates a cgroup with cpuset for CPU isolation -/// 2. Starts a Firecracker process (and moves it into the cgroup) -/// 3. Configures the VM via REST API -/// 4. Creates vsock listeners for result collection -/// 5. Boots the VM -/// 6. Collects results via vsock -/// 7. Cleans up (including cgroup) +/// 2. Starts Firecracker under the jailer, placed in the cgroup before exec +/// 3. Verifies the placement landed +/// 4. Configures the VM via REST API +/// 5. Creates vsock listeners for result collection and hands them to the jail +/// 6. Boots the VM +/// 7. Collects results via vsock +/// 8. Cleans up (including cgroup) +/// +/// The cgroup is a fidelity mechanism, so its absence degrades: no CPU layout, +/// no isolation, or a cgroup that cannot be created means the job proceeds with +/// a warning. Placement and verification are conditional on the cgroup +/// existing, but when it does they are hard requirements: a host that cannot +/// isolate is a declared limitation, while a cgroup that exists but does not +/// contain the VMM is a silent lie about where the benchmark ran. /// /// Returns the benchmark output including exit code and stdout. #[expect( @@ -102,16 +117,15 @@ pub fn run_firecracker( config: &FirecrackerJobConfig, cancel_flag: Option<&Arc>, ) -> Result { - let vm_id = uuid::Uuid::new_v4().to_string(); - let api_socket_path = format!("{}/firecracker-{vm_id}.sock", config.work_dir); - let vsock_uds_path = format!("{}/vsock-{vm_id}.sock", config.work_dir); + let vm_id = config.vm_id.as_str(); + let jail = &config.jail; let start_time = Instant::now(); // Step 0: Create cgroup with cpuset if CPU layout is provided let cgroup = if let Some(layout) = &config.cpu_layout { if layout.has_isolation() { - match crate::jail::CgroupManager::new(&vm_id) { + match CgroupManager::new(vm_id) { Ok(cg) => { // Apply cpuset to pin Firecracker to benchmark cores if let Err(e) = cg.apply_cpuset(layout) { @@ -140,26 +154,50 @@ pub fn run_firecracker( None }; - // Step 1: Start Firecracker process - println!("Starting Firecracker process..."); + // Step 1: Start the jailed Firecracker process. + // + // The cgroup descriptor is opened before the fork so the placement inside + // `pre_exec` is a bare write on an existing descriptor. Placing the VMM + // before it execs, rather than after it is already running, keeps it from + // booting its API and touching memory on the wrong cores first. + println!("Starting jailed Firecracker process..."); let housekeeping_cores = config .cpu_layout .as_ref() .map(|l| l.housekeeping.clone()) .unwrap_or_default(); - let mut fc_process = FirecrackerProcess::start( - config.firecracker_bin.as_str(), - &api_socket_path, - &vm_id, - config.log_level.as_str(), + let cgroup_procs = cgroup + .as_ref() + .map(CgroupManager::open_procs) + .transpose() + .map_err(FirecrackerError::CgroupPlacement)?; + let mut fc_process = FirecrackerProcess::start(JailedSpawn { + jailer_bin: &config.jailer_bin, + exec_file: &config.firecracker_bin, + vm_id, + chroot_base_dir: &config.chroot_base_dir, + netns: &config.netns, + api_socket: jail.api_socket(), + log_level: config.log_level.as_str(), housekeeping_cores, - )?; + cgroup_procs, + })?; - // Move Firecracker process into cgroup for CPU isolation - if let Some(cg) = &cgroup - && let Err(e) = cg.add_pid(fc_process.pid()) - { - eprintln!("Warning: failed to add Firecracker to cgroup: {e}"); + // Step 1b: Verify the placement landed. + // + // `spawn` returns only after `pre_exec` and the exec have completed, so + // this read is race free. A failed write already surfaced as a failed + // spawn; this catches a write that succeeded against the wrong cgroup. + if let Some(cg) = &cgroup { + let placed = cg + .contains_pid(fc_process.pid()) + .map_err(FirecrackerError::CgroupPlacement)?; + if !placed { + return Err(FirecrackerError::CgroupMissingPid { + pid: fc_process.pid(), + cgroup: cg.path().to_owned(), + }); + } } let client = fc_process.client(); @@ -173,26 +211,33 @@ pub fn run_firecracker( smt: false, })?; + // Every path in an API body is the chroot view: these resolve inside the + // jail, not on the host filesystem the runner sees. client.put_boot_source(&BootSource { - kernel_image_path: config.kernel_path.to_string(), + kernel_image_path: jail.kernel().chroot().clone(), boot_args: config.boot_args.clone(), })?; client.put_drive(&Drive { drive_id: "rootfs".to_owned(), - path_on_host: config.rootfs_path.to_string(), + path_on_host: jail.rootfs().chroot().clone(), is_root_device: true, is_read_only: false, })?; client.put_vsock(&VsockConfig { guest_cid: GUEST_CID, - uds_path: vsock_uds_path.clone(), + uds_path: jail.vsock().chroot().clone(), })?; // Step 3: Create vsock listeners (must be before boot) println!("Setting up vsock listeners..."); - let vsock_listener = VsockListener::new(&vsock_uds_path)?; + let vsock_listener = VsockListener::new(jail.vsock().host())?; + // Firecracker connects out to these as the unprivileged jail user, so it + // needs write access to the inodes. After bind and before InstanceStart. + vsock_listener + .chown_to_jail() + .map_err(FirecrackerError::Chown)?; // Step 4: Boot the VM println!("Booting VM..."); diff --git a/plus/bencher_runner/src/firecracker/process.rs b/plus/bencher_runner/src/firecracker/process.rs index 230b7891a..deb6dd698 100644 --- a/plus/bencher_runner/src/firecracker/process.rs +++ b/plus/bencher_runner/src/firecracker/process.rs @@ -1,50 +1,133 @@ -//! Firecracker process management. +//! Jailed Firecracker process management. #![expect(clippy::print_stderr, reason = "process management prints diagnostics")] +use std::fs::File; +use std::os::unix::process::CommandExt as _; use std::process::{Child, Command}; use std::time::Duration; +use camino::Utf8Path; + use crate::firecracker::client::FirecrackerClient; use crate::firecracker::config::{Action, ActionType}; use crate::firecracker::error::FirecrackerError; +use crate::jail::{HostPath, JAIL_GID, JAIL_UID, JailFile}; + +/// Everything needed to spawn the VMM under the jailer. +#[derive(Debug)] +pub struct JailedSpawn<'a> { + /// The jailer binary, which runs as root and execs Firecracker in place. + pub jailer_bin: &'a Utf8Path, + /// The staged Firecracker binary, outside the jail. + /// + /// The jailer copies this into the chroot itself and rejects a multiply + /// linked file, so it is neither placed in the chroot by hand nor + /// hardlinked anywhere. Its base name determines the chroot layout. + pub exec_file: &'a Utf8Path, + /// The jailer `--id`, which is also the chroot name and the cgroup name. + pub vm_id: &'a str, + /// The jailer `--chroot-base-dir`. + pub chroot_base_dir: &'a Utf8Path, + /// Handle of the empty network namespace the VMM joins. + pub netns: &'a Utf8Path, + /// The REST API socket, in both views. + pub api_socket: &'a JailFile, + /// Firecracker process log level. + pub log_level: &'a str, + /// Cores the stderr reader thread is pinned to. + pub housekeeping_cores: Vec, + /// Pre-opened `cgroup.procs`, when a cgroup exists to place the VMM in. + pub cgroup_procs: Option, +} -/// A running Firecracker process. +/// A running, jailed Firecracker process. pub struct FirecrackerProcess { child: Child, - api_socket_path: String, + api_socket_path: HostPath, stderr_thread: Option>, } impl FirecrackerProcess { - /// Start a new Firecracker process. + /// Start Firecracker under the jailer and wait for its API socket. /// - /// Spawns `firecracker --api-sock --id --level ` - /// and waits for the API socket to become ready. - /// A background thread reads stderr and prints lines prefixed with `[firecracker]`. - pub fn start( - firecracker_bin: &str, - api_socket_path: &str, - vm_id: &str, - log_level: &str, - housekeeping_cores: Vec, - ) -> Result { - // Remove stale socket if it exists - drop(std::fs::remove_file(api_socket_path)); - - let mut child = Command::new(firecracker_bin) - .arg("--api-sock") - .arg(api_socket_path) + /// The jailer builds the chroot, creates `/dev/kvm`, drops to the + /// unprivileged jail uid, joins the empty network namespace, and execs + /// Firecracker. Neither `--daemonize` nor `--new-pid-ns` is passed: + /// both make the jailer fork, which would break the pid identity that + /// [`Self::pid`] and [`Self::kill_after_grace_period`] rely on. Without + /// them the jailer execs in place and the child pid stays the VMM. + /// + /// No cgroup flags are passed either. The runner owns the cgroup end to + /// end because it has to create, verify, read metrics from, and remove it, + /// and the cpuset partition specifically needs read-back verification that + /// the jailer's write-once interface cannot provide. + /// + /// A background thread reads stderr and prints lines prefixed with + /// `[firecracker]`. The jailer inherits that stdio and its own diagnostics + /// appear under the same prefix. + pub fn start(spawn: JailedSpawn<'_>) -> Result { + let JailedSpawn { + jailer_bin, + exec_file, + vm_id, + chroot_base_dir, + netns, + api_socket, + log_level, + housekeeping_cores, + cgroup_procs, + } = spawn; + + let mut command = Command::new(jailer_bin); + command .arg("--id") .arg(vm_id) + .arg("--exec-file") + .arg(exec_file) + .arg("--uid") + .arg(JAIL_UID.to_string()) + .arg("--gid") + .arg(JAIL_GID.to_string()) + .arg("--chroot-base-dir") + .arg(chroot_base_dir) + .arg("--netns") + .arg(netns) + .arg("--") + // `--id` is deliberately not forwarded: the jailer already passes + // it to Firecracker, and Firecracker rejects a duplicate argument. + .arg("--api-sock") + .arg(api_socket.chroot().as_str()) .arg("--level") .arg(log_level) .stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::piped()) - .spawn() - .map_err(|e| { - FirecrackerError::ProcessStart(format!("failed to spawn {firecracker_bin}: {e}")) - })?; + .stderr(std::process::Stdio::piped()); + + if let Some(procs) = cgroup_procs { + // Cgroup membership is inherited across `fork` and survives + // `execve`, so writing to the pre-opened descriptor here places + // the child before it execs the jailer, and Firecracker inherits + // the membership through the jailer's own exec. Doing it here + // rather than after spawn also means the VMM never boots its API + // or touches memory on the wrong cores first. + #[expect( + unsafe_code, + reason = "cgroup placement must happen between fork and exec" + )] + // SAFETY: the closure runs in the forked child before `execve`, + // where only async-signal-safe work is permitted. It performs a + // single `write` of a fixed one-byte buffer on a descriptor that + // was opened before the fork: no allocation, no path resolution, + // and no locks. A failed write is reported to the parent over the + // CLOEXEC pipe and surfaces as a failed `spawn`. + unsafe { + command.pre_exec(move || place_in_cgroup(&procs)); + } + } + + let mut child = command.spawn().map_err(|e| { + FirecrackerError::ProcessStart(format!("failed to spawn {jailer_bin}: {e}")) + })?; // Spawn a thread to read stderr line-by-line let stderr = child.stderr.take().ok_or_else(|| { @@ -68,7 +151,7 @@ impl FirecrackerProcess { let process = Self { child, - api_socket_path: api_socket_path.to_owned(), + api_socket_path: api_socket.host().clone(), stderr_thread: Some(stderr_thread), }; @@ -119,8 +202,11 @@ impl FirecrackerProcess { } /// Clean up socket files. + /// + /// The chroot itself is reclaimed wholesale by the jail teardown; this + /// only keeps the socket from outliving the process within a job. pub fn cleanup(&self) { - drop(std::fs::remove_file(&self.api_socket_path)); + drop(std::fs::remove_file(self.api_socket_path.as_path())); } /// Join the stderr reader thread if it exists. @@ -137,3 +223,13 @@ impl Drop for FirecrackerProcess { self.cleanup(); } } + +/// Join the calling task to the cgroup behind a pre-opened `cgroup.procs`. +/// +/// The kernel reads `0` as the calling task, which is why no pid has to be +/// formatted (and no allocation performed) inside the forked child. +fn place_in_cgroup(mut procs: &File) -> std::io::Result<()> { + use std::io::Write as _; + + procs.write_all(b"0") +} diff --git a/plus/bencher_runner/src/firecracker/vsock.rs b/plus/bencher_runner/src/firecracker/vsock.rs index 4043de3ff..db325fef2 100644 --- a/plus/bencher_runner/src/firecracker/vsock.rs +++ b/plus/bencher_runner/src/firecracker/vsock.rs @@ -2,8 +2,12 @@ //! //! Firecracker's vsock implementation uses Unix domain sockets on the host side. //! When the guest connects to CID 2 (host) on port N, Firecracker connects to -//! `{uds_path}_{N}` on the host. The host must have Unix listeners at those -//! paths before VM boot. +//! `{uds_path}_{N}`. The runner binds those sockets, from outside the chroot, +//! before VM boot; Firecracker reaches the same inodes at the chroot view of +//! the path and creates the base `uds_path` itself. +//! +//! Unix domain sockets are scoped by the filesystem, not by the network +//! namespace, so the empty namespace the VMM joins does not affect them. use std::io::Read as _; use std::os::fd::AsFd as _; @@ -12,9 +16,12 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; +use camino::Utf8Path; use nix::poll::{PollFd, PollFlags, PollTimeout, poll}; use crate::firecracker::error::FirecrackerError; +use crate::jail::HostPath; +use crate::jail::chroot::chown_to_jail; /// Poll timeout for vsock listeners (50ms). /// @@ -32,6 +39,9 @@ mod ports { pub const STDERR: u32 = 5001; pub const EXIT_CODE: u32 = 5002; pub const OUTPUT_FILES: u32 = 5005; + + /// Every port the runner listens on. + pub const ALL: [u32; 4] = [STDOUT, STDERR, EXIT_CODE, OUTPUT_FILES]; } /// Results collected from the guest via vsock. @@ -61,9 +71,11 @@ pub struct VsockListener { impl VsockListener { /// Create vsock listeners for all expected ports. /// - /// Creates Unix listeners at `{vsock_uds_path}_{port}` for each port. - /// These must be created before the VM boots. - pub fn new(vsock_uds_path: &str) -> Result { + /// Creates Unix listeners at `{vsock_uds_path}_{port}` for each port, + /// using the host view of the path: the runner binds them from outside + /// the chroot. These must be created before the VM boots. + pub fn new(vsock_uds_path: &HostPath) -> Result { + let vsock_uds_path = vsock_uds_path.as_str(); let stdout_path = format!("{vsock_uds_path}_{}", ports::STDOUT); let stderr_path = format!("{vsock_uds_path}_{}", ports::STDERR); let exit_code_path = format!("{vsock_uds_path}_{}", ports::EXIT_CODE); @@ -263,18 +275,31 @@ impl VsockListener { }) } + /// Hand the listener sockets to the jail uid and gid. + /// + /// Firecracker connects out to these sockets as the unprivileged jail + /// user, so it needs write permission on the inodes. After `pivot_root` + /// the only directory it traverses is `/`, which the jailer chowns itself, + /// so the inodes are all that is left to hand over. Must run after bind + /// and before `InstanceStart`. + pub fn chown_to_jail(&self) -> Result<(), crate::error::JailError> { + for port in ports::ALL { + chown_to_jail(Utf8Path::new(&self.socket_path(port)))?; + } + Ok(()) + } + /// Remove all socket files created by this listener. pub fn cleanup(&self) { - for port in [ - ports::STDOUT, - ports::STDERR, - ports::EXIT_CODE, - ports::OUTPUT_FILES, - ] { - let path = format!("{}_{port}", self.vsock_uds_path); - drop(std::fs::remove_file(path)); + for port in ports::ALL { + drop(std::fs::remove_file(self.socket_path(port))); } } + + /// The host path of the listener socket for a port. + fn socket_path(&self, port: u32) -> String { + format!("{}_{port}", self.vsock_uds_path) + } } impl Drop for VsockListener { @@ -333,11 +358,22 @@ mod tests { /// Short grace period for tests to avoid slowing down the test suite. const TEST_GRACE_PERIOD: Duration = Duration::from_millis(50); + /// Helper: resolve the jail paths for a temp directory standing in for a + /// chroot root, so the tests bind the same host paths production does. + fn jail_paths(dir: &tempfile::TempDir) -> crate::jail::JailPaths { + let root = Utf8Path::from_path(dir.path()).unwrap(); + crate::jail::JailPaths::new(root) + } + + /// Helper: the host path of the vsock base in a temp directory. + fn vsock_base(dir: &tempfile::TempDir) -> String { + jail_paths(dir).vsock().host().as_str().to_owned() + } + /// Helper: create a `VsockListener` in a temp directory. fn listener_in_tmpdir() -> (tempfile::TempDir, VsockListener) { let dir = tempfile::tempdir().unwrap(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); - let listener = VsockListener::new(&base).unwrap(); + let listener = VsockListener::new(jail_paths(&dir).vsock().host()).unwrap(); (dir, listener) } @@ -352,8 +388,8 @@ mod tests { #[test] fn vsock_listener_creates_socket_files() { let dir = tempfile::tempdir().unwrap(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); - let _listener = VsockListener::new(&base).unwrap(); + let base = vsock_base(&dir); + let _listener = VsockListener::new(jail_paths(&dir).vsock().host()).unwrap(); for port in [5000, 5001, 5002, 5005] { let path = format!("{base}_{port}"); @@ -367,10 +403,10 @@ mod tests { #[test] fn vsock_listener_cleanup_removes_files() { let dir = tempfile::tempdir().unwrap(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); + let base = vsock_base(&dir); { - let _listener = VsockListener::new(&base).unwrap(); + let _listener = VsockListener::new(jail_paths(&dir).vsock().host()).unwrap(); // listener drops here } @@ -386,7 +422,7 @@ mod tests { #[test] fn collect_all_ports() { let (dir, listener) = listener_in_tmpdir(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); + let base = vsock_base(&dir); // Build protocol-encoded data: 1 file, path="out.bin", content=\x00\x01\x02 let mut encoded = Vec::new(); @@ -428,7 +464,7 @@ mod tests { #[test] fn collect_exit_code_only() { let (dir, listener) = listener_in_tmpdir(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); + let base = vsock_base(&dir); let base_clone = base.clone(); let sender = std::thread::spawn(move || { @@ -474,7 +510,7 @@ mod tests { #[test] fn collect_non_utf8_stdout() { let (dir, listener) = listener_in_tmpdir(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); + let base = vsock_base(&dir); let base_clone = base.clone(); let sender = std::thread::spawn(move || { @@ -503,7 +539,7 @@ mod tests { #[test] fn collect_exit_code_triggers_final_pass() { let (dir, listener) = listener_in_tmpdir(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); + let base = vsock_base(&dir); let base_clone = base.clone(); let sender = std::thread::spawn(move || { diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index 003cdd6a3..db54e491e 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -252,9 +252,33 @@ impl CgroupManager { self.write_file("cgroup.procs", &pid.to_string()) } - /// Add a process by PID to this cgroup. - pub fn add_pid(&self, pid: u32) -> Result<(), RunnerError> { - self.write_file("cgroup.procs", &pid.to_string()) + /// Open this cgroup's `cgroup.procs` for writing. + /// + /// The descriptor is opened before the fork so the `pre_exec` closure that + /// joins the cgroup performs only a `write` of a fixed byte on an existing + /// descriptor: no allocation, no path resolution, nothing that is not + /// async-signal-safe. + pub fn open_procs(&self) -> Result { + let path = self.cgroup_path.join("cgroup.procs"); + fs::OpenOptions::new() + .write(true) + .open(&path) + .map_err(|e| JailError::OpenCgroupProcs { path, source: e }) + } + + /// Whether `pid` is a member of this cgroup. + /// + /// A failed `pre_exec` write already surfaces as a failed spawn; this is + /// for the different case of a write that succeeded against the wrong + /// destination. A cgroup that exists but does not contain the VMM is a + /// silent lie about where the benchmark ran. + pub fn contains_pid(&self, pid: u32) -> Result { + let path = self.cgroup_path.join("cgroup.procs"); + let procs = fs::read_to_string(&path).map_err(|e| JailError::ReadCgroup { + path: path.clone(), + source: e, + })?; + Ok(procs_contains_pid(&procs, pid)) } /// Write to a cgroup file. @@ -315,6 +339,15 @@ pub(crate) fn effective_mems(cgroup: &Utf8Path) -> String { } } +/// Whether a `cgroup.procs` listing contains `pid`. +/// +/// Matches whole lines: pid `7` must not be satisfied by pid `70`. +fn procs_contains_pid(procs: &str, pid: u32) -> bool { + procs + .lines() + .any(|line| line.trim().parse::() == Ok(pid)) +} + /// Return the first required controller missing from a /// `cgroup.subtree_control` listing, or `None` when all are enabled. /// @@ -356,6 +389,43 @@ mod tests { assert_eq!(missing_required_controller("cpu memory"), Some("pids")); } + #[test] + fn procs_contains_pid_matches_whole_lines() { + assert!(procs_contains_pid("7\n70\n701\n", 7)); + assert!(procs_contains_pid("7\n70\n701\n", 701)); + // A prefix match must not count: pid 7 is not pid 70. + assert!(!procs_contains_pid("70\n701\n", 7)); + assert!(!procs_contains_pid("", 7)); + assert!(!procs_contains_pid("\n", 7)); + } + + #[test] + fn open_procs_reports_a_missing_cgroup() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let manager = CgroupManager { + cgroup_path: root.join("absent"), + created: false, + }; + + manager.open_procs().unwrap_err(); + manager.contains_pid(1).unwrap_err(); + } + + #[test] + fn contains_pid_reads_the_listing() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + fs::write(root.join("cgroup.procs"), "123\n456\n").unwrap(); + let manager = CgroupManager { + cgroup_path: root, + created: false, + }; + + assert!(manager.contains_pid(456).unwrap()); + assert!(!manager.contains_pid(789).unwrap()); + } + #[test] fn effective_mems_reads_file() { let dir = tempfile::tempdir().unwrap(); diff --git a/plus/bencher_runner/src/jail/chroot.rs b/plus/bencher_runner/src/jail/chroot.rs new file mode 100644 index 000000000..a5611dc33 --- /dev/null +++ b/plus/bencher_runner/src/jail/chroot.rs @@ -0,0 +1,143 @@ +//! The per-job chroot the jailer confines Firecracker to. +//! +//! The runner builds the job's artifacts directly inside the chroot rather +//! than copying them in afterwards, which is legal because the jailer uses +//! `create_dir_all` for the chroot and does nothing if the path already +//! exists. Because the artifacts no longer live in a `TempDir`, this type +//! carries the cleanup responsibility that `TempDir` used to. + +#![expect(clippy::print_stderr, reason = "chroot teardown prints diagnostics")] + +use std::fs; +use std::os::unix::fs::{PermissionsExt as _, chown}; + +use camino::{Utf8Path, Utf8PathBuf}; + +use crate::error::JailError; +use crate::jail::{JAIL_GID, JAIL_UID, StateDir}; + +/// A job's chroot tree, removed when this value is dropped. +/// +/// The jailer cleans up nothing by design, so teardown is the runner's job. +/// `Drop` covers completion, timeout, cancellation, and every error return; +/// the sweep in `prepare_host` covers the exits that never unwind. +#[derive(Debug)] +pub struct JailDir { + dir: Utf8PathBuf, + root: Utf8PathBuf, +} + +impl JailDir { + /// Create the chroot tree for `vm_id` at mode 0700. + pub fn create(state: &StateDir, vm_id: &str) -> Result { + let dir = state.jail_dir(vm_id); + let root = state.jail_root(vm_id); + + fs::create_dir_all(&root).map_err(|e| JailError::CreateJail { + path: root.clone(), + source: e, + })?; + // The jailer chowns the chroot root to the jail uid but does not + // change the mode of a directory that already exists, so the runner + // sets it. The tree holds the guest rootfs. + for path in [&dir, &root] { + fs::set_permissions(path, fs::Permissions::from_mode(0o700)).map_err(|e| { + JailError::CreateJail { + path: path.clone(), + source: e, + } + })?; + } + + Ok(Self { dir, root }) + } + + /// The chroot root, which becomes `/` inside the jail. + #[must_use] + pub fn root(&self) -> &Utf8Path { + &self.root + } +} + +impl Drop for JailDir { + fn drop(&mut self) { + if let Err(e) = fs::remove_dir_all(&self.dir) + && e.kind() != std::io::ErrorKind::NotFound + { + eprintln!("Warning: failed to remove jail {}: {e}", self.dir); + } + } +} + +/// Hand a file the runner placed inside the chroot to the jail uid and gid. +/// +/// The jailer creates and chowns the chroot root and the device nodes it +/// makes, but it does not recursively chown what the runner put there. Every +/// artifact Firecracker touches has to be handed over explicitly, and getting +/// it wrong produces an opaque boot failure, so each one is checked. +pub fn chown_to_jail(path: &Utf8Path) -> Result<(), JailError> { + chown(path, Some(JAIL_UID), Some(JAIL_GID)).map_err(|e| JailError::ChownJail { + path: path.to_owned(), + source: e, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn state_in_tmpdir() -> (tempfile::TempDir, StateDir) { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let state = StateDir::new(root.join("state")); + state.create().unwrap(); + (dir, state) + } + + #[test] + fn create_builds_a_private_chroot_tree() { + let (_dir, state) = state_in_tmpdir(); + + let jail = JailDir::create(&state, "vm-1").unwrap(); + + assert_eq!(jail.root(), state.jail_root("vm-1")); + assert!(jail.root().is_dir()); + for path in [state.jail_dir("vm-1"), state.jail_root("vm-1")] { + let mode = fs::metadata(&path).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o700, "{path} should be private"); + } + } + + #[test] + fn create_tolerates_an_existing_directory() { + let (_dir, state) = state_in_tmpdir(); + fs::create_dir_all(state.jail_root("vm-1")).unwrap(); + + JailDir::create(&state, "vm-1").unwrap(); + } + + #[test] + fn drop_removes_the_whole_tree() { + let (_dir, state) = state_in_tmpdir(); + + { + let jail = JailDir::create(&state, "vm-1").unwrap(); + fs::write(jail.root().join("rootfs.ext4"), b"guest").unwrap(); + fs::create_dir_all(jail.root().join("dev")).unwrap(); + } + + assert!( + !state.jail_dir("vm-1").exists(), + "the chroot is the runner's to reclaim, not the jailer's" + ); + assert!(state.jail_parent().exists()); + } + + #[test] + fn drop_tolerates_an_already_removed_tree() { + let (_dir, state) = state_in_tmpdir(); + let jail = JailDir::create(&state, "vm-1").unwrap(); + fs::remove_dir_all(state.jail_dir("vm-1")).unwrap(); + drop(jail); + } +} diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 4902a80ab..dc4abb3d5 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -9,8 +9,12 @@ #[cfg(target_os = "linux")] mod cgroup; #[cfg(target_os = "linux")] +pub mod chroot; +#[cfg(target_os = "linux")] pub mod netns; #[cfg(target_os = "linux")] +pub mod paths; +#[cfg(target_os = "linux")] pub mod state; #[cfg(target_os = "linux")] @@ -18,6 +22,10 @@ pub use cgroup::CgroupManager; #[cfg(target_os = "linux")] pub(crate) use cgroup::{BENCHER_CGROUP_BASE, effective_mems}; #[cfg(target_os = "linux")] +pub use chroot::JailDir; +#[cfg(target_os = "linux")] +pub use paths::{ChrootPath, HostPath, JailFile, JailPaths}; +#[cfg(target_os = "linux")] pub use state::StateDir; use serde::{Deserialize, Serialize}; diff --git a/plus/bencher_runner/src/jail/paths.rs b/plus/bencher_runner/src/jail/paths.rs new file mode 100644 index 000000000..a4db233cc --- /dev/null +++ b/plus/bencher_runner/src/jail/paths.rs @@ -0,0 +1,208 @@ +//! The two views of every file inside the jail chroot. +//! +//! Once Firecracker is jailed, every path it receives resolves inside the +//! chroot, while the runner reaches the same file from outside. The two views +//! are different types rather than two strings, so handing Firecracker a host +//! path (or the runner a chroot path) is a compile error instead of a boot +//! that hangs waiting for a socket that will never appear. + +use camino::{Utf8Path, Utf8PathBuf}; +use serde::Serialize; + +/// A path as the runner sees it: the host filesystem, outside the chroot. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostPath(Utf8PathBuf); + +impl HostPath { + /// The path as a [`Utf8Path`]. + #[must_use] + pub fn as_path(&self) -> &Utf8Path { + &self.0 + } + + /// The path as a string. + #[must_use] + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +impl std::fmt::Display for HostPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +/// A path as the jailed Firecracker process sees it, rooted at the chroot. +/// +/// Serializes as the bare path, since these are what the Firecracker API +/// request bodies carry. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(transparent)] +pub struct ChrootPath(Utf8PathBuf); + +impl ChrootPath { + /// The path as a string. + #[must_use] + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +impl std::fmt::Display for ChrootPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +/// One file in the jail, in both views. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JailFile { + host: HostPath, + chroot: ChrootPath, +} + +impl JailFile { + /// Build both views of `name` directly under the chroot root. + #[must_use] + pub fn new(jail_root: &Utf8Path, name: &str) -> Self { + Self { + host: HostPath(jail_root.join(name)), + chroot: ChrootPath(Utf8Path::new("/").join(name)), + } + } + + /// The path the runner uses. + #[must_use] + pub fn host(&self) -> &HostPath { + &self.host + } + + /// The path Firecracker receives. + #[must_use] + pub fn chroot(&self) -> &ChrootPath { + &self.chroot + } +} + +/// Every file the runner places in, or reaches inside, a jail chroot. +#[derive(Debug, Clone)] +pub struct JailPaths { + root: Utf8PathBuf, + api_socket: JailFile, + kernel: JailFile, + rootfs: JailFile, + vsock: JailFile, +} + +impl JailPaths { + /// Resolve both views of every jail file for a chroot rooted at + /// `jail_root`. + #[must_use] + pub fn new(jail_root: &Utf8Path) -> Self { + Self { + root: jail_root.to_owned(), + api_socket: JailFile::new(jail_root, "api.sock"), + kernel: JailFile::new(jail_root, "vmlinux"), + rootfs: JailFile::new(jail_root, "rootfs.ext4"), + vsock: JailFile::new(jail_root, "v.sock"), + } + } + + /// The chroot root on the host, which becomes `/` inside the jail. + #[must_use] + pub fn root(&self) -> &Utf8Path { + &self.root + } + + /// The Firecracker REST API socket. + #[must_use] + pub fn api_socket(&self) -> &JailFile { + &self.api_socket + } + + /// The guest kernel image, which Firecracker reads. + #[must_use] + pub fn kernel(&self) -> &JailFile { + &self.kernel + } + + /// The guest rootfs image, which Firecracker reads and writes. + #[must_use] + pub fn rootfs(&self) -> &JailFile { + &self.rootfs + } + + /// The base path of the vsock Unix domain sockets. + #[must_use] + pub fn vsock(&self) -> &JailFile { + &self.vsock + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const JAIL_ROOT: &str = "/var/lib/bencher-runner/jail/firecracker/vm-1/root"; + + fn paths() -> JailPaths { + JailPaths::new(Utf8Path::new(JAIL_ROOT)) + } + + #[test] + fn chroot_view_is_rooted_at_the_chroot() { + let paths = paths(); + assert_eq!(paths.api_socket().chroot().as_str(), "/api.sock"); + assert_eq!(paths.kernel().chroot().as_str(), "/vmlinux"); + assert_eq!(paths.rootfs().chroot().as_str(), "/rootfs.ext4"); + assert_eq!(paths.vsock().chroot().as_str(), "/v.sock"); + } + + #[test] + fn host_view_is_under_the_jail_root() { + let paths = paths(); + assert_eq!( + paths.api_socket().host().as_str(), + format!("{JAIL_ROOT}/api.sock") + ); + assert_eq!( + paths.kernel().host().as_str(), + format!("{JAIL_ROOT}/vmlinux") + ); + assert_eq!( + paths.rootfs().host().as_str(), + format!("{JAIL_ROOT}/rootfs.ext4") + ); + assert_eq!(paths.vsock().host().as_str(), format!("{JAIL_ROOT}/v.sock")); + } + + #[test] + fn the_two_views_round_trip_through_the_jail_root() { + let paths = paths(); + for file in [ + paths.api_socket(), + paths.kernel(), + paths.rootfs(), + paths.vsock(), + ] { + let relative = Utf8Path::new(file.chroot().as_str()) + .strip_prefix("/") + .unwrap(); + assert_eq!( + paths.root().join(relative), + file.host().as_path(), + "the chroot view of {file:?} must resolve to its host view" + ); + } + } + + #[test] + fn chroot_paths_serialize_as_bare_strings() { + let paths = paths(); + assert_eq!( + serde_json::to_string(paths.rootfs().chroot()).unwrap(), + "\"/rootfs.ext4\"" + ); + } +} diff --git a/plus/bencher_runner/src/vm.rs b/plus/bencher_runner/src/vm.rs index b1ef180fd..379f4cac1 100644 --- a/plus/bencher_runner/src/vm.rs +++ b/plus/bencher_runner/src/vm.rs @@ -8,9 +8,10 @@ use std::sync::atomic::AtomicBool; use camino::{Utf8Path, Utf8PathBuf}; use crate::error::RunnerError; +use crate::jail::{JailDir, JailPaths, StateDir, chroot, netns, state}; use crate::run::{RunOutput, prepare_oci_workspace}; -/// Execute a single benchmark run in a Firecracker microVM. +/// Execute a single benchmark run in a jailed Firecracker microVM. pub fn vm_execute( config: &crate::Config, cancel_flag: Option<&Arc>, @@ -27,24 +28,33 @@ pub fn vm_execute( println!(" Memory: {} MiB", config.memory.to_mib()); println!(" Timeout: {} seconds", config.timeout_secs); + // The jail root is a function of the VM id, and the job's artifacts are + // built inside it rather than copied in afterwards, so the id is minted + // before any of them exist. Dropping this guard removes the chroot tree, + // which is what the workspace temp directory used to cover. + let vm_id = uuid::Uuid::new_v4().to_string(); + let state_dir = StateDir::new(config.state_dir.clone()); + let jail_dir = JailDir::create(&state_dir, &vm_id)?; + let jail = JailPaths::new(jail_dir.root()); + println!(" Jail: {}", jail.root()); + let workspace = prepare_oci_workspace(config)?; let work_dir = &workspace.work_dir; let unpack_dir = &workspace.unpack_dir; let oci_config = workspace.oci_config; - let rootfs_path = work_dir.join("rootfs.ext4"); - - // Get kernel path - use bundled, provided, or find system kernel - let kernel_path = if let Some(kernel) = &config.kernel { - kernel.clone() + // Everything Firecracker reads has to be inside the chroot, so the kernel + // lands in the jail root whatever its source: bundled, supplied by the + // job, or found on the host. + let kernel_dest = jail.kernel().host().as_path(); + if let Some(kernel) = &config.kernel { + copy_into_jail(kernel, kernel_dest)?; } else if crate::kernel::KERNEL_BUNDLED { - let kernel_dest = work_dir.join("vmlinux"); - crate::kernel::write_kernel_to_file(&kernel_dest)?; + crate::kernel::write_kernel_to_file(kernel_dest)?; println!(" Extracted bundled kernel to {kernel_dest}"); - kernel_dest } else { - find_kernel()? - }; + copy_into_jail(&find_kernel()?, kernel_dest)?; + } let command = oci_config.command; let working_dir = &oci_config.working_dir; @@ -65,38 +75,60 @@ pub fn vm_execute( println!("Installing init binary..."); install_init_binary(unpack_dir)?; - // Step 6: Create ext4 rootfs + // Step 6: Create the ext4 rootfs directly in the jail root + let rootfs_dest = jail.rootfs().host().as_path(); println!( - "Creating ext4 at {rootfs_path} ({} MiB)...", + "Creating ext4 at {rootfs_dest} ({} MiB)...", config.disk.to_mib() ); - bencher_rootfs::create_ext4_with_size(unpack_dir, &rootfs_path, config.disk.to_mib())?; + bencher_rootfs::create_ext4_with_size(unpack_dir, rootfs_dest, config.disk.to_mib())?; + + // The jailer chowns the chroot root and the device nodes it creates, but + // not what the runner placed inside, so hand over each artifact + // explicitly: Firecracker writes the rootfs and reads the kernel. + chroot::chown_to_jail(rootfs_dest)?; + chroot::chown_to_jail(kernel_dest)?; - // Step 7–8: Build Firecracker config and run the microVM - let fc_config = build_firecracker_config(config, work_dir, kernel_path, rootfs_path)?; + // Step 7-8: Build Firecracker config and run the microVM + let fc_config = build_firecracker_config(config, work_dir, vm_id, &state_dir, jail)?; let run_output = run_firecracker(&fc_config, cancel_flag)?; Ok(run_output) } -/// Build the Firecracker job config: resolve the binary and convert types. +/// Build the Firecracker job config: stage the binaries and convert types. fn build_firecracker_config( config: &crate::Config, work_dir: &Utf8Path, - kernel_path: Utf8PathBuf, - rootfs_path: Utf8PathBuf, + vm_id: String, + state_dir: &StateDir, + jail: JailPaths, ) -> Result { - let firecracker_bin = if crate::firecracker_bin::FIRECRACKER_BUNDLED { - let fc_dest = work_dir.join("firecracker"); - crate::firecracker_bin::write_firecracker_to_file(&fc_dest)?; - println!(" Extracted bundled firecracker to {fc_dest}"); - fc_dest + // The jailer copies `--exec-file` into the chroot itself and rejects a + // multiply linked file, so Firecracker is staged outside the jail and is + // never placed in the chroot by hand or hardlinked. Its base name is what + // the jailer derives the chroot layout from, so it is fixed. + let firecracker_bin = work_dir.join(state::EXEC_FILE_NAME); + if crate::firecracker_bin::FIRECRACKER_BUNDLED { + crate::firecracker_bin::write_firecracker_to_file(&firecracker_bin)?; + println!(" Extracted bundled firecracker to {firecracker_bin}"); } else { - find_firecracker_binary()? + copy_binary(&find_firecracker_binary()?, &firecracker_bin)?; + } + + // The jailer runs outside the chroot and is never copied into it, so it + // can be used wherever it is found. + let jailer_bin = if crate::jailer_bin::JAILER_BUNDLED { + let jailer_dest = work_dir.join("jailer"); + crate::jailer_bin::write_jailer_to_file(&jailer_dest)?; + println!(" Extracted bundled jailer to {jailer_dest}"); + jailer_dest + } else { + find_jailer_binary()? }; - println!("Launching Firecracker microVM..."); + println!("Launching jailed Firecracker microVM..."); let vcpus = u8::try_from(u32::from(config.vcpus)).map_err(|_err| { crate::error::ConfigError::OutOfRange { name: "vCPU count", @@ -112,13 +144,15 @@ fn build_firecracker_config( Ok(crate::firecracker::FirecrackerJobConfig { firecracker_bin, - kernel_path, - rootfs_path, + jailer_bin, + vm_id, + jail, + chroot_base_dir: state_dir.chroot_base(), + netns: netns::handle_path(), vcpus, memory_mib, boot_args: config.kernel_cmdline.clone(), timeout_secs: config.timeout_secs, - work_dir: work_dir.to_owned(), cpu_layout: config.cpu_layout.clone(), log_level: config.sandbox_log_level, max_file_count: config.max_file_count, @@ -128,6 +162,31 @@ fn build_firecracker_config( }) } +/// Copy a file the jailed VMM has to read into the jail root. +/// +/// The rule is uniform: a path that resolves outside the chroot is +/// unreachable once Firecracker is confined, whatever produced it. +fn copy_into_jail(src: &Utf8Path, dest: &Utf8Path) -> Result<(), RunnerError> { + std::fs::copy(src, dest).map_err(|e| crate::error::ConfigError::CopyFile { + src: src.to_owned(), + dest: dest.to_owned(), + source: e, + })?; + println!(" Copied {src} into the jail at {dest}"); + Ok(()) +} + +/// Stage an executable found on the host, preserving its executable bit. +fn copy_binary(src: &Utf8Path, dest: &Utf8Path) -> Result<(), RunnerError> { + use std::os::unix::fs::PermissionsExt as _; + + copy_into_jail(src, dest)?; + let mut perms = std::fs::metadata(dest)?.permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(dest, perms)?; + Ok(()) +} + /// Write the init config for the VM. /// /// This creates `/etc/bencher/config.json` which is read by `bencher-init`. @@ -248,6 +307,33 @@ fn find_firecracker_binary() -> Result { .into()) } +/// Find the jailer binary on the system (fallback when not bundled). +fn find_jailer_binary() -> Result { + let candidates = [ + // Next to the current executable + std::env::current_exe() + .ok() + .and_then(|p| p.parent().map(|d| d.join("jailer"))) + .and_then(|p| Utf8PathBuf::try_from(p).ok()), + // Common installation paths + Some(Utf8PathBuf::from("/usr/local/bin/jailer")), + Some(Utf8PathBuf::from("/usr/bin/jailer")), + ]; + + for candidate in candidates.into_iter().flatten() { + if candidate.exists() { + return Ok(candidate); + } + } + + Err(crate::error::ConfigError::BinaryNotFound { + name: "jailer".to_owned(), + hint: "Install from: https://github.com/firecracker-microvm/firecracker/releases" + .to_owned(), + } + .into()) +} + /// Find the kernel image on the system. fn find_kernel() -> Result { let candidates = [ From 8caa692701e6c57d491a226d872f2e1eef68b69e Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Mon, 27 Jul 2026 23:20:39 +0000 Subject: [PATCH 04/91] Test jail confinement, placement policy, and teardown Unit coverage for the pieces that can be exercised without KVM: both path views and their round trip, the chroot layout against the jailer's documented template, the sweep removing stale jails while leaving unrelated entries alone, and that placement and verification are skipped rather than failed when no cgroup exists. The negative cases are covered too: an unbuildable chroot is an error rather than a warning, and a cgroup that does not contain the VMM aborts. The integration scenarios extend the existing KVM-gated runner harness rather than adding a second one. Two invariants only exist while the VMM is alive and cannot be recovered from the runner's output afterwards, so scenarios gain an optional host-side probe: it finds the VMM by its root directory, which the jailer chroots before exec, then checks that it dropped root to the user its jail was handed to and that it is already in its cgroup. Placement happens before the exec, so membership holds the first time the process is observable. Teardown is checked after completion and after cancellation, since the jailer cleans up nothing and each leftover chroot holds a VMM binary and a full rootfs image. Scenarios now run against their own state directory, so jail assertions are scoped to the scenario and never touch a real runner's state. --- plus/bencher_runner/src/firecracker/mod.rs | 98 +++++- plus/bencher_runner/src/jail/cgroup.rs | 13 + plus/bencher_runner/src/jail/chroot.rs | 12 + tasks/test_runner/src/task/scenarios.rs | 350 ++++++++++++++++++++- 4 files changed, 455 insertions(+), 18 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/mod.rs b/plus/bencher_runner/src/firecracker/mod.rs index 2d5bd85a6..6b591e2bf 100644 --- a/plus/bencher_runner/src/firecracker/mod.rs +++ b/plus/bencher_runner/src/firecracker/mod.rs @@ -166,11 +166,7 @@ pub fn run_firecracker( .as_ref() .map(|l| l.housekeeping.clone()) .unwrap_or_default(); - let cgroup_procs = cgroup - .as_ref() - .map(CgroupManager::open_procs) - .transpose() - .map_err(FirecrackerError::CgroupPlacement)?; + let cgroup_procs = placement_target(cgroup.as_ref())?; let mut fc_process = FirecrackerProcess::start(JailedSpawn { jailer_bin: &config.jailer_bin, exec_file: &config.firecracker_bin, @@ -188,17 +184,7 @@ pub fn run_firecracker( // `spawn` returns only after `pre_exec` and the exec have completed, so // this read is race free. A failed write already surfaced as a failed // spawn; this catches a write that succeeded against the wrong cgroup. - if let Some(cg) = &cgroup { - let placed = cg - .contains_pid(fc_process.pid()) - .map_err(FirecrackerError::CgroupPlacement)?; - if !placed { - return Err(FirecrackerError::CgroupMissingPid { - pid: fc_process.pid(), - cgroup: cg.path().to_owned(), - }); - } - } + verify_placement(cgroup.as_ref(), fc_process.pid())?; let client = fc_process.client(); @@ -331,6 +317,44 @@ pub fn run_firecracker( }) } +/// Open the descriptor the VMM will be placed through, if there is a cgroup. +/// +/// Placement is conditional on the cgroup existing, not unconditional: a host +/// with no CPU layout or no isolation gets no cgroup, and therefore no +/// placement, which is the existing degrade behavior. When the cgroup does +/// exist, failing to open it aborts the job. +fn placement_target( + cgroup: Option<&CgroupManager>, +) -> Result, FirecrackerError> { + cgroup + .map(CgroupManager::open_procs) + .transpose() + .map_err(FirecrackerError::CgroupPlacement) +} + +/// Confirm the VMM landed in its cgroup, if there is one. +/// +/// Skipped, not failed, when no cgroup exists. When one does, a pid that is +/// not a member aborts the job: a cgroup that exists but does not contain the +/// VMM is a silent lie about which cores the benchmark ran on, which is worse +/// than a declared absence of isolation. +fn verify_placement(cgroup: Option<&CgroupManager>, pid: u32) -> Result<(), FirecrackerError> { + let Some(cgroup) = cgroup else { + return Ok(()); + }; + let placed = cgroup + .contains_pid(pid) + .map_err(FirecrackerError::CgroupPlacement)?; + if placed { + Ok(()) + } else { + Err(FirecrackerError::CgroupMissingPid { + pid, + cgroup: cgroup.path().to_owned(), + }) + } +} + /// Decode the length-prefixed binary protocol for multiple output files. fn decode_output_files( data: &[u8], @@ -352,6 +376,48 @@ fn parse_exit_code(s: &str) -> i32 { mod tests { use super::*; + // --- placement is conditional on the cgroup existing --- + + #[test] + fn no_cgroup_skips_placement() { + // A host that cannot isolate is a declared limitation, so the job + // proceeds without a cgroup rather than failing. + assert!( + placement_target(None).unwrap().is_none(), + "no cgroup means nothing to place through" + ); + } + + #[test] + fn no_cgroup_skips_verification() { + verify_placement(None, 1).unwrap(); + } + + #[test] + fn a_cgroup_without_the_pid_aborts() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + std::fs::write(root.join("cgroup.procs"), "999\n").unwrap(); + let cgroup = CgroupManager::detached(root); + + let err = verify_placement(Some(&cgroup), 123).unwrap_err(); + + assert!( + matches!(err, FirecrackerError::CgroupMissingPid { pid: 123, .. }), + "a cgroup that does not contain the VMM must abort, got: {err}" + ); + } + + #[test] + fn a_cgroup_holding_the_pid_verifies() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + std::fs::write(root.join("cgroup.procs"), "123\n456\n").unwrap(); + let cgroup = CgroupManager::detached(root); + + verify_placement(Some(&cgroup), 123).unwrap(); + } + #[test] fn parse_exit_code_zero() { assert_eq!(parse_exit_code("0"), 0); diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index db54e491e..c1c577a5b 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -58,6 +58,19 @@ impl CgroupManager { }) } + /// Wrap an existing cgroup directory without creating or removing it. + /// + /// For tests that exercise the placement logic against a stand-in tree + /// rather than the real cgroup filesystem. + #[cfg(test)] + #[must_use] + pub fn detached(cgroup_path: Utf8PathBuf) -> Self { + Self { + cgroup_path, + created: false, + } + } + /// Enable controllers in a cgroup. /// /// Enables cpu, memory, and pids controllers (required), and io/cpuset controllers diff --git a/plus/bencher_runner/src/jail/chroot.rs b/plus/bencher_runner/src/jail/chroot.rs index a5611dc33..bbbf67dd6 100644 --- a/plus/bencher_runner/src/jail/chroot.rs +++ b/plus/bencher_runner/src/jail/chroot.rs @@ -133,6 +133,18 @@ mod tests { assert!(state.jail_parent().exists()); } + #[test] + fn an_unbuildable_chroot_is_an_error_not_a_warning() { + // A chroot that cannot be built is a confinement failure, so it has + // to abort the job rather than degrade into an unjailed run. + let (_dir, state) = state_in_tmpdir(); + // A file where the jail directory has to go makes the tree + // impossible to create. + fs::write(state.jail_dir("vm-1"), b"in the way").unwrap(); + + JailDir::create(&state, "vm-1").unwrap_err(); + } + #[test] fn drop_tolerates_an_already_removed_tree() { let (_dir, state) = state_in_tmpdir(); diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index 3ee044963..900955096 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -37,6 +37,15 @@ fn extract_json_substr(line: &str) -> &str { &line[start..end] } +/// A host-side check run while the runner is still executing. +/// +/// Some confinement invariants only exist while the VMM is alive and cannot +/// be recovered from the runner's output afterwards. The probe receives the +/// runner's state directory and returns `Ok(false)` while the VMM has not +/// appeared yet, `Ok(true)` once the invariant has been observed to hold, and +/// `Err` once it has been observed to be violated. +type Probe = fn(&Utf8Path) -> Result; + /// Test scenario definition. struct Scenario { name: &'static str, @@ -47,6 +56,8 @@ struct Scenario { cancel_after_secs: Option, /// Whether to use `--sandbox firecracker` (default: true). sandboxed: bool, + /// If set, a host-side check run while the runner is executing. + probe: Option, validate: fn(&ScenarioOutput) -> Result<()>, } @@ -105,6 +116,7 @@ impl Scenarios { let runner_bin = ensure_runner_bin()?; let mut scenarios = all_scenarios(); + scenarios.extend(jail_scenarios()); scenarios.extend(nosandbox_scenarios()); if let Some(name) = &self.scenario { @@ -125,6 +137,7 @@ impl Scenarios { /// List all available scenarios. fn list_scenarios() { let mut scenarios = all_scenarios(); + scenarios.extend(jail_scenarios()); scenarios.extend(nosandbox_scenarios()); println!("Available scenarios:"); println!(); @@ -179,16 +192,23 @@ fn run_scenario(scenario: &Scenario, runner_bin: &Utf8Path) -> Result<()> { let image_path = build_test_image(scenario.name, scenario.dockerfile) .with_context(|| format!("Failed to build image for {}", scenario.name))?; + // Every scenario gets its own state directory, so jail assertions are + // scoped to the scenario and never touch a real runner's state. + let state_dir = scenario_state_dir(); + drop(fs::remove_dir_all(&state_dir)); + // Prepend --sandbox firecracker for sandboxed scenarios - let mut args: Vec<&str> = Vec::new(); + let mut args: Vec<&str> = vec!["--state-dir", state_dir.as_str()]; if scenario.sandboxed { args.extend(["--sandbox", "firecracker"]); } args.extend(scenario.extra_args); - // Run the runner (with optional cancellation) + // Run the runner (with optional cancellation or host-side probe) let output = if let Some(secs) = scenario.cancel_after_secs { run_runner_with_cancel(&image_path, &args, Duration::from_secs(secs), runner_bin) + } else if let Some(probe) = scenario.probe { + run_runner_with_probe(&image_path, &args, probe, &state_dir, runner_bin) } else { run_runner(&image_path, &args, runner_bin) } @@ -219,6 +239,7 @@ fn all_scenarios() -> Vec { dockerfile: r#"FROM busybox CMD ["echo", "hello from vm"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -236,6 +257,7 @@ CMD ["echo", "hello from vm"]"#, ENV MY_VAR=test_value CMD ["sh", "-c", "echo $MY_VAR"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -258,6 +280,7 @@ CMD ["sh", "-c", "echo $MY_VAR"]"#, WORKDIR /myapp CMD ["pwd"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -274,6 +297,7 @@ CMD ["pwd"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo '{\"result\": 42}' > /tmp/output.json && cat /tmp/output.json"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--output", "/tmp/output.json"], validate: |output| { @@ -290,6 +314,7 @@ CMD ["sh", "-c", "echo '{\"result\": 42}' > /tmp/output.json && cat /tmp/output. dockerfile: r#"FROM busybox CMD ["sh", "-c", "exit 42"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -307,6 +332,7 @@ CMD ["sh", "-c", "exit 42"]"#, dockerfile: r#"FROM busybox CMD ["sleep", "3600"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { @@ -324,6 +350,7 @@ CMD ["sleep", "3600"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo test > /data.txt && cat /data.txt"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -343,6 +370,7 @@ CMD ["sh", "-c", "echo test > /data.txt && cat /data.txt"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo stdout && echo stderr >&2"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -360,6 +388,7 @@ CMD ["sh", "-c", "echo stdout && echo stderr >&2"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "cat /proc/cpuinfo | grep processor | wc -l"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "10", "--vcpus", "4"], validate: |output| { @@ -387,6 +416,7 @@ CMD ["sh", "-c", "cat /proc/cpuinfo | grep processor | wc -l"]"#, ENTRYPOINT ["echo"] CMD ["hello", "world"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -403,6 +433,7 @@ CMD ["hello", "world"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "ping -c 1 -W 1 8.8.8.8 2>&1 || echo no_network"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -427,6 +458,7 @@ CMD ["sh", "-c", "ping -c 1 -W 1 8.8.8.8 2>&1 || echo no_network"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "dd if=/dev/zero bs=1M count=20 2>/dev/null | tr '\\0' 'A' && echo DONE"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "120", "--max-output-size", "10485760"], validate: |output| { @@ -449,6 +481,7 @@ CMD ["sh", "-c", "dd if=/dev/zero bs=1M count=20 2>/dev/null | tr '\\0' 'A' && e dockerfile: r#"FROM busybox CMD ["sh", "-c", "trap '' TERM INT; echo started; while true; do sleep 1; done"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { @@ -480,6 +513,7 @@ CMD ["sh", "-c", "trap '' TERM INT; echo started; while true; do sleep 1; done"] dockerfile: r#"FROM busybox CMD ["id"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -506,6 +540,7 @@ CMD ["id"]"#, dockerfile: r#"FROM busybox CMD ["echo", "kvm_test_ok"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -528,6 +563,7 @@ CMD ["echo", "kvm_test_ok"]"#, dockerfile: r#"FROM busybox CMD ["cat", "/proc/version"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -552,6 +588,7 @@ CMD ["cat", "/proc/version"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "touch /tmp/write_test && echo write_ok"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -577,6 +614,7 @@ CMD ["sh", "-c", "touch /tmp/write_test && echo write_ok"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo partial_output_marker && sleep 3600"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "10"], validate: |output| { @@ -603,6 +641,7 @@ CMD ["sh", "-c", "echo partial_output_marker && sleep 3600"]"#, dockerfile: r#"FROM busybox CMD ["sleep", "3600"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { @@ -636,6 +675,7 @@ FROM busybox COPY --from=build /test_iopl /test_iopl CMD ["/test_iopl"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -665,6 +705,7 @@ CMD ["/test_iopl"]"#, dockerfile: r#"FROM busybox CMD ["echo", "UNIQUE_VM_OUTPUT_a7f3b2c9"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -693,6 +734,7 @@ CMD ["echo", "UNIQUE_VM_OUTPUT_a7f3b2c9"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "ls /proc | grep -E '^[0-9]+$' | wc -l"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -725,6 +767,7 @@ CMD ["sh", "-c", "ls /proc | grep -E '^[0-9]+$' | wc -l"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "cat /proc/version && echo PID1=$(cat /proc/1/cmdline | tr '\\0' ' ')"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -752,6 +795,7 @@ CMD ["sh", "-c", "cat /proc/version && echo PID1=$(cat /proc/1/cmdline | tr '\\0 dockerfile: r#"FROM busybox CMD ["echo", "metrics_test"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -774,6 +818,7 @@ CMD ["echo", "metrics_test"]"#, dockerfile: r#"FROM busybox CMD ["echo", "fast_benchmark"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -811,6 +856,7 @@ CMD ["echo", "fast_benchmark"]"#, dockerfile: r#"FROM busybox CMD ["sleep", "3600"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { @@ -849,6 +895,7 @@ CMD ["sleep", "3600"]"#, dockerfile: r#"FROM busybox CMD ["echo", "hmac_test_output"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -881,6 +928,7 @@ CMD ["echo", "hmac_test_output"]"#, dockerfile: r#"FROM busybox CMD ["echo", "transport_test"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -915,6 +963,7 @@ CMD ["echo", "transport_test"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo started && sleep 3600"]"#, cancel_after_secs: Some(5), + probe: None, sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { @@ -941,6 +990,7 @@ CMD ["sh", "-c", "echo started && sleep 3600"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo error_output >&2"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -961,6 +1011,7 @@ CMD ["sh", "-c", "echo error_output >&2"]"#, dockerfile: r#"FROM busybox CMD ["true"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -983,6 +1034,7 @@ CMD ["true"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1017,6 +1069,7 @@ CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, // which differs from exec form ["echo", "shell_form_works"]. dockerfile: "FROM busybox\nCMD echo shell_form_works", cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1040,6 +1093,7 @@ CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, dockerfile: r#"FROM busybox ENTRYPOINT ["echo", "entrypoint_only_works"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1062,6 +1116,7 @@ ENTRYPOINT ["echo", "entrypoint_only_works"]"#, // in OCI config. CMD is ignored when ENTRYPOINT uses shell form. dockerfile: "FROM busybox\nENTRYPOINT echo shell_entrypoint_works", cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1091,6 +1146,7 @@ ENTRYPOINT ["echo", "entrypoint_only_works"]"#, ENTRYPOINT echo ep_marker CMD ["cmd_arg"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1120,6 +1176,7 @@ CMD ["cmd_arg"]"#, dockerfile: r#"FROM busybox RUN echo "no command set""#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "30"], validate: |output| { @@ -1147,6 +1204,7 @@ RUN echo "no command set""#, dockerfile: r#"FROM ghcr.io/bencherdev/bencher:latest CMD ["mock"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { @@ -1190,6 +1248,7 @@ FROM gcr.io/distroless/cc-debian12 COPY --from=builder /tmp/hello /usr/bin/hello CMD ["/usr/bin/hello"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { @@ -1228,6 +1287,7 @@ CMD ["/usr/bin/hello"]"#, dockerfile: r#"FROM busybox CMD ["echo", "rapid_exit_marker"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1261,6 +1321,7 @@ CMD ["echo", "rapid_exit_marker"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "exit 137"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1293,6 +1354,7 @@ ENV B1=val11 B2=val12 B3=val13 B4=val14 B5=val15 B6=val16 B7=val17 B8=val18 B9=v ENV LARGE_VALUE=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA CMD ["sh", "-c", "echo A1=$A1 B10=$B10 LARGE_LEN=${#LARGE_VALUE}"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1325,6 +1387,7 @@ CMD ["sh", "-c", "echo A1=$A1 B10=$B10 LARGE_LEN=${#LARGE_VALUE}"]"#, dockerfile: r#"FROM busybox CMD ["echo", "no file written"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--output", "/nonexistent/path.json"], validate: |output| { @@ -1343,6 +1406,7 @@ CMD ["echo", "no file written"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "dd if=/dev/urandom bs=1024 count=2048 2>/dev/null | base64 > /tmp/output.json && echo done"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--output", "/tmp/output.json"], validate: |output| { @@ -1359,6 +1423,7 @@ CMD ["sh", "-c", "dd if=/dev/urandom bs=1024 count=2048 2>/dev/null | base64 > / dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo stdout_marker && echo stderr_marker >&2 && echo '{\"data\":true}' > /tmp/out.json"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--output", "/tmp/out.json"], validate: |output| { @@ -1381,6 +1446,7 @@ CMD ["sh", "-c", "echo stdout_marker && echo stderr_marker >&2 && echo '{\"data\ dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo '{\"result\": 1}' > /tmp/a.json && echo '{\"result\": 2}' > /tmp/b.json && echo done"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &[ "--timeout", @@ -1410,6 +1476,7 @@ RUN mkdir -p /opt && echo "b" > /opt/file_b.txt RUN echo "c" > /var/file_c.txt CMD ["sh", "-c", "cat /tmp/file_a.txt /opt/file_b.txt /var/file_c.txt"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1434,6 +1501,7 @@ CMD ["sh", "-c", "cat /tmp/file_a.txt /opt/file_b.txt /var/file_c.txt"]"#, RUN echo "target" > /tmp/target.txt && ln -s /tmp/target.txt /tmp/link.txt CMD ["cat", "/tmp/link.txt"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1460,6 +1528,7 @@ CMD ["cat", "/tmp/link.txt"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo partial_stdout && echo partial_stderr >&2 && exit 1"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1479,6 +1548,7 @@ CMD ["sh", "-c", "echo partial_stdout && echo partial_stderr >&2 && exit 1"]"#, dockerfile: r#"FROM busybox CMD ["sleep", "3600"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "1"], validate: |output| { @@ -1495,6 +1565,7 @@ CMD ["sleep", "3600"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "dd if=/dev/zero bs=1024 count=50 2>/dev/null | tr '\\0' 'X'"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--max-output-size", "1024"], validate: |output| { @@ -1518,6 +1589,7 @@ ENV LD_LIBRARY_PATH=/testlib ENV SAFE_VAR=safe_value CMD ["sh", "-c", "echo LD_PRELOAD=$LD_PRELOAD LD_LIBRARY_PATH=$LD_LIBRARY_PATH SAFE=$SAFE_VAR"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1557,6 +1629,7 @@ CMD ["sh", "-c", "echo LD_PRELOAD=$LD_PRELOAD LD_LIBRARY_PATH=$LD_LIBRARY_PATH S dockerfile: r#"FROM busybox CMD ["free", "-m"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--memory", "64", "--timeout", "60"], validate: |output| { @@ -1584,6 +1657,7 @@ CMD ["free", "-m"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "df -m / | tail -1 | awk '{print $2}'"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--disk", "64", "--timeout", "60"], validate: |output| { @@ -1611,6 +1685,7 @@ CMD ["sh", "-c", "df -m / | tail -1 | awk '{print $2}'"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "df -m / | tail -1 | awk '{print \"TOTAL_MB=\" $2}'"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--disk", "64", "--timeout", "60"], validate: |output| { @@ -1643,6 +1718,7 @@ CMD ["sh", "-c", "df -m / | tail -1 | awk '{print \"TOTAL_MB=\" $2}'"]"#, dockerfile: r#"FROM busybox CMD ["nproc"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1668,6 +1744,7 @@ CMD ["nproc"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "wget -q -O /dev/null http://detectportal.firefox.com/success.txt && echo net_ok || echo net_fail"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "30", "--network"], validate: |output| { @@ -1698,6 +1775,7 @@ CMD ["sh", "-c", "wget -q -O /dev/null http://detectportal.firefox.com/success.t RUN mkdir -p /data && echo "content_ok" > /data/file.txt CMD ["cat", "/data/file.txt"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1721,6 +1799,7 @@ CMD ["cat", "/data/file.txt"]"#, RUN mkdir -p /data && printf '#!/bin/sh\necho hello' > /data/test.sh && chmod +x /data/test.sh CMD ["sh", "-c", "test -x /data/test.sh && echo perm_ok"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1747,6 +1826,7 @@ CMD ["sh", "-c", "test -x /data/test.sh && echo perm_ok"]"#, RUN mkdir -p /data/restricted && chmod 750 /data/restricted CMD ["stat", "-c", "%a", "/data/restricted"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1773,6 +1853,7 @@ CMD ["stat", "-c", "%a", "/data/restricted"]"#, // Use Docker's multi-line ENV syntax with quotes for values with spaces. dockerfile: "FROM busybox\nENV SPACED=\"hello world\" WITH_EQ=\"key=value\"\nCMD [\"sh\", \"-c\", \"echo SPACED=$SPACED EQ=$WITH_EQ\"]", cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1802,6 +1883,7 @@ CMD ["stat", "-c", "%a", "/data/restricted"]"#, ENTRYPOINT ["echo", "image_ep"] CMD ["image_cmd"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--entrypoint", "echo", "cli_ep"], validate: |output| { @@ -1839,6 +1921,7 @@ CMD ["image_cmd"]"#, ENTRYPOINT ["echo"] CMD ["image_cmd"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--cmd", "cli_cmd"], validate: |output| { @@ -1869,6 +1952,7 @@ CMD ["image_cmd"]"#, ENTRYPOINT ["echo", "image_ep"] CMD ["image_cmd"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &[ "--timeout", @@ -1906,6 +1990,7 @@ CMD ["image_cmd"]"#, ENV MY_VAR=image_value CMD ["sh", "-c", "echo MY_VAR=$MY_VAR"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--env", "MY_VAR=cli_value"], validate: |output| { @@ -1931,6 +2016,7 @@ CMD ["sh", "-c", "echo MY_VAR=$MY_VAR"]"#, ENV EXISTING=from_image CMD ["sh", "-c", "echo EXISTING=$EXISTING NEW=$NEW_VAR"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--env", "NEW_VAR=from_cli"], validate: |output| { @@ -1959,6 +2045,7 @@ CMD ["sh", "-c", "echo EXISTING=$EXISTING NEW=$NEW_VAR"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo A=$A B=$B"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--env", "A=one", "--env", "B=two"], validate: |output| { @@ -1981,6 +2068,7 @@ CMD ["sh", "-c", "echo A=$A B=$B"]"#, dockerfile: r#"FROM busybox CMD ["hello", "world"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--entrypoint", "echo"], validate: |output| { @@ -2006,6 +2094,7 @@ CMD ["hello", "world"]"#, dockerfile: r#"FROM busybox CMD ["echo", "iter_output"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--iter", "3"], validate: |output| { @@ -2026,6 +2115,7 @@ CMD ["echo", "iter_output"]"#, dockerfile: r#"FROM busybox CMD ["echo", "should_not_appear"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--iter", "0"], validate: |output| { @@ -2044,6 +2134,7 @@ CMD ["echo", "should_not_appear"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--iter", "3"], validate: |output| { @@ -2070,6 +2161,7 @@ CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, cancel_after_secs: None, + probe: None, sandboxed: true, extra_args: &["--timeout", "60", "--iter", "3", "--allow-failure"], validate: |output| { @@ -2105,6 +2197,7 @@ fn nosandbox_scenarios() -> Vec { dockerfile: r#"FROM busybox:musl CMD ["echo", "hello from host"]"#, cancel_after_secs: None, + probe: None, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2126,6 +2219,7 @@ CMD ["echo", "hello from host"]"#, ENV MY_VAR=host_test_value CMD ["sh", "-c", "echo $MY_VAR"]"#, cancel_after_secs: None, + probe: None, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2148,6 +2242,7 @@ CMD ["sh", "-c", "echo $MY_VAR"]"#, dockerfile: r#"FROM busybox:musl CMD ["echo", "local_metrics_test"]"#, cancel_after_secs: None, + probe: None, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2180,6 +2275,7 @@ CMD ["echo", "local_metrics_test"]"#, dockerfile: r#"FROM busybox:musl CMD ["sh", "-c", "exit 42"]"#, cancel_after_secs: None, + probe: None, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2372,6 +2468,256 @@ fn ensure_runner_bin() -> Result { Ok(runner_bin) } +// --------------------------------------------------------------------------- +// Jail confinement +// --------------------------------------------------------------------------- + +/// How long to wait for the jailed VMM to appear before giving up. +/// +/// Generous: the runner pulls and unpacks the image and builds the rootfs +/// before the VMM is spawned. +const PROBE_TIMEOUT: Duration = Duration::from_mins(3); + +/// How often to look for the jailed VMM. +const PROBE_INTERVAL: Duration = Duration::from_millis(100); + +/// The state directory scenarios run against. +fn scenario_state_dir() -> Utf8PathBuf { + super::work_dir().join("state") +} + +/// The directory holding one chroot per jailed VMM. +fn jail_parent(state_dir: &Utf8Path) -> Utf8PathBuf { + state_dir.join("jail").join("firecracker") +} + +/// Scenarios covering the confinement of the VMM itself. +fn jail_scenarios() -> Vec { + vec![ + Scenario { + name: "jail_confinement", + description: "VMM runs unprivileged in its cgroup, and its chroot is reclaimed", + dockerfile: r#"FROM busybox +CMD ["echo", "jailed"]"#, + cancel_after_secs: None, + probe: Some(probe_confinement), + sandboxed: true, + extra_args: &["--timeout", "60"], + validate: |output| { + if !output.stdout.contains("jailed") { + bail!("Expected 'jailed' in output, got: {}", output.stdout); + } + assert_no_chroot_remains(&scenario_state_dir()) + }, + }, + Scenario { + name: "jail_teardown_on_cancel", + description: "A cancelled job leaves no chroot behind", + dockerfile: r#"FROM busybox +CMD ["sleep", "300"]"#, + cancel_after_secs: Some(20), + probe: None, + sandboxed: true, + extra_args: &["--timeout", "300"], + validate: |_output| assert_no_chroot_remains(&scenario_state_dir()), + }, + ] +} + +/// Assert every chroot has been reclaimed. +/// +/// The jailer cleans up nothing by design, so a leftover here means the +/// runner's teardown did not run: each one holds a copy of the VMM binary and +/// a full guest rootfs image. +fn assert_no_chroot_remains(state_dir: &Utf8Path) -> Result<()> { + let parent = jail_parent(state_dir); + let leftovers: Vec = match fs::read_dir(&parent) { + Ok(entries) => entries + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(), + Err(_) => Vec::new(), + }; + if leftovers.is_empty() { + Ok(()) + } else { + bail!("Chroots left behind under {parent}: {leftovers:?}") + } +} + +/// Check that the jailed VMM is unprivileged and already in its cgroup. +/// +/// Both invariants disappear with the process, so they cannot be recovered +/// from the runner's output. Cgroup membership in particular must already +/// hold the first time the VMM is seen: it is established before the exec, +/// not after the VM is running. +fn probe_confinement(state_dir: &Utf8Path) -> Result { + let parent = jail_parent(state_dir); + let Some((vm_id, jail_root)) = find_jail(&parent) else { + return Ok(false); + }; + let Some(pid) = find_jailed_vmm(&jail_root) else { + return Ok(false); + }; + + check_unprivileged(pid, &jail_root)?; + check_cgroup_membership(&vm_id, pid)?; + + Ok(true) +} + +/// Find the single chroot under the jail parent, if one exists yet. +fn find_jail(parent: &Utf8Path) -> Option<(String, Utf8PathBuf)> { + for entry in fs::read_dir(parent).ok()?.flatten() { + if !entry.file_type().is_ok_and(|file_type| file_type.is_dir()) { + continue; + } + let vm_id = entry.file_name().to_string_lossy().into_owned(); + let jail_root = parent.join(&vm_id).join("root"); + if jail_root.is_dir() { + return Some((vm_id, jail_root)); + } + } + None +} + +/// Find the pid of the VMM confined to `jail_root`, if it is running yet. +/// +/// The jailer chroots before exec, so the process's root directory is the +/// jail root. That identifies it unambiguously, even if another Firecracker +/// is running on the host. +fn find_jailed_vmm(jail_root: &Utf8Path) -> Option { + for entry in fs::read_dir("/proc").ok()?.flatten() { + let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { + continue; + }; + let Ok(root) = fs::read_link(format!("/proc/{pid}/root")) else { + continue; + }; + if root == Path::new(jail_root.as_str()) { + return Some(pid); + } + } + None +} + +/// Check the VMM dropped root and runs as the user the jail was handed to. +fn check_unprivileged(pid: u32, jail_root: &Utf8Path) -> Result<()> { + let status = fs::read_to_string(format!("/proc/{pid}/status")) + .with_context(|| format!("Failed to read the status of the VMM (pid {pid})"))?; + let uid_line = status + .lines() + .find_map(|line| line.strip_prefix("Uid:")) + .context("No Uid line in the VMM's /proc status")?; + let vmm_uid: u32 = uid_line + .split_whitespace() + .next() + .context("Empty Uid line in the VMM's /proc status")? + .parse() + .context("Unparsable uid in the VMM's /proc status")?; + + if vmm_uid == 0 { + bail!("The VMM (pid {pid}) is running as root; the jailer did not drop privilege"); + } + + // The jailer chowns the chroot root to the jail uid, so the two must + // agree: a VMM running as some other unprivileged user would not be + // confined to the jail it was given. + let jail_uid = jail_root_uid(jail_root)?; + if vmm_uid != jail_uid { + bail!("The VMM (pid {pid}) runs as uid {vmm_uid} but its jail is owned by uid {jail_uid}"); + } + + Ok(()) +} + +/// The uid the jailer handed the chroot root to. +fn jail_root_uid(jail_root: &Utf8Path) -> Result { + use std::os::unix::fs::MetadataExt as _; + + let uid = fs::metadata(jail_root) + .with_context(|| format!("Failed to stat the jail root {jail_root}"))? + .uid(); + if uid == 0 { + bail!("The jail root {jail_root} is still owned by root"); + } + Ok(uid) +} + +/// Check the VMM is in its cgroup. +/// +/// Placement happens before the exec, so the pid is already a member the +/// first time the process is observable. +fn check_cgroup_membership(vm_id: &str, pid: u32) -> Result<()> { + let procs_path = format!("/sys/fs/cgroup/bencher/{vm_id}/cgroup.procs"); + // No cgroup means no isolation was possible on this host, which is a + // declared limitation rather than a confinement failure. + let Ok(procs) = fs::read_to_string(&procs_path) else { + return Ok(()); + }; + if procs.lines().any(|line| line.trim() == pid.to_string()) { + Ok(()) + } else { + bail!("The VMM (pid {pid}) is not in {procs_path}, which holds: {procs:?}") + } +} + +/// Run the runner while checking a host-side invariant. +fn run_runner_with_probe( + image_path: &Utf8Path, + args: &[&str], + probe: Probe, + state_dir: &Utf8Path, + runner_bin: &Utf8Path, +) -> Result { + let mut child = Command::new(runner_bin.as_str()) + .arg("run") + .arg("--image") + .arg(image_path.as_str()) + .args(args) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn()?; + + let deadline = std::time::Instant::now() + PROBE_TIMEOUT; + let mut observed = None; + loop { + match probe(state_dir) { + Ok(true) => { + observed = Some(Ok(())); + break; + }, + Ok(false) => {}, + Err(e) => { + observed = Some(Err(e)); + break; + }, + } + // Stop looking once the runner is gone or the wait is hopeless: the + // output is collected either way so the failure can be explained. + if child.try_wait()?.is_some() || std::time::Instant::now() >= deadline { + break; + } + std::thread::sleep(PROBE_INTERVAL); + } + + let output = child.wait_with_output()?; + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + + match observed { + Some(Ok(())) => Ok(ScenarioOutput { + stdout, + stderr, + exit_code: output.status.code().unwrap_or(-1), + }), + Some(Err(e)) => Err(e).with_context(|| format!("stdout: {stdout}\nstderr: {stderr}")), + None => bail!( + "The jailed VMM was never observed within {PROBE_TIMEOUT:?}.\nstdout: {stdout}\nstderr: {stderr}" + ), + } +} + /// Run the runner and capture output. fn run_runner( image_path: &Utf8Path, From 14dda3bf579e74dbc4a9ca69ddc9cdd2d978a013 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Mon, 27 Jul 2026 23:29:48 +0000 Subject: [PATCH 05/91] Identify the jailed VMM by inode, not by root path The jailer unshares a mount namespace and pivot_roots onto a bind mount of the chroot before exec, so the confined process's root path reads back as `/` from the host and cannot identify it. The bind mount preserves the device and inode of the chroot directory, so comparing those through /proc//root picks out exactly the VMM confined to a given jail. Also corrects two comments against the jailer's actual behavior: it does set the mode of the chroot root even when the directory already exists, and its hard link check is on the destination inside the chroot rather than on the source that is copied in. --- plus/bencher_runner/src/firecracker/mod.rs | 4 ++++ plus/bencher_runner/src/firecracker/process.rs | 7 ++++--- plus/bencher_runner/src/jail/chroot.rs | 15 ++++++++------- tasks/test_runner/src/task/scenarios.rs | 18 +++++++++++++----- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/mod.rs b/plus/bencher_runner/src/firecracker/mod.rs index 6b591e2bf..8cbcf0aac 100644 --- a/plus/bencher_runner/src/firecracker/mod.rs +++ b/plus/bencher_runner/src/firecracker/mod.rs @@ -3,6 +3,10 @@ //! This module manages Firecracker microVMs for running benchmarks in isolation. //! Instead of a custom VMM, we use Firecracker as an external process controlled //! via its REST API over a Unix domain socket. +//! +//! The VMM is never a plain child of the runner. It runs under the Firecracker +//! jailer, in a chroot as an unprivileged user with no host network, and is +//! placed in its cgroup before it execs. See [`crate::jail`]. #![expect( clippy::print_stdout, diff --git a/plus/bencher_runner/src/firecracker/process.rs b/plus/bencher_runner/src/firecracker/process.rs index deb6dd698..c301ad452 100644 --- a/plus/bencher_runner/src/firecracker/process.rs +++ b/plus/bencher_runner/src/firecracker/process.rs @@ -20,9 +20,10 @@ pub struct JailedSpawn<'a> { pub jailer_bin: &'a Utf8Path, /// The staged Firecracker binary, outside the jail. /// - /// The jailer copies this into the chroot itself and rejects a multiply - /// linked file, so it is neither placed in the chroot by hand nor - /// hardlinked anywhere. Its base name determines the chroot layout. + /// The jailer copies this into the chroot itself, and refuses to write + /// over a multiply linked destination, so it is neither placed in the + /// chroot by hand nor hardlinked there. Its base name determines the + /// chroot layout, so it is fixed rather than incidental. pub exec_file: &'a Utf8Path, /// The jailer `--id`, which is also the chroot name and the cgroup name. pub vm_id: &'a str, diff --git a/plus/bencher_runner/src/jail/chroot.rs b/plus/bencher_runner/src/jail/chroot.rs index bbbf67dd6..2bf7d5db2 100644 --- a/plus/bencher_runner/src/jail/chroot.rs +++ b/plus/bencher_runner/src/jail/chroot.rs @@ -37,9 +37,9 @@ impl JailDir { path: root.clone(), source: e, })?; - // The jailer chowns the chroot root to the jail uid but does not - // change the mode of a directory that already exists, so the runner - // sets it. The tree holds the guest rootfs. + // The jailer eventually sets the chroot root to 0700 owned by the jail + // user, but only once it runs. The runner builds the guest rootfs in + // here before that, so the tree is private from the moment it exists. for path in [&dir, &root] { fs::set_permissions(path, fs::Permissions::from_mode(0o700)).map_err(|e| { JailError::CreateJail { @@ -71,10 +71,11 @@ impl Drop for JailDir { /// Hand a file the runner placed inside the chroot to the jail uid and gid. /// -/// The jailer creates and chowns the chroot root and the device nodes it -/// makes, but it does not recursively chown what the runner put there. Every -/// artifact Firecracker touches has to be handed over explicitly, and getting -/// it wrong produces an opaque boot failure, so each one is checked. +/// The jailer chowns the chroot root and the device nodes it makes, but that +/// chown is not recursive: files the runner placed inside keep the ownership +/// they were created with, which is root. Every artifact Firecracker touches +/// has to be handed over explicitly, and getting it wrong produces an opaque +/// boot failure, so each one is checked. pub fn chown_to_jail(path: &Utf8Path) -> Result<(), JailError> { chown(path, Some(JAIL_UID), Some(JAIL_GID)).map_err(|e| JailError::ChownJail { path: path.to_owned(), diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index 900955096..1c7b9f730 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -2583,18 +2583,26 @@ fn find_jail(parent: &Utf8Path) -> Option<(String, Utf8PathBuf)> { /// Find the pid of the VMM confined to `jail_root`, if it is running yet. /// -/// The jailer chroots before exec, so the process's root directory is the -/// jail root. That identifies it unambiguously, even if another Firecracker -/// is running on the host. +/// The jailer pivots into a private mount namespace, so the process's root +/// path reads back as `/` and is useless as an identifier. Its identity is +/// compared instead: the bind mount the jailer pivots onto preserves the +/// device and inode of the chroot directory, so stat'ing through +/// `/proc//root` and stat'ing the jail root agree for exactly the VMM +/// confined to this jail and for no other process on the host. fn find_jailed_vmm(jail_root: &Utf8Path) -> Option { + use std::os::unix::fs::MetadataExt as _; + + let jail = fs::metadata(jail_root).ok()?; for entry in fs::read_dir("/proc").ok()?.flatten() { let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { continue; }; - let Ok(root) = fs::read_link(format!("/proc/{pid}/root")) else { + // Following the magic symlink crosses into the process's own mount + // namespace, which a privileged reader is allowed to do. + let Ok(root) = fs::metadata(format!("/proc/{pid}/root")) else { continue; }; - if root == Path::new(jail_root.as_str()) { + if root.dev() == jail.dev() && root.ino() == jail.ino() { return Some(pid); } } From c23870197fad39b3c034a582a7e1005c8238c62a Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 00:00:55 +0000 Subject: [PATCH 06/91] Propagate the state directory to daemon jobs build_config_from_job never passed the runner's state directory through, so runner up --state-dir prepared and swept one directory while every job built its chroot under the default. The sweep guarded a location that never held a jail, so a SIGKILL, a crash, or a self-update exec leaked a full guest rootfs permanently, and the tree that did hold jails was created by create_dir_all at 0755 rather than the documented 0700 owned by root. The up config is now destructured rather than read field by field. The bug was not that the default was wrong, it was that a builder omission was invisible: Config::new supplies the documented default, so forgetting a with_* call reads as working code. Destructuring makes the omission a build error, which is the same reason the codebase prefers it elsewhere. Left the serde attribute alone deliberately: Config is never deserialized on either job path, so the serde default was not what hid this. --- plus/bencher_runner/src/up/job.rs | 69 ++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/plus/bencher_runner/src/up/job.rs b/plus/bencher_runner/src/up/job.rs index eafac983f..50a1ee0ba 100644 --- a/plus/bencher_runner/src/up/job.rs +++ b/plus/bencher_runner/src/up/job.rs @@ -302,10 +302,36 @@ fn build_metric_output( /// authenticated image pulls. /// /// CPU layout from the up config is passed through for core isolation. +/// +/// The up config is destructured rather than read field by field: every +/// runner-level setting has to reach the job, and a field that is added but +/// never passed through is silently ignored at runtime. Destructuring makes +/// that a build error instead. fn build_config_from_job( up_config: &UpConfig, job: &JsonClaimedJob, ) -> Result { + let UpConfig { + // Protocol and identity settings, which do not describe a run. + host: _, + key: _, + runner: _, + poll_timeout_secs: _, + tuning: _, + no_auto_update: _, + update_channel: _, + max_download_size: _, + allow_no_sandbox: _, + // Run settings, every one of which must reach the config below. + cpu_layout, + max_output_size, + max_file_count, + max_symlinks, + grace_period, + sandbox_log_level, + state_dir, + } = up_config; + let spec = &job.spec; let config = &job.config; @@ -342,35 +368,39 @@ fn build_config_from_job( // Pass all file paths through for multi-file output extraction runner_config = runner_config.with_file_paths_opt(config.file_paths.clone()); + // Pass through the runner's state directory: the jail chroot for the + // job is built under it. + runner_config = runner_config.with_state_dir(state_dir.clone()); + // Pass through CPU layout for core isolation - if let Some(cpu_layout) = &up_config.cpu_layout + if let Some(cpu_layout) = cpu_layout && cpu_layout.has_isolation() { runner_config = runner_config.with_cpu_layout(cpu_layout.clone()); } // Pass through max output size if configured - if let Some(max_output_size) = up_config.max_output_size { + if let Some(max_output_size) = *max_output_size { runner_config = runner_config.with_max_output_size(max_output_size); } // Pass through max file count if configured - if let Some(max_file_count) = up_config.max_file_count { + if let Some(max_file_count) = *max_file_count { runner_config = runner_config.with_max_file_count(max_file_count); } // Pass through max symlinks if configured - if let Some(max_symlinks) = up_config.max_symlinks { + if let Some(max_symlinks) = *max_symlinks { runner_config = runner_config.with_max_symlinks(max_symlinks); } // Pass through grace period if configured - if let Some(grace_period) = up_config.grace_period { + if let Some(grace_period) = *grace_period { runner_config = runner_config.with_grace_period(grace_period); } // Pass through sandbox log level - runner_config.sandbox_log_level = up_config.sandbox_log_level; + runner_config.sandbox_log_level = *sandbox_log_level; // Pass through sandbox mode from the job spec runner_config = runner_config.with_sandbox(spec.sandbox); @@ -784,6 +814,33 @@ mod tests { ); } + #[test] + fn state_dir_passed_through() { + // The daemon prepares and sweeps the state directory it was given, so + // a job that builds its jail somewhere else leaks a chroot on every + // unclean exit and creates the real tree outside the swept location. + let mut up_config = test_up_config(); + up_config.state_dir = Utf8PathBuf::from("/mnt/fast/runner-state"); + let job = test_job(1, mib_to_bytes(512), mib_to_bytes(1024), 300, false); + + let result = build_config_from_job(&up_config, &job).unwrap(); + + assert_eq!( + result.state_dir, + Utf8PathBuf::from("/mnt/fast/runner-state") + ); + } + + #[test] + fn state_dir_defaults_when_the_daemon_uses_the_default() { + let up_config = test_up_config(); + let job = test_job(1, mib_to_bytes(512), mib_to_bytes(1024), 300, false); + + let result = build_config_from_job(&up_config, &job).unwrap(); + + assert_eq!(result.state_dir, crate::jail::DEFAULT_STATE_DIR); + } + #[test] fn cpu_layout_passed_through() { let up_config = test_up_config(); From a62f31a265bc6819cbb6cdff77c5ececadb95625 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 00:01:09 +0000 Subject: [PATCH 07/91] Unwind every stacked mount at the network namespace handle Bind mounting over a file does not report EBUSY, so mounts stack. Against a handle carrying two of them the single detach removed only the top one, the unlink then failed with EBUSY and the error was discarded, and File::create on the surviving nsfs mount failed with EPERM even as root. ensure() then failed permanently: runner up refused to start and every sandboxed runner run failed, until an operator looped umount by hand. Verified on a real kernel. With two mounts stacked, one detach leaves one mount, the unlink reports 'Device or resource busy' and the create reports 'Operation not permitted'; unwinding in a loop leaves none, and both the unlink and the create then succeed. The loop is bounded, since a path that reports a successful unmount forever is a kernel fault and the unlink that follows reports the real state either way. The unlink error is no longer discarded: a handle that cannot be cleared is a confinement failure, not something to paper over with a create that fails more confusingly. Note on the namespace creation this guards: the plan called for forking a child that unshares, and this uses a dedicated thread instead. Namespaces are per task, so unsharing on a thread moves only that thread and leaves the runner on the host network, while avoiding fork in a process that has threads, where only async-signal-safe work is permitted before exec. That is why /proc/thread-self is required rather than /proc/self, which resolves through the thread group leader and would pin the host namespace. --- plus/bencher_runner/src/jail/netns.rs | 73 +++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/plus/bencher_runner/src/jail/netns.rs b/plus/bencher_runner/src/jail/netns.rs index c98aa4ded..e511d8fb1 100644 --- a/plus/bencher_runner/src/jail/netns.rs +++ b/plus/bencher_runner/src/jail/netns.rs @@ -30,6 +30,13 @@ const NETNS_NAME: &str = "bencher-jail"; /// whether a handle is a live namespace distinct from the host's. const SELF_NETNS: &str = "/proc/self/ns/net"; +/// How many stacked mounts to unwind at the handle before giving up. +/// +/// Bounded rather than unbounded: a path that keeps reporting a successful +/// unmount forever is a kernel fault, and the failed unlink that follows +/// reports the real state either way. +const MAX_STACKED_MOUNTS: usize = 32; + /// The calling *thread's* network namespace. /// /// `/proc/self` resolves through the thread group leader, so it must not be @@ -60,10 +67,7 @@ pub fn ensure() -> Result { return Ok(handle); } - // Clear whatever is at the path. A bind mount over a file does not - // report EBUSY, so mounts would otherwise stack up silently. - let _detached = umount2(handle.as_std_path(), MntFlags::MNT_DETACH); - drop(fs::remove_file(&handle)); + clear(&handle)?; // The bind mount needs a regular file to land on. fs::File::create(&handle).map_err(|e| JailError::NetnsHandle { @@ -79,6 +83,33 @@ pub fn ensure() -> Result { Ok(handle) } +/// Remove whatever is at the handle path, mounts included. +/// +/// Bind mounting over a file does not report `EBUSY`, so mounts stack: a +/// handle that has been recreated more than once carries more than one. A +/// single detach unwinds only the top mount, the unlink of the still-mounted +/// path then fails with `EBUSY`, and `File::create` on the surviving nsfs +/// mount fails with `EPERM` even as root. Unwinding one mount at a time and +/// reporting a failed unlink is what keeps a stacked handle from wedging the +/// host: without it, `ensure` fails permanently and every sandboxed job with +/// it, until an operator loops `umount` by hand. +fn clear(handle: &Utf8Path) -> Result<(), JailError> { + for _ in 0..MAX_STACKED_MOUNTS { + if umount2(handle.as_std_path(), MntFlags::MNT_DETACH).is_err() { + break; + } + } + + match fs::remove_file(handle) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(JailError::NetnsHandle { + path: handle.to_owned(), + source: e, + }), + } +} + /// Whether `handle` is a live network namespace other than the runner's own. /// /// Every namespace inode lives on the single kernel `nsfs`, so sharing a @@ -148,6 +179,40 @@ mod tests { assert!(!is_live_netns(&root.join("absent"))); } + #[test] + fn clear_removes_a_plain_handle() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let path = root.join("net"); + fs::write(&path, b"").unwrap(); + + clear(&path).unwrap(); + + assert!(!path.exists()); + } + + #[test] + fn clear_is_idempotent_on_a_missing_handle() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + clear(&root.join("absent")).unwrap(); + } + + #[test] + fn clear_reports_a_handle_it_cannot_remove() { + // A directory stands in for the unremovable handle: the real case is + // a still-mounted path, which unlinks with EBUSY. Either way the + // failure has to surface rather than be swallowed into a confusing + // File::create error further down. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let path = root.join("net"); + fs::create_dir(&path).unwrap(); + + clear(&path).unwrap_err(); + } + #[test] fn the_runners_own_namespace_is_not_a_distinct_netns() { // The handle must be a namespace *other* than the one the runner is From 15116c1d40c7b07ad19704352754f68a49b730b1 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 00:01:20 +0000 Subject: [PATCH 08/91] Serialize the jail lifecycle with an advisory lock The sweep removes every chroot it finds, on the reasoning that jobs are serial so anything left is stale. That reasoning was an assumption, not a constraint: a one-shot runner run started while the daemon had a job in flight would remove_dir_all the live chroot out from under a running VMM. Both paths now resolve to the same state directory, so nothing kept them apart. An advisory flock on /.lock is held across prepare_host and for the life of a job. It is declared before the jail guard so it outlives the teardown it protects, and the kernel releases it if the holder dies, so a crashed runner cannot wedge future runs. The same lock closes the race where two processes clearing and rebinding the network namespace handle at once stack mounts on it. Unlike the host tuning lock, which degrades to skipping tuning when contended, this one waits: a runner that proceeded without it would destroy another runner's work, so declining to hold it is not an option. It tries once without blocking first so that waiting is announced rather than looking like a hang. The lock file sits beside the chroot base rather than inside it, so the sweep can never reach it. --- plus/bencher_runner/src/error.rs | 14 +++ plus/bencher_runner/src/jail/lock.rs | 155 +++++++++++++++++++++++++++ plus/bencher_runner/src/jail/mod.rs | 13 +++ plus/bencher_runner/src/vm.rs | 10 +- 4 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 plus/bencher_runner/src/jail/lock.rs diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index 520143dfc..e06fd0f2f 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -94,6 +94,20 @@ pub enum JailError { #[error("The network namespace thread panicked")] NetnsThread, + #[cfg(target_os = "linux")] + #[error("Failed to open the jail lock {path}: {source}")] + OpenJailLock { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to take the jail lock {path}: {source}")] + JailLock { + path: Utf8PathBuf, + source: std::io::Error, + }, + #[cfg(target_os = "linux")] #[error("Failed to create jail chroot {path}: {source}")] CreateJail { diff --git a/plus/bencher_runner/src/jail/lock.rs b/plus/bencher_runner/src/jail/lock.rs new file mode 100644 index 000000000..aa0cb523b --- /dev/null +++ b/plus/bencher_runner/src/jail/lock.rs @@ -0,0 +1,155 @@ +//! Advisory lock serializing the jail lifecycle across runner processes. +//! +//! The sweep reclaims every chroot it finds, on the reasoning that jobs are +//! serial and so anything left is stale. That reasoning is only sound while it +//! is enforced: a one-shot `runner run` started on a host where the `runner up` +//! daemon has a job in flight would otherwise `remove_dir_all` the live chroot +//! out from under a running VMM. The same lock closes the race on the network +//! namespace handle, where two processes clearing and rebinding it at once can +//! stack mounts. +//! +//! Unlike the host tuning lock, which degrades to skipping tuning when it is +//! contended, this one waits. A runner that proceeded without it would destroy +//! another runner's work, so declining to hold it is not an option. + +#![expect(clippy::print_stdout, reason = "prints why the runner is waiting")] + +use std::fs::{File, OpenOptions}; +use std::os::fd::AsRawFd as _; + +use camino::Utf8Path; + +use crate::error::JailError; + +/// Lock file name inside the state directory. +/// +/// It lives beside the chroot base rather than inside it, so the sweep (which +/// only removes directories under `/jail/firecracker`) can never +/// reach it. +const LOCK_FILE: &str = ".lock"; + +/// Holds the jail lock for as long as it is alive. +/// +/// The kernel releases a `flock` when the holder exits or dies, so a crashed +/// runner cannot wedge future runs. +#[derive(Debug)] +pub struct JailLock { + /// The locked file, held only for its `flock`. + _file: File, +} + +impl JailLock { + /// Take the jail lock, waiting for whichever runner holds it. + /// + /// The state directory must already exist: the lock guards the contents, + /// so creating the directory is not something it can protect. + pub fn acquire(state_dir: &Utf8Path) -> Result { + let path = state_dir.join(LOCK_FILE); + let file = OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(&path) + .map_err(|e| JailError::OpenJailLock { + path: path.clone(), + source: e, + })?; + + // Try once without blocking, so waiting can be announced rather than + // looking like a hang. + if flock(&file, libc::LOCK_EX | libc::LOCK_NB).is_ok() { + return Ok(Self { _file: file }); + } + println!(" Waiting for another bencher runner to release {path}..."); + + flock(&file, libc::LOCK_EX).map_err(|e| JailError::JailLock { + path: path.clone(), + source: e, + })?; + + Ok(Self { _file: file }) + } +} + +/// Apply `flock` to a file, retrying if a signal interrupts the wait. +fn flock(file: &File, operation: libc::c_int) -> std::io::Result<()> { + loop { + #[expect( + unsafe_code, + reason = "flock has no std wrapper; the fd is owned and valid" + )] + // SAFETY: `file` is an open, owned descriptor for the duration of the + // call; flock does not touch memory. + let ret = unsafe { libc::flock(file.as_raw_fd(), operation) }; + if ret == 0 { + return Ok(()); + } + let err = std::io::Error::last_os_error(); + if err.kind() != std::io::ErrorKind::Interrupted { + return Err(err); + } + } +} + +#[cfg(test)] +mod tests { + use camino::Utf8PathBuf; + + use super::*; + + fn state_in_tmpdir() -> (tempfile::TempDir, Utf8PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + (dir, root) + } + + #[test] + fn the_lock_can_be_taken() { + let (_dir, state) = state_in_tmpdir(); + + let lock = JailLock::acquire(&state).unwrap(); + + assert!(state.join(LOCK_FILE).exists()); + drop(lock); + } + + #[test] + fn a_released_lock_can_be_retaken() { + let (_dir, state) = state_in_tmpdir(); + drop(JailLock::acquire(&state).unwrap()); + + JailLock::acquire(&state).unwrap(); + } + + #[test] + fn a_held_lock_makes_a_second_runner_wait() { + // `flock` is per open file description, so a second `acquire` in this + // process contends exactly as another process would. The waiter is + // released only once the holder drops, which is what keeps a sweep + // from running while another runner has a job in flight. + let (_dir, state) = state_in_tmpdir(); + let held = JailLock::acquire(&state).unwrap(); + + let waiter = { + let state = state.clone(); + std::thread::spawn(move || JailLock::acquire(&state)) + }; + + // The waiter must still be blocked while the lock is held. + std::thread::sleep(std::time::Duration::from_millis(200)); + assert!( + !waiter.is_finished(), + "a second runner must wait rather than proceed to sweep" + ); + + drop(held); + waiter.join().unwrap().unwrap(); + } + + #[test] + fn a_missing_state_directory_is_an_error() { + let (_dir, state) = state_in_tmpdir(); + + JailLock::acquire(&state.join("absent")).unwrap_err(); + } +} diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index dc4abb3d5..214c34f84 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -11,6 +11,8 @@ mod cgroup; #[cfg(target_os = "linux")] pub mod chroot; #[cfg(target_os = "linux")] +pub mod lock; +#[cfg(target_os = "linux")] pub mod netns; #[cfg(target_os = "linux")] pub mod paths; @@ -24,6 +26,8 @@ pub(crate) use cgroup::{BENCHER_CGROUP_BASE, effective_mems}; #[cfg(target_os = "linux")] pub use chroot::JailDir; #[cfg(target_os = "linux")] +pub use lock::JailLock; +#[cfg(target_os = "linux")] pub use paths::{ChrootPath, HostPath, JailFile, JailPaths}; #[cfg(target_os = "linux")] pub use state::StateDir; @@ -57,10 +61,19 @@ pub const JAIL_GID: u32 = 60613; /// /// Failure is fatal. Untrusted code never runs with silently degraded /// confinement, so a host that cannot be prepared does not execute a job. +/// +/// The sweep and the network namespace handle are both taken under the jail +/// lock. The sweep removes every chroot it finds on the reasoning that jobs +/// are serial, so it must not run while another runner has one in flight, and +/// two processes rebinding the namespace handle at once can stack mounts on +/// it. Holding the lock makes serialization a constraint rather than an +/// assumption. #[cfg(target_os = "linux")] pub fn prepare_host(state_dir: &camino::Utf8Path) -> Result<(), crate::error::JailError> { let state = StateDir::new(state_dir.to_owned()); state.create()?; + + let _lock = JailLock::acquire(state.path())?; state::sweep_jails(&state.jail_parent()); netns::ensure()?; Ok(()) diff --git a/plus/bencher_runner/src/vm.rs b/plus/bencher_runner/src/vm.rs index 379f4cac1..67259f904 100644 --- a/plus/bencher_runner/src/vm.rs +++ b/plus/bencher_runner/src/vm.rs @@ -8,7 +8,7 @@ use std::sync::atomic::AtomicBool; use camino::{Utf8Path, Utf8PathBuf}; use crate::error::RunnerError; -use crate::jail::{JailDir, JailPaths, StateDir, chroot, netns, state}; +use crate::jail::{JailDir, JailLock, JailPaths, StateDir, chroot, netns, state}; use crate::run::{RunOutput, prepare_oci_workspace}; /// Execute a single benchmark run in a jailed Firecracker microVM. @@ -28,12 +28,18 @@ pub fn vm_execute( println!(" Memory: {} MiB", config.memory.to_mib()); println!(" Timeout: {} seconds", config.timeout_secs); + let state_dir = StateDir::new(config.state_dir.clone()); + + // Held for the whole job. Another runner's sweep removes every chroot it + // finds, so it must not run while this one is live. Declared before the + // jail guard so the lock outlives the teardown it protects. + let _lock = JailLock::acquire(state_dir.path())?; + // The jail root is a function of the VM id, and the job's artifacts are // built inside it rather than copied in afterwards, so the id is minted // before any of them exist. Dropping this guard removes the chroot tree, // which is what the workspace temp directory used to cover. let vm_id = uuid::Uuid::new_v4().to_string(); - let state_dir = StateDir::new(config.state_dir.clone()); let jail_dir = JailDir::create(&state_dir, &vm_id)?; let jail = JailPaths::new(jail_dir.root()); println!(" Jail: {}", jail.root()); From fdcc034545ecdea0d967587b4c07706c7331fd1a Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 00:01:29 +0000 Subject: [PATCH 09/91] Assert the jailer argument vector The command line was built inline in the spawn and nothing covered it, so only a live KVM boot would catch a regression. Forwarding --id after the separator is the sharpest case: the jailer already passes it to Firecracker, which rejects the duplicate and fails every job at startup, with an error that points at Firecracker rather than at the command line that caused it. Extracted so it can be asserted anywhere: --id present exactly once, the --api-sock value carrying the chroot view with no host jail path anywhere in the vector, the uid, gid, chroot base and netns flags all present, the separator, nothing after it but --api-sock and --level, and none of the cgroup or forking flags the design deliberately omits. The spawn destructures the remaining fields rather than reading them, so adding one without deciding what it does on the command line is a build error. --- .../bencher_runner/src/firecracker/process.rs | 194 +++++++++++++++--- 1 file changed, 170 insertions(+), 24 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/process.rs b/plus/bencher_runner/src/firecracker/process.rs index c301ad452..9092a6b70 100644 --- a/plus/bencher_runner/src/firecracker/process.rs +++ b/plus/bencher_runner/src/firecracker/process.rs @@ -67,39 +67,25 @@ impl FirecrackerProcess { /// `[firecracker]`. The jailer inherits that stdio and its own diagnostics /// appear under the same prefix. pub fn start(spawn: JailedSpawn<'_>) -> Result { + let args = jailer_args(&spawn); + + // Destructured rather than read field by field so that adding a field + // without deciding what it does here is a build error. let JailedSpawn { jailer_bin, - exec_file, - vm_id, - chroot_base_dir, - netns, + exec_file: _, + vm_id: _, + chroot_base_dir: _, + netns: _, api_socket, - log_level, + log_level: _, housekeeping_cores, cgroup_procs, } = spawn; let mut command = Command::new(jailer_bin); command - .arg("--id") - .arg(vm_id) - .arg("--exec-file") - .arg(exec_file) - .arg("--uid") - .arg(JAIL_UID.to_string()) - .arg("--gid") - .arg(JAIL_GID.to_string()) - .arg("--chroot-base-dir") - .arg(chroot_base_dir) - .arg("--netns") - .arg(netns) - .arg("--") - // `--id` is deliberately not forwarded: the jailer already passes - // it to Firecracker, and Firecracker rejects a duplicate argument. - .arg("--api-sock") - .arg(api_socket.chroot().as_str()) - .arg("--level") - .arg(log_level) + .args(&args) .stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::piped()); @@ -225,6 +211,41 @@ impl Drop for FirecrackerProcess { } } +/// Build the jailer's argument vector. +/// +/// Split out from the spawn so it can be asserted without a host that can +/// boot a VM. Getting this wrong fails every job at startup and, before the +/// VM boots, produces errors that point at Firecracker rather than at the +/// command line that caused them. +fn jailer_args(spawn: &JailedSpawn<'_>) -> Vec { + vec![ + "--id".to_owned(), + spawn.vm_id.to_owned(), + "--exec-file".to_owned(), + spawn.exec_file.to_string(), + "--uid".to_owned(), + JAIL_UID.to_string(), + "--gid".to_owned(), + JAIL_GID.to_string(), + "--chroot-base-dir".to_owned(), + spawn.chroot_base_dir.to_string(), + "--netns".to_owned(), + spawn.netns.to_string(), + // No cgroup flags of any kind, and neither --daemonize nor + // --new-pid-ns: see `FirecrackerProcess::start`. + "--".to_owned(), + // `--id` is deliberately not forwarded: the jailer already passes it + // to Firecracker, which rejects the duplicate with DuplicateArgument + // and fails every job at startup. + "--api-sock".to_owned(), + // The chroot view. Firecracker binds this after it has been confined, + // so the host path would name a directory it cannot reach. + spawn.api_socket.chroot().as_str().to_owned(), + "--level".to_owned(), + spawn.log_level.to_owned(), + ] +} + /// Join the calling task to the cgroup behind a pre-opened `cgroup.procs`. /// /// The kernel reads `0` as the calling task, which is why no pid has to be @@ -234,3 +255,128 @@ fn place_in_cgroup(mut procs: &File) -> std::io::Result<()> { procs.write_all(b"0") } + +#[cfg(test)] +mod tests { + use camino::Utf8Path; + + use super::*; + use crate::jail::JailPaths; + + const JAIL_ROOT: &str = "/var/lib/bencher-runner/jail/firecracker/vm-1/root"; + + fn spawn_for(jail: &JailPaths) -> JailedSpawn<'_> { + JailedSpawn { + jailer_bin: Utf8Path::new("/tmp/work/jailer"), + exec_file: Utf8Path::new("/tmp/work/firecracker"), + vm_id: "vm-1", + chroot_base_dir: Utf8Path::new("/var/lib/bencher-runner/jail"), + netns: Utf8Path::new("/run/netns/bencher-jail"), + api_socket: jail.api_socket(), + log_level: "Warning", + housekeeping_cores: Vec::new(), + cgroup_procs: None, + } + } + + fn args() -> Vec { + let jail = JailPaths::new(Utf8Path::new(JAIL_ROOT)); + jailer_args(&spawn_for(&jail)) + } + + /// The value following `flag`, if the flag is present. + fn value_of<'a>(args: &'a [String], flag: &str) -> Option<&'a str> { + let index = args.iter().position(|arg| arg == flag)?; + args.get(index + 1).map(String::as_str) + } + + #[test] + fn confinement_flags_are_all_present() { + let args = args(); + + assert_eq!(value_of(&args, "--id"), Some("vm-1")); + assert_eq!( + value_of(&args, "--exec-file"), + Some("/tmp/work/firecracker") + ); + assert_eq!(value_of(&args, "--uid"), Some("60613")); + assert_eq!(value_of(&args, "--gid"), Some("60613")); + assert_eq!( + value_of(&args, "--chroot-base-dir"), + Some("/var/lib/bencher-runner/jail") + ); + assert_eq!(value_of(&args, "--netns"), Some("/run/netns/bencher-jail")); + } + + #[test] + fn id_appears_exactly_once() { + // The jailer passes `--id` to Firecracker itself. Forwarding it again + // after the separator makes Firecracker reject the duplicate and fail + // every job at startup. + let args = args(); + + assert_eq!( + args.iter().filter(|arg| *arg == "--id").count(), + 1, + "--id must be given to the jailer only: {args:?}" + ); + } + + #[test] + fn only_the_api_socket_and_level_are_forwarded() { + let args = args(); + let separator = args + .iter() + .position(|arg| arg == "--") + .expect("the jailer needs a -- separator before Firecracker's own arguments"); + + assert_eq!( + args.get(separator + 1..), + Some( + [ + "--api-sock".to_owned(), + "/api.sock".to_owned(), + "--level".to_owned(), + "Warning".to_owned(), + ] + .as_slice() + ) + ); + } + + #[test] + fn the_api_socket_is_the_chroot_view() { + // Firecracker binds the socket after it has been confined, so it must + // receive the path as it will exist inside the chroot. The host view + // names a directory the jailed process cannot reach. + let jail = JailPaths::new(Utf8Path::new(JAIL_ROOT)); + let args = jailer_args(&spawn_for(&jail)); + + assert_eq!(value_of(&args, "--api-sock"), Some("/api.sock")); + assert!( + !args.iter().any(|arg| arg.contains(JAIL_ROOT)), + "no host-side jail path may reach the jailed process: {args:?}" + ); + } + + #[test] + fn no_cgroup_or_forking_flags_are_passed() { + // The runner owns the cgroup end to end, and both --daemonize and + // --new-pid-ns make the jailer fork, which breaks the pid identity the + // process management relies on. + let args = args(); + + for forbidden in [ + "--cgroup", + "--parent-cgroup", + "--cgroup-version", + "--daemonize", + "--new-pid-ns", + ] { + assert!( + !args.iter().any(|arg| arg == forbidden), + "{forbidden} must not be passed: {args:?}" + ); + } + } +} From 4322071b35afca0e6ea699123b3ebd3dd18c7620 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 03:33:21 +0000 Subject: [PATCH 10/91] Make the jail uid and gid configurable, defaulting to 61016 Self-hosted runners land on customer hardware whose id allocation Bencher does not control. A local process owning the jail uid can signal the VMM and, depending on the ptrace scope, trace it, so an operator whose host already allocates in this range needs a way out. --jail-uid and --jail-gid go through both entry points the way --state-dir does. The default is 61016, Bencher's historic default self-hosted API server port, retired in favor of the IANA-registered 6610. It reads as a project convention rather than an arbitrary pick, and still lands in the unallocated gap between the ids systemd-homed claims (60001-60513) and the DynamicUser range (61184-65519). prepare_host warns when the configured id resolves to a named account. The jailer needs no passwd entry, so a name resolving there is the cheap signal that the host allocates in this range. It reads /etc/passwd and /etc/group directly rather than calling getpwuid: the runner ships as a self-contained binary and a local account is exactly what matters. A warning rather than a refusal, since an operator who deliberately created the account is a legitimate setup and only they can tell the two apart. --- plus/bencher_runner/src/config.rs | 14 ++ plus/bencher_runner/src/firecracker/mod.rs | 7 +- .../bencher_runner/src/firecracker/process.rs | 14 +- plus/bencher_runner/src/firecracker/vsock.rs | 6 +- plus/bencher_runner/src/jail/chroot.rs | 7 +- plus/bencher_runner/src/jail/mod.rs | 163 ++++++++++++++++-- plus/bencher_runner/src/lib.rs | 2 +- plus/bencher_runner/src/run.rs | 5 +- plus/bencher_runner/src/up/job.rs | 21 +++ plus/bencher_runner/src/up/mod.rs | 5 +- plus/bencher_runner/src/vm.rs | 5 +- .../docs-reference/runner/de/runner-run.mdx | 14 ++ .../docs-reference/runner/de/runner-up.mdx | 14 ++ .../docs-reference/runner/en/runner-run.mdx | 14 ++ .../docs-reference/runner/en/runner-up.mdx | 14 ++ .../docs-reference/runner/es/runner-run.mdx | 14 ++ .../docs-reference/runner/es/runner-up.mdx | 14 ++ .../docs-reference/runner/fr/runner-run.mdx | 14 ++ .../docs-reference/runner/fr/runner-up.mdx | 14 ++ .../docs-reference/runner/ja/runner-run.mdx | 14 ++ .../docs-reference/runner/ja/runner-up.mdx | 14 ++ .../docs-reference/runner/ko/runner-run.mdx | 14 ++ .../docs-reference/runner/ko/runner-up.mdx | 14 ++ .../docs-reference/runner/pt/runner-run.mdx | 14 ++ .../docs-reference/runner/pt/runner-up.mdx | 14 ++ .../docs-reference/runner/ru/runner-run.mdx | 14 ++ .../docs-reference/runner/ru/runner-up.mdx | 14 ++ .../docs-reference/runner/zh/runner-run.mdx | 14 ++ .../docs-reference/runner/zh/runner-up.mdx | 14 ++ services/runner/src/parser/mod.rs | 8 + services/runner/src/parser/up.rs | 8 + services/runner/src/runner/run.rs | 4 + services/runner/src/runner/up.rs | 4 + 33 files changed, 494 insertions(+), 31 deletions(-) diff --git a/plus/bencher_runner/src/config.rs b/plus/bencher_runner/src/config.rs index ab5396e17..1e2614755 100644 --- a/plus/bencher_runner/src/config.rs +++ b/plus/bencher_runner/src/config.rs @@ -158,6 +158,12 @@ pub struct Config { /// This field is not serialized. #[serde(skip, default = "default_state_dir")] pub state_dir: Utf8PathBuf, + + /// The unprivileged uid and gid the jailed VMM drops to. + /// + /// This field is not serialized. + #[serde(skip)] + pub jail_user: crate::jail::JailUser, } fn default_vcpus() -> Cpu { @@ -267,6 +273,7 @@ impl Config { sandbox_log_level: SandboxLogLevel::default(), sandbox: None, state_dir: default_state_dir(), + jail_user: crate::jail::JailUser::default(), } } @@ -464,6 +471,13 @@ impl Config { self } + /// Set the unprivileged uid and gid the jailed VMM drops to. + #[must_use] + pub fn with_jail_user(mut self, jail_user: crate::jail::JailUser) -> Self { + self.jail_user = jail_user; + self + } + /// Set the CPU layout for core isolation. /// /// When set, the Firecracker process will be pinned to benchmark cores diff --git a/plus/bencher_runner/src/firecracker/mod.rs b/plus/bencher_runner/src/firecracker/mod.rs index 8cbcf0aac..7fdf1197f 100644 --- a/plus/bencher_runner/src/firecracker/mod.rs +++ b/plus/bencher_runner/src/firecracker/mod.rs @@ -32,7 +32,7 @@ use std::time::{Duration, Instant}; use camino::Utf8PathBuf; use crate::cpu::CpuLayout; -use crate::jail::{CgroupManager, JailPaths}; +use crate::jail::{CgroupManager, JailPaths, JailUser}; use crate::metrics::{self, RunMetrics}; pub use error::FirecrackerError; @@ -67,6 +67,8 @@ pub struct FirecrackerJobConfig { pub vm_id: String, /// Both views of every file inside the jail chroot. pub jail: JailPaths, + /// The unprivileged uid and gid the VMM drops to. + pub jail_user: JailUser, /// The jailer's `--chroot-base-dir`. pub chroot_base_dir: Utf8PathBuf, /// Handle of the empty network namespace the VMM joins. @@ -175,6 +177,7 @@ pub fn run_firecracker( jailer_bin: &config.jailer_bin, exec_file: &config.firecracker_bin, vm_id, + jail_user: config.jail_user, chroot_base_dir: &config.chroot_base_dir, netns: &config.netns, api_socket: jail.api_socket(), @@ -226,7 +229,7 @@ pub fn run_firecracker( // Firecracker connects out to these as the unprivileged jail user, so it // needs write access to the inodes. After bind and before InstanceStart. vsock_listener - .chown_to_jail() + .chown_to_jail(config.jail_user) .map_err(FirecrackerError::Chown)?; // Step 4: Boot the VM diff --git a/plus/bencher_runner/src/firecracker/process.rs b/plus/bencher_runner/src/firecracker/process.rs index 9092a6b70..381683ba4 100644 --- a/plus/bencher_runner/src/firecracker/process.rs +++ b/plus/bencher_runner/src/firecracker/process.rs @@ -11,7 +11,7 @@ use camino::Utf8Path; use crate::firecracker::client::FirecrackerClient; use crate::firecracker::config::{Action, ActionType}; use crate::firecracker::error::FirecrackerError; -use crate::jail::{HostPath, JAIL_GID, JAIL_UID, JailFile}; +use crate::jail::{HostPath, JailFile, JailUser}; /// Everything needed to spawn the VMM under the jailer. #[derive(Debug)] @@ -27,6 +27,8 @@ pub struct JailedSpawn<'a> { pub exec_file: &'a Utf8Path, /// The jailer `--id`, which is also the chroot name and the cgroup name. pub vm_id: &'a str, + /// The unprivileged uid and gid the VMM drops to. + pub jail_user: JailUser, /// The jailer `--chroot-base-dir`. pub chroot_base_dir: &'a Utf8Path, /// Handle of the empty network namespace the VMM joins. @@ -75,6 +77,7 @@ impl FirecrackerProcess { jailer_bin, exec_file: _, vm_id: _, + jail_user: _, chroot_base_dir: _, netns: _, api_socket, @@ -224,9 +227,9 @@ fn jailer_args(spawn: &JailedSpawn<'_>) -> Vec { "--exec-file".to_owned(), spawn.exec_file.to_string(), "--uid".to_owned(), - JAIL_UID.to_string(), + spawn.jail_user.uid.to_string(), "--gid".to_owned(), - JAIL_GID.to_string(), + spawn.jail_user.gid.to_string(), "--chroot-base-dir".to_owned(), spawn.chroot_base_dir.to_string(), "--netns".to_owned(), @@ -270,6 +273,7 @@ mod tests { jailer_bin: Utf8Path::new("/tmp/work/jailer"), exec_file: Utf8Path::new("/tmp/work/firecracker"), vm_id: "vm-1", + jail_user: JailUser::default(), chroot_base_dir: Utf8Path::new("/var/lib/bencher-runner/jail"), netns: Utf8Path::new("/run/netns/bencher-jail"), api_socket: jail.api_socket(), @@ -299,8 +303,8 @@ mod tests { value_of(&args, "--exec-file"), Some("/tmp/work/firecracker") ); - assert_eq!(value_of(&args, "--uid"), Some("60613")); - assert_eq!(value_of(&args, "--gid"), Some("60613")); + assert_eq!(value_of(&args, "--uid"), Some("61016")); + assert_eq!(value_of(&args, "--gid"), Some("61016")); assert_eq!( value_of(&args, "--chroot-base-dir"), Some("/var/lib/bencher-runner/jail") diff --git a/plus/bencher_runner/src/firecracker/vsock.rs b/plus/bencher_runner/src/firecracker/vsock.rs index db325fef2..79c1e435f 100644 --- a/plus/bencher_runner/src/firecracker/vsock.rs +++ b/plus/bencher_runner/src/firecracker/vsock.rs @@ -20,8 +20,8 @@ use camino::Utf8Path; use nix::poll::{PollFd, PollFlags, PollTimeout, poll}; use crate::firecracker::error::FirecrackerError; -use crate::jail::HostPath; use crate::jail::chroot::chown_to_jail; +use crate::jail::{HostPath, JailUser}; /// Poll timeout for vsock listeners (50ms). /// @@ -282,9 +282,9 @@ impl VsockListener { /// the only directory it traverses is `/`, which the jailer chowns itself, /// so the inodes are all that is left to hand over. Must run after bind /// and before `InstanceStart`. - pub fn chown_to_jail(&self) -> Result<(), crate::error::JailError> { + pub fn chown_to_jail(&self, jail_user: JailUser) -> Result<(), crate::error::JailError> { for port in ports::ALL { - chown_to_jail(Utf8Path::new(&self.socket_path(port)))?; + chown_to_jail(Utf8Path::new(&self.socket_path(port)), jail_user)?; } Ok(()) } diff --git a/plus/bencher_runner/src/jail/chroot.rs b/plus/bencher_runner/src/jail/chroot.rs index 2bf7d5db2..10f2d1606 100644 --- a/plus/bencher_runner/src/jail/chroot.rs +++ b/plus/bencher_runner/src/jail/chroot.rs @@ -14,7 +14,7 @@ use std::os::unix::fs::{PermissionsExt as _, chown}; use camino::{Utf8Path, Utf8PathBuf}; use crate::error::JailError; -use crate::jail::{JAIL_GID, JAIL_UID, StateDir}; +use crate::jail::{JailUser, StateDir}; /// A job's chroot tree, removed when this value is dropped. /// @@ -76,8 +76,9 @@ impl Drop for JailDir { /// they were created with, which is root. Every artifact Firecracker touches /// has to be handed over explicitly, and getting it wrong produces an opaque /// boot failure, so each one is checked. -pub fn chown_to_jail(path: &Utf8Path) -> Result<(), JailError> { - chown(path, Some(JAIL_UID), Some(JAIL_GID)).map_err(|e| JailError::ChownJail { +pub fn chown_to_jail(path: &Utf8Path, jail_user: JailUser) -> Result<(), JailError> { + let JailUser { uid, gid } = jail_user; + chown(path, Some(uid), Some(gid)).map_err(|e| JailError::ChownJail { path: path.to_owned(), source: e, }) diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 214c34f84..4456d0043 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -37,21 +37,51 @@ use serde::{Deserialize, Serialize}; /// Default location of the runner's persistent state directory. pub const DEFAULT_STATE_DIR: &str = "/var/lib/bencher-runner"; -/// The unprivileged uid the jailed Firecracker VMM runs as. +/// Default unprivileged uid the jailed Firecracker VMM runs as. /// /// One dedicated id, not one per job: jobs run serially and each gets a fresh /// chroot that is swept, so a per-job allocator adds a scheme without closing -/// a live vector. The value sits in the gap between the ids `systemd-homed` -/// claims and the `DynamicUser` range (61184-65519), and well clear of both -/// the regular user range and `nobody` (65534), so it is unlikely to collide -/// with an account that owns anything on the host. No passwd entry is needed: -/// the jailer sets the numeric id directly. -pub const JAIL_UID: u32 = 60613; - -/// The unprivileged gid the jailed Firecracker VMM runs as. +/// a live vector. /// -/// See [`JAIL_UID`]. -pub const JAIL_GID: u32 = 60613; +/// The number is Bencher's historic default self-hosted API server port, +/// retired in favor of the IANA-registered 6610, so it reads as a project +/// convention rather than an arbitrary pick. It also lands in the unallocated +/// gap between the ids `systemd-homed` claims (60001-60513) and the +/// `DynamicUser` range (61184-65519), clear of both the regular user range and +/// `nobody` (65534). No passwd entry is needed: the jailer sets the numeric id +/// directly. +/// +/// This is a default rather than a fixed constant because self-hosted runners +/// land on hardware whose id allocation Bencher does not control. See +/// `--jail-uid`. +pub const DEFAULT_JAIL_UID: u32 = 61016; + +/// Default unprivileged gid the jailed Firecracker VMM runs as. +/// +/// See [`DEFAULT_JAIL_UID`]. +pub const DEFAULT_JAIL_GID: u32 = 61016; + +/// The uid and gid the jailed Firecracker VMM drops to. +/// +/// A host process owning this uid can signal the VMM and, depending on the +/// `ptrace` scope, trace it, so it must not be an id the host allocates to +/// anything else. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JailUser { + /// The uid the VMM drops to. + pub uid: u32, + /// The gid the VMM drops to. + pub gid: u32, +} + +impl Default for JailUser { + fn default() -> Self { + Self { + uid: DEFAULT_JAIL_UID, + gid: DEFAULT_JAIL_GID, + } + } +} /// Prepare the host for jailed execution. /// @@ -69,21 +99,89 @@ pub const JAIL_GID: u32 = 60613; /// it. Holding the lock makes serialization a constraint rather than an /// assumption. #[cfg(target_os = "linux")] -pub fn prepare_host(state_dir: &camino::Utf8Path) -> Result<(), crate::error::JailError> { +pub fn prepare_host( + state_dir: &camino::Utf8Path, + jail_user: JailUser, +) -> Result<(), crate::error::JailError> { let state = StateDir::new(state_dir.to_owned()); state.create()?; + warn_on_named_account(jail_user); + let _lock = JailLock::acquire(state.path())?; state::sweep_jails(&state.jail_parent()); netns::ensure()?; Ok(()) } +/// Warn when the jail uid or gid belongs to a named account. +/// +/// The jailer needs no passwd entry, so a name resolving here is the cheap +/// signal that the host allocates ids in this range: whatever owns that +/// account can signal the VMM and may be able to trace it. A warning rather +/// than a refusal, because an operator who deliberately created the account is +/// a legitimate setup and only they can tell the two apart. +#[cfg(target_os = "linux")] +#[expect(clippy::print_stderr, reason = "host preparation prints diagnostics")] +fn warn_on_named_account(jail_user: JailUser) { + let JailUser { uid, gid } = jail_user; + if let Some(name) = passwd_name(uid) { + eprintln!( + "Warning: jail uid {uid} belongs to the existing account '{name}'. That account can signal the jailed VMM; pass --jail-uid to pick an unallocated id." + ); + } + if let Some(name) = group_name(gid) { + eprintln!( + "Warning: jail gid {gid} belongs to the existing group '{name}'. Pass --jail-gid to pick an unallocated id." + ); + } +} + +/// The account name for a uid, read from `/etc/passwd`. +/// +/// Deliberately not a `getpwuid` call: the runner ships as a self-contained +/// binary and pulling in NSS would make it depend on the host's resolver +/// configuration. A local account is what matters here, and that is the file. +#[cfg(target_os = "linux")] +fn passwd_name(uid: u32) -> Option { + lookup_name("/etc/passwd", uid) +} + +/// The group name for a gid, read from `/etc/group`. +#[cfg(target_os = "linux")] +fn group_name(gid: u32) -> Option { + lookup_name("/etc/group", gid) +} + +/// Find the name whose record carries `id` in a colon-separated database. +/// +/// Both `/etc/passwd` and `/etc/group` put the name first and the numeric id +/// third. +#[cfg(target_os = "linux")] +fn lookup_name(path: &str, id: u32) -> Option { + let database = std::fs::read_to_string(path).ok()?; + lookup_name_in(&database, id) +} + +/// Find the name whose record carries `id`, given the database contents. +#[cfg(target_os = "linux")] +fn lookup_name_in(database: &str, id: u32) -> Option { + database.lines().find_map(|line| { + let mut fields = line.split(':'); + let name = fields.next()?; + let _password = fields.next()?; + (fields.next()?.parse::().ok()? == id).then(|| name.to_owned()) + }) +} + /// Prepare the host for jailed execution. /// /// The jail is Linux-only, as is the VM executor it protects. #[cfg(not(target_os = "linux"))] -pub fn prepare_host(_state_dir: &camino::Utf8Path) -> Result<(), crate::error::JailError> { +pub fn prepare_host( + _state_dir: &camino::Utf8Path, + _jail_user: JailUser, +) -> Result<(), crate::error::JailError> { Ok(()) } @@ -183,6 +281,45 @@ impl ResourceLimits { mod tests { use super::*; + #[cfg(target_os = "linux")] + #[test] + fn the_default_jail_user_is_outside_the_allocated_ranges() { + // systemd-homed takes 60001-60513 and DynamicUser takes 61184-65519. + // An id inside either would collide with something the host allocates. + for id in [DEFAULT_JAIL_UID, DEFAULT_JAIL_GID] { + assert_eq!(id, 61016, "the jail id is a project convention"); + assert!(id > 60513, "{id} must clear the systemd-homed range"); + assert!(id < 61184, "{id} must clear the DynamicUser range"); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn a_named_account_is_found_by_id() { + let passwd = "root:x:0:0:root:/root:/bin/bash\nbuild:x:61016:61016:CI build user:/home/build:/bin/sh\n"; + + assert_eq!(lookup_name_in(passwd, 61016).as_deref(), Some("build")); + assert_eq!(lookup_name_in(passwd, 0).as_deref(), Some("root")); + } + + #[cfg(target_os = "linux")] + #[test] + fn an_unallocated_id_has_no_name() { + let passwd = + "root:x:0:0:root:/root:/bin/bash\ndaemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin\n"; + + assert_eq!(lookup_name_in(passwd, 61016), None); + } + + #[cfg(target_os = "linux")] + #[test] + fn malformed_records_are_skipped() { + let passwd = "\nnot-a-record\nshort:x\nbuild:x:61016:61016::/home/build:/bin/sh\n"; + + assert_eq!(lookup_name_in(passwd, 61016).as_deref(), Some("build")); + assert_eq!(lookup_name_in("", 61016), None); + } + #[test] fn resource_limits_defaults() { let limits = ResourceLimits::default(); diff --git a/plus/bencher_runner/src/lib.rs b/plus/bencher_runner/src/lib.rs index 6c96d3232..4feb878ca 100644 --- a/plus/bencher_runner/src/lib.rs +++ b/plus/bencher_runner/src/lib.rs @@ -62,7 +62,7 @@ pub use config::Config; #[cfg(feature = "plus")] pub use error::{ConfigError, ExecutionError, JailError, RunnerError}; #[cfg(feature = "plus")] -pub use jail::{DEFAULT_STATE_DIR, ResourceLimits}; +pub use jail::{DEFAULT_JAIL_GID, DEFAULT_JAIL_UID, DEFAULT_STATE_DIR, JailUser, ResourceLimits}; #[cfg(feature = "plus")] pub use log_level::SandboxLogLevel; #[cfg(feature = "plus")] diff --git a/plus/bencher_runner/src/run.rs b/plus/bencher_runner/src/run.rs index f50f204d1..b648daac4 100644 --- a/plus/bencher_runner/src/run.rs +++ b/plus/bencher_runner/src/run.rs @@ -73,6 +73,8 @@ pub struct RunArgs { pub sandbox: Option, /// The runner's persistent state directory. pub state_dir: Utf8PathBuf, + /// The unprivileged uid and gid the jailed VMM drops to. + pub jail_user: crate::jail::JailUser, } /// Build a `Config` from CLI `RunArgs`. @@ -124,6 +126,7 @@ fn build_config_from_run_args(args: &RunArgs) -> Result Result<(), RunnerError> { // execute on the host without a jail, and that path has no business // requiring the runner's state directory. if config.sandbox.is_some() { - crate::jail::prepare_host(&config.state_dir)?; + crate::jail::prepare_host(&config.state_dir, config.jail_user)?; } // Detect the CPU layout after tuning (disabling SMT changes the core diff --git a/plus/bencher_runner/src/up/job.rs b/plus/bencher_runner/src/up/job.rs index 50a1ee0ba..ec462b87e 100644 --- a/plus/bencher_runner/src/up/job.rs +++ b/plus/bencher_runner/src/up/job.rs @@ -330,6 +330,7 @@ fn build_config_from_job( grace_period, sandbox_log_level, state_dir, + jail_user, } = up_config; let spec = &job.spec; @@ -371,6 +372,7 @@ fn build_config_from_job( // Pass through the runner's state directory: the jail chroot for the // job is built under it. runner_config = runner_config.with_state_dir(state_dir.clone()); + runner_config = runner_config.with_jail_user(*jail_user); // Pass through CPU layout for core isolation if let Some(cpu_layout) = cpu_layout @@ -574,6 +576,7 @@ mod tests { update_channel: bencher_valid::UpdateChannel::default(), max_download_size: None, state_dir: Utf8PathBuf::from(crate::jail::DEFAULT_STATE_DIR), + jail_user: crate::jail::JailUser::default(), } } @@ -841,6 +844,24 @@ mod tests { assert_eq!(result.state_dir, crate::jail::DEFAULT_STATE_DIR); } + #[test] + fn jail_user_passed_through() { + // A host that allocates ids in the default range needs the override to + // reach the job, or the VMM shares a uid with a local account that can + // signal it. + let mut up_config = test_up_config(); + up_config.jail_user = crate::jail::JailUser { + uid: 4242, + gid: 4243, + }; + let job = test_job(1, mib_to_bytes(512), mib_to_bytes(1024), 300, false); + + let result = build_config_from_job(&up_config, &job).unwrap(); + + assert_eq!(result.jail_user.uid, 4242); + assert_eq!(result.jail_user.gid, 4243); + } + #[test] fn cpu_layout_passed_through() { let up_config = test_up_config(); diff --git a/plus/bencher_runner/src/up/mod.rs b/plus/bencher_runner/src/up/mod.rs index 0915ac203..6d0b140fc 100644 --- a/plus/bencher_runner/src/up/mod.rs +++ b/plus/bencher_runner/src/up/mod.rs @@ -69,6 +69,8 @@ pub struct UpConfig { pub max_download_size: Option, /// The runner's persistent state directory. pub state_dir: camino::Utf8PathBuf, + /// The unprivileged uid and gid the jailed VMM drops to. + pub jail_user: crate::jail::JailUser, } pub struct Up { @@ -110,7 +112,8 @@ impl Up { // that exited without unwinding, and ensure the empty network // namespace. The daemon claims sandboxed jobs, so this is required // before the first one arrives, not on demand. - crate::jail::prepare_host(&self.config.state_dir).map_err(crate::RunnerError::from)?; + crate::jail::prepare_host(&self.config.state_dir, self.config.jail_user) + .map_err(crate::RunnerError::from)?; println!(" State directory: {}", self.config.state_dir); // Serialize host-global tuning across runner processes. Declared diff --git a/plus/bencher_runner/src/vm.rs b/plus/bencher_runner/src/vm.rs index 67259f904..d5d2c1efd 100644 --- a/plus/bencher_runner/src/vm.rs +++ b/plus/bencher_runner/src/vm.rs @@ -92,8 +92,8 @@ pub fn vm_execute( // The jailer chowns the chroot root and the device nodes it creates, but // not what the runner placed inside, so hand over each artifact // explicitly: Firecracker writes the rootfs and reads the kernel. - chroot::chown_to_jail(rootfs_dest)?; - chroot::chown_to_jail(kernel_dest)?; + chroot::chown_to_jail(rootfs_dest, config.jail_user)?; + chroot::chown_to_jail(kernel_dest, config.jail_user)?; // Step 7-8: Build Firecracker config and run the microVM let fc_config = build_firecracker_config(config, work_dir, vm_id, &state_dir, jail)?; @@ -153,6 +153,7 @@ fn build_firecracker_config( jailer_bin, vm_id, jail, + jail_user: config.jail_user, chroot_base_dir: state_dir.chroot_base(), netns: netns::handle_path(), vcpus, diff --git a/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx index 3a9928cfc..70f755de5 100644 --- a/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx @@ -40,6 +40,20 @@ und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor Standardmäßig wird `/var/lib/bencher-runner` verwendet. Kann auch mit der Umgebungsvariable `BENCHER_STATE_DIR` gesetzt werden. +### `--jail-uid ` + +Die unprivilegierte Benutzer-ID, auf die der Sandbox-Prozess wechselt. +Standardmäßig wird `61016` verwendet. +Ein Prozess auf dem Host, dem diese Benutzer-ID gehört, kann Signale an die Sandbox senden, +setzen Sie sie also auf eine ID, die der Host nichts anderem zuweist. +Kann auch mit der Umgebungsvariable `BENCHER_JAIL_UID` gesetzt werden. + +### `--jail-gid ` + +Die unprivilegierte Gruppen-ID, auf die der Sandbox-Prozess wechselt. +Standardmäßig wird `61016` verwendet. +Kann auch mit der Umgebungsvariable `BENCHER_JAIL_GID` gesetzt werden. + ### `--iter ` Die Anzahl der auszuführenden Benchmark-Iterationen. diff --git a/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx index a1cbe4e07..1d3c1f5d0 100644 --- a/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx @@ -43,6 +43,20 @@ und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor Standardmäßig wird `/var/lib/bencher-runner` verwendet. Kann auch mit der Umgebungsvariable `BENCHER_STATE_DIR` gesetzt werden. +### `--jail-uid ` + +Die unprivilegierte Benutzer-ID, auf die der Sandbox-Prozess wechselt. +Standardmäßig wird `61016` verwendet. +Ein Prozess auf dem Host, dem diese Benutzer-ID gehört, kann Signale an die Sandbox senden, +setzen Sie sie also auf eine ID, die der Host nichts anderem zuweist. +Kann auch mit der Umgebungsvariable `BENCHER_JAIL_UID` gesetzt werden. + +### `--jail-gid ` + +Die unprivilegierte Gruppen-ID, auf die der Sandbox-Prozess wechselt. +Standardmäßig wird `61016` verwendet. +Kann auch mit der Umgebungsvariable `BENCHER_JAIL_GID` gesetzt werden. + ### `--danger-allow-no-sandbox` Erlaubt das Ausführen von Jobs ohne [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx index 51bdf1b4f..c16060ed4 100644 --- a/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx @@ -40,6 +40,20 @@ and jails left behind by an unclean exit are swept from it before a Job runs. By default, `/var/lib/bencher-runner` is used. Can also be set with the `BENCHER_STATE_DIR` environment variable. +### `--jail-uid ` + +The unprivileged user ID the sandbox process drops to. +By default, `61016` is used. +A process on the host that owns this user ID can signal the sandbox, +so set it to an ID the host does not allocate to anything else. +Can also be set with the `BENCHER_JAIL_UID` environment variable. + +### `--jail-gid ` + +The unprivileged group ID the sandbox process drops to. +By default, `61016` is used. +Can also be set with the `BENCHER_JAIL_GID` environment variable. + ### `--iter ` The number of benchmark iterations to execute. diff --git a/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx index 68bd2cf86..313387328 100644 --- a/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx @@ -43,6 +43,20 @@ and jails left behind by an unclean exit are swept from it before a Job runs. By default, `/var/lib/bencher-runner` is used. Can also be set with the `BENCHER_STATE_DIR` environment variable. +### `--jail-uid ` + +The unprivileged user ID the sandbox process drops to. +By default, `61016` is used. +A process on the host that owns this user ID can signal the sandbox, +so set it to an ID the host does not allocate to anything else. +Can also be set with the `BENCHER_JAIL_UID` environment variable. + +### `--jail-gid ` + +The unprivileged group ID the sandbox process drops to. +By default, `61016` is used. +Can also be set with the `BENCHER_JAIL_GID` environment variable. + ### `--danger-allow-no-sandbox` Allow executing Jobs without a [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx index ea020e287..cbe53278b 100644 --- a/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx @@ -40,6 +40,20 @@ y las jaulas que quedaron tras una salida no limpia se eliminan de él antes de Por defecto, se usa `/var/lib/bencher-runner`. También se puede establecer con la variable de entorno `BENCHER_STATE_DIR`. +### `--jail-uid ` + +El ID de usuario sin privilegios al que cambia el proceso del sandbox. +Por defecto, se usa `61016`. +Un proceso del host que sea propietario de ese ID de usuario puede enviar señales al sandbox, +así que asígnele un ID que el host no destine a ninguna otra cosa. +También se puede establecer con la variable de entorno `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +El ID de grupo sin privilegios al que cambia el proceso del sandbox. +Por defecto, se usa `61016`. +También se puede establecer con la variable de entorno `BENCHER_JAIL_GID`. + ### `--iter ` El número de iteraciones de benchmark a ejecutar. diff --git a/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx index b95cc5edb..1f1442d8d 100644 --- a/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx @@ -43,6 +43,20 @@ y las jaulas que quedaron tras una salida no limpia se eliminan de él antes de Por defecto, se usa `/var/lib/bencher-runner`. También se puede establecer con la variable de entorno `BENCHER_STATE_DIR`. +### `--jail-uid ` + +El ID de usuario sin privilegios al que cambia el proceso del sandbox. +Por defecto, se usa `61016`. +Un proceso del host que sea propietario de ese ID de usuario puede enviar señales al sandbox, +así que asígnele un ID que el host no destine a ninguna otra cosa. +También se puede establecer con la variable de entorno `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +El ID de grupo sin privilegios al que cambia el proceso del sandbox. +Por defecto, se usa `61016`. +También se puede establecer con la variable de entorno `BENCHER_JAIL_GID`. + ### `--danger-allow-no-sandbox` Permite ejecutar Jobs sin un [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx index 51a4a912a..cdee6d8ab 100644 --- a/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx @@ -40,6 +40,20 @@ et les prisons laissées par un arrêt brutal en sont supprimées avant l'exécu Par défaut, `/var/lib/bencher-runner` est utilisé. Peut également être défini avec la variable d'environnement `BENCHER_STATE_DIR`. +### `--jail-uid ` + +L'identifiant d'utilisateur non privilégié que le processus du bac à sable adopte. +Par défaut, `61016` est utilisé. +Un processus de l'hôte possédant cet identifiant peut envoyer des signaux au bac à sable, +choisissez donc un identifiant que l'hôte n'attribue à rien d'autre. +Peut également être défini avec la variable d'environnement `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +L'identifiant de groupe non privilégié que le processus du bac à sable adopte. +Par défaut, `61016` est utilisé. +Peut également être défini avec la variable d'environnement `BENCHER_JAIL_GID`. + ### `--iter ` Le nombre d'itérations de benchmark à exécuter. diff --git a/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx index 3247b3f5c..e3aaea32a 100644 --- a/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx @@ -43,6 +43,20 @@ et les prisons laissées par un arrêt brutal en sont supprimées avant l'exécu Par défaut, `/var/lib/bencher-runner` est utilisé. Peut également être défini avec la variable d'environnement `BENCHER_STATE_DIR`. +### `--jail-uid ` + +L'identifiant d'utilisateur non privilégié que le processus du bac à sable adopte. +Par défaut, `61016` est utilisé. +Un processus de l'hôte possédant cet identifiant peut envoyer des signaux au bac à sable, +choisissez donc un identifiant que l'hôte n'attribue à rien d'autre. +Peut également être défini avec la variable d'environnement `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +L'identifiant de groupe non privilégié que le processus du bac à sable adopte. +Par défaut, `61016` est utilisé. +Peut également être défini avec la variable d'environnement `BENCHER_JAIL_GID`. + ### `--danger-allow-no-sandbox` Autorise l'exécution de Jobs sans [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx index 99f2e23f2..9d7de68a6 100644 --- a/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx @@ -40,6 +40,20 @@ Runner の永続的な状態ディレクトリ。 デフォルトでは `/var/lib/bencher-runner` が使用されます。 環境変数 `BENCHER_STATE_DIR` でも設定できます。 +### `--jail-uid ` + +サンドボックスプロセスが移行する非特権ユーザー ID。 +デフォルトでは `61016` が使用されます。 +このユーザー ID を所有するホスト上のプロセスはサンドボックスにシグナルを送れるため、 +ホストが他の用途に割り当てていない ID を指定してください。 +環境変数 `BENCHER_JAIL_UID` でも設定できます。 + +### `--jail-gid ` + +サンドボックスプロセスが移行する非特権グループ ID。 +デフォルトでは `61016` が使用されます。 +環境変数 `BENCHER_JAIL_GID` でも設定できます。 + ### `--iter ` 実行するベンチマークの反復回数。 diff --git a/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx index f37039cd1..abd2ee11c 100644 --- a/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx @@ -42,6 +42,20 @@ Runner の永続的な状態ディレクトリ。 デフォルトでは `/var/lib/bencher-runner` が使用されます。 環境変数 `BENCHER_STATE_DIR` でも設定できます。 +### `--jail-uid ` + +サンドボックスプロセスが移行する非特権ユーザー ID。 +デフォルトでは `61016` が使用されます。 +このユーザー ID を所有するホスト上のプロセスはサンドボックスにシグナルを送れるため、 +ホストが他の用途に割り当てていない ID を指定してください。 +環境変数 `BENCHER_JAIL_UID` でも設定できます。 + +### `--jail-gid ` + +サンドボックスプロセスが移行する非特権グループ ID。 +デフォルトでは `61016` が使用されます。 +環境変数 `BENCHER_JAIL_GID` でも設定できます。 + ### `--danger-allow-no-sandbox` [Sandbox][sandbox] なしでの Job の実行を許可します。 diff --git a/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx index c95316de5..40525fd5a 100644 --- a/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx @@ -40,6 +40,20 @@ Runner의 영구 상태 디렉터리입니다. 기본적으로 `/var/lib/bencher-runner`가 사용됩니다. `BENCHER_STATE_DIR` 환경 변수로도 설정할 수 있습니다. +### `--jail-uid ` + +샌드박스 프로세스가 전환할 비특권 사용자 ID입니다. +기본적으로 `61016`이 사용됩니다. +이 사용자 ID를 소유한 호스트의 프로세스는 샌드박스에 시그널을 보낼 수 있으므로, +호스트가 다른 용도로 할당하지 않는 ID를 지정하십시오. +`BENCHER_JAIL_UID` 환경 변수로도 설정할 수 있습니다. + +### `--jail-gid ` + +샌드박스 프로세스가 전환할 비특권 그룹 ID입니다. +기본적으로 `61016`이 사용됩니다. +`BENCHER_JAIL_GID` 환경 변수로도 설정할 수 있습니다. + ### `--iter ` 실행할 벤치마크 반복 횟수입니다. diff --git a/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx index ad18677b1..bc801e149 100644 --- a/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx @@ -43,6 +43,20 @@ Runner의 영구 상태 디렉터리입니다. 기본적으로 `/var/lib/bencher-runner`가 사용됩니다. `BENCHER_STATE_DIR` 환경 변수로도 설정할 수 있습니다. +### `--jail-uid ` + +샌드박스 프로세스가 전환할 비특권 사용자 ID입니다. +기본적으로 `61016`이 사용됩니다. +이 사용자 ID를 소유한 호스트의 프로세스는 샌드박스에 시그널을 보낼 수 있으므로, +호스트가 다른 용도로 할당하지 않는 ID를 지정하십시오. +`BENCHER_JAIL_UID` 환경 변수로도 설정할 수 있습니다. + +### `--jail-gid ` + +샌드박스 프로세스가 전환할 비특권 그룹 ID입니다. +기본적으로 `61016`이 사용됩니다. +`BENCHER_JAIL_GID` 환경 변수로도 설정할 수 있습니다. + ### `--danger-allow-no-sandbox` [Sandbox][sandbox] 없이 Job 실행을 허용합니다. diff --git a/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx index 269798242..d7a1ce6e7 100644 --- a/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx @@ -40,6 +40,20 @@ e as jails deixadas por um encerramento não limpo são removidas dele antes de Por padrão, `/var/lib/bencher-runner` é usado. Também pode ser definido com a variável de ambiente `BENCHER_STATE_DIR`. +### `--jail-uid ` + +O ID de usuário sem privilégios para o qual o processo do sandbox muda. +Por padrão, `61016` é usado. +Um processo no host que seja dono desse ID de usuário pode enviar sinais ao sandbox, +portanto defina um ID que o host não destine a mais nada. +Também pode ser definido com a variável de ambiente `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +O ID de grupo sem privilégios para o qual o processo do sandbox muda. +Por padrão, `61016` é usado. +Também pode ser definido com a variável de ambiente `BENCHER_JAIL_GID`. + ### `--iter ` O número de iterações de benchmark a executar. diff --git a/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx index 9ef519f8a..214442783 100644 --- a/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx @@ -43,6 +43,20 @@ e as jails deixadas por um encerramento não limpo são removidas dele antes de Por padrão, `/var/lib/bencher-runner` é usado. Também pode ser definido com a variável de ambiente `BENCHER_STATE_DIR`. +### `--jail-uid ` + +O ID de usuário sem privilégios para o qual o processo do sandbox muda. +Por padrão, `61016` é usado. +Um processo no host que seja dono desse ID de usuário pode enviar sinais ao sandbox, +portanto defina um ID que o host não destine a mais nada. +Também pode ser definido com a variável de ambiente `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +O ID de grupo sem privilégios para o qual o processo do sandbox muda. +Por padrão, `61016` é usado. +Também pode ser definido com a variável de ambiente `BENCHER_JAIL_GID`. + ### `--danger-allow-no-sandbox` Permite executar Jobs sem um [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx index 936714ca3..7f0b1920b 100644 --- a/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx @@ -40,6 +40,20 @@ JWT-токен для аутентификации в реестре при за По умолчанию используется `/var/lib/bencher-runner`. Также может быть задан переменной окружения `BENCHER_STATE_DIR`. +### `--jail-uid ` + +Непривилегированный идентификатор пользователя, на который переходит процесс песочницы. +По умолчанию используется `61016`. +Процесс на хосте, которому принадлежит этот идентификатор, может посылать сигналы песочнице, +поэтому выберите идентификатор, который хост ни на что другое не выделяет. +Также может быть задан переменной окружения `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +Непривилегированный идентификатор группы, на который переходит процесс песочницы. +По умолчанию используется `61016`. +Также может быть задан переменной окружения `BENCHER_JAIL_GID`. + ### `--iter ` Количество итераций бенчмарка для выполнения. diff --git a/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx index 62f66e1e2..f7e983924 100644 --- a/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx @@ -43,6 +43,20 @@ UUID или slug Runner, от имени которого работать. По умолчанию используется `/var/lib/bencher-runner`. Также может быть задан переменной окружения `BENCHER_STATE_DIR`. +### `--jail-uid ` + +Непривилегированный идентификатор пользователя, на который переходит процесс песочницы. +По умолчанию используется `61016`. +Процесс на хосте, которому принадлежит этот идентификатор, может посылать сигналы песочнице, +поэтому выберите идентификатор, который хост ни на что другое не выделяет. +Также может быть задан переменной окружения `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +Непривилегированный идентификатор группы, на который переходит процесс песочницы. +По умолчанию используется `61016`. +Также может быть задан переменной окружения `BENCHER_JAIL_GID`. + ### `--danger-allow-no-sandbox` Разрешить выполнение Job без [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx index f54a7b90f..85b05cf95 100644 --- a/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx @@ -40,6 +40,20 @@ Runner 的持久状态目录。 默认使用 `/var/lib/bencher-runner`。 也可以通过 `BENCHER_STATE_DIR` 环境变量设置。 +### `--jail-uid ` + +沙箱进程切换到的非特权用户 ID。 +默认使用 `61016`。 +宿主机上拥有该用户 ID 的进程可以向沙箱发送信号， +因此请设置为宿主机未分配给其他用途的 ID。 +也可以通过 `BENCHER_JAIL_UID` 环境变量设置。 + +### `--jail-gid ` + +沙箱进程切换到的非特权组 ID。 +默认使用 `61016`。 +也可以通过 `BENCHER_JAIL_GID` 环境变量设置。 + ### `--iter ` 要执行的基准测试迭代次数。 diff --git a/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx index e16b8acb1..3ae2067d2 100644 --- a/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx @@ -42,6 +42,20 @@ Runner 的持久状态目录。 默认使用 `/var/lib/bencher-runner`。 也可以通过 `BENCHER_STATE_DIR` 环境变量设置。 +### `--jail-uid ` + +沙箱进程切换到的非特权用户 ID。 +默认使用 `61016`。 +宿主机上拥有该用户 ID 的进程可以向沙箱发送信号， +因此请设置为宿主机未分配给其他用途的 ID。 +也可以通过 `BENCHER_JAIL_UID` 环境变量设置。 + +### `--jail-gid ` + +沙箱进程切换到的非特权组 ID。 +默认使用 `61016`。 +也可以通过 `BENCHER_JAIL_GID` 环境变量设置。 + ### `--danger-allow-no-sandbox` 允许在没有 [Sandbox][sandbox] 的情况下执行 Job。 diff --git a/services/runner/src/parser/mod.rs b/services/runner/src/parser/mod.rs index 6deecb3df..102969e39 100644 --- a/services/runner/src/parser/mod.rs +++ b/services/runner/src/parser/mod.rs @@ -66,6 +66,14 @@ pub struct CliRun { #[arg(long, env = "BENCHER_STATE_DIR", default_value = bencher_runner::DEFAULT_STATE_DIR)] pub state_dir: Utf8PathBuf, + /// Unprivileged uid the jailed sandbox process drops to. + #[arg(long, env = "BENCHER_JAIL_UID", default_value_t = bencher_runner::DEFAULT_JAIL_UID)] + pub jail_uid: u32, + + /// Unprivileged gid the jailed sandbox process drops to. + #[arg(long, env = "BENCHER_JAIL_GID", default_value_t = bencher_runner::DEFAULT_JAIL_GID)] + pub jail_gid: u32, + /// Output file paths inside guest (may be repeated). #[arg(long)] pub output: Vec, diff --git a/services/runner/src/parser/up.rs b/services/runner/src/parser/up.rs index e6d5f076d..066fff16a 100644 --- a/services/runner/src/parser/up.rs +++ b/services/runner/src/parser/up.rs @@ -27,6 +27,14 @@ pub struct CliUp { #[arg(long, env = "BENCHER_STATE_DIR", default_value = bencher_runner::DEFAULT_STATE_DIR)] pub state_dir: Utf8PathBuf, + /// Unprivileged uid the jailed sandbox process drops to. + #[arg(long, env = "BENCHER_JAIL_UID", default_value_t = bencher_runner::DEFAULT_JAIL_UID)] + pub jail_uid: u32, + + /// Unprivileged gid the jailed sandbox process drops to. + #[arg(long, env = "BENCHER_JAIL_GID", default_value_t = bencher_runner::DEFAULT_JAIL_GID)] + pub jail_gid: u32, + #[command(flatten)] pub tuning: CliTuning, diff --git a/services/runner/src/runner/run.rs b/services/runner/src/runner/run.rs index 764bf4d09..ad5e3a87d 100644 --- a/services/runner/src/runner/run.rs +++ b/services/runner/src/runner/run.rs @@ -55,6 +55,10 @@ impl TryFrom for Run { sandbox_log_level: task.sandbox_log_level, sandbox: task.sandbox, state_dir: task.state_dir, + jail_user: bencher_runner::JailUser { + uid: task.jail_uid, + gid: task.jail_gid, + }, }, }) } diff --git a/services/runner/src/runner/up.rs b/services/runner/src/runner/up.rs index 55a655163..cfcd43274 100644 --- a/services/runner/src/runner/up.rs +++ b/services/runner/src/runner/up.rs @@ -32,6 +32,10 @@ impl TryFrom for Up { update_channel: task.update_channel, max_download_size: task.max_download_size, state_dir: task.state_dir, + jail_user: bencher_runner::JailUser { + uid: task.jail_uid, + gid: task.jail_gid, + }, }, }) } From 4e982758d07c7ac9f933f4fcc246fe7a79bc9cfd Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 03:33:33 +0000 Subject: [PATCH 11/91] Run the KVM scenarios as root in CI The scenario job ran cargo test-runner scenarios without sudo. Verified on a real kernel that this cannot work now that the sandbox is jailed: run unprivileged, the jailer fails at ChangeFileOwner with EPERM and exits 1, leaving a half-built chroot, and prepare_host does not even get that far since creating the network namespace directory is denied and unshare is not permitted. Run as root the same command builds the chroot at 0700, populates its device nodes, and leaves Firecracker running as the jail uid. The udev rule that makes /dev/kvm world accessible is enough to use KVM unprivileged but not to build the jail around it: the design drops privilege rather than starting without it. The scenario run is elevated on its own rather than making the whole job root. A --build-only mode builds the binaries as the CI user, and the elevated step runs the built harness directly with BENCHER_RUNNER_BIN, so cargo never runs as root and neither the target directory nor cargo's cache is left root-owned. Verified: zero root-owned files under the target directory after the elevated run. The harness now refuses to run unprivileged with a message naming both steps, rather than failing partway through the first scenario at a mknod. It also honors CARGO_TARGET_DIR, which it previously ignored, so a redirected build does not report success and then a missing binary. --- .github/workflows/runner.yml | 14 ++++- tasks/test_runner/src/parser/mod.rs | 8 +++ tasks/test_runner/src/task/scenarios.rs | 70 +++++++++++++++++++++++-- 3 files changed, 88 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index bccb999e1..c6bfe04ce 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -54,8 +54,20 @@ jobs: run: | echo '{"features": {"containerd-snapshotter": true}}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker + - name: Build scenario binaries + run: cargo test-runner scenarios --build-only - name: Run scenarios - run: cargo test-runner scenarios + # The scenarios run as root. The sandbox is built by dropping + # privilege rather than by starting without it: the jailer creates the + # chroot's device nodes with mknod, chowns the tree to the jail user, + # pivot_roots, and joins a network namespace. The udev rule above makes + # /dev/kvm usable by an unprivileged process, which is necessary but + # not sufficient. + # + # Only this step is elevated. The build above stays as the CI user, so + # cargo's cache and target directory are not left root-owned, and + # BENCHER_RUNNER_BIN keeps cargo out of the elevated step entirely. + run: sudo BENCHER_RUNNER_BIN=./target/debug/runner ./target/debug/test_runner scenarios build_runner: name: Build Runner (${{ matrix.build }}) diff --git a/tasks/test_runner/src/parser/mod.rs b/tasks/test_runner/src/parser/mod.rs index 8ad029769..12ef33720 100644 --- a/tasks/test_runner/src/parser/mod.rs +++ b/tasks/test_runner/src/parser/mod.rs @@ -21,4 +21,12 @@ pub struct TaskScenarios { /// List all available scenarios #[clap(long, short)] pub list: bool, + + /// Build the binaries the scenarios need, then exit. + /// + /// The scenarios themselves need root, and building as root would leave + /// cargo's cache and target directory root-owned. This splits the build + /// out so it can run unprivileged before the elevated run. + #[clap(long, conflicts_with_all = ["scenario", "list"])] + pub build_only: bool, } diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index 1c7b9f730..275c989ab 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -73,6 +73,7 @@ struct ScenarioOutput { pub struct Scenarios { scenario: Option, list: bool, + build_only: bool, } impl TryFrom for Scenarios { @@ -82,6 +83,7 @@ impl TryFrom for Scenarios { Ok(Self { scenario: task.scenario, list: task.list, + build_only: task.build_only, }) } } @@ -93,7 +95,29 @@ impl Scenarios { return Ok(()); } - // Check prerequisites + if self.build_only { + let runner_bin = ensure_runner_bin()?; + println!("Built runner: {runner_bin}"); + println!("Run the scenarios with:"); + println!(" sudo {RUNNER_BIN_ENV}={runner_bin} scenarios"); + return Ok(()); + } + + // Check prerequisites. + // + // Root is one of them. The jailer creates the chroot's device nodes + // with mknod, chowns the tree to the jail user, pivot_roots, and joins + // a network namespace, none of which an unprivileged process can do. + // A udev rule that makes /dev/kvm world accessible is enough to *use* + // KVM without root but not to build the jail around it. + if !is_root() { + bail!( + "The scenarios must run as root: the sandbox is built by dropping privilege, not by starting without it.\n\ + Build unprivileged first, then run elevated:\n\ + \x20 cargo test-runner scenarios --build-only\n\ + \x20 sudo {RUNNER_BIN_ENV}=./target/debug/runner ./target/debug/test_runner scenarios" + ); + } if !kvm_available() { bail!("KVM is not available (/dev/kvm not found)"); } @@ -2312,6 +2336,31 @@ fn kvm_available() -> bool { Path::new("/dev/kvm").exists() } +/// The cargo target directory the builds above land in. +/// +/// `CARGO_TARGET_DIR` is honored rather than assumed away: a caller that +/// redirects it would otherwise have the binaries built in one place and +/// looked for in another, and the harness would report a missing binary +/// immediately after reporting a successful build. +fn target_dir() -> Utf8PathBuf { + std::env::var_os("CARGO_TARGET_DIR").map_or_else( + || super::workspace_root().join("target"), + |dir| Utf8PathBuf::from(dir.to_string_lossy().into_owned()), + ) +} + +/// Whether this process is running as root. +fn is_root() -> bool { + #[expect( + unsafe_code, + reason = "geteuid has no std wrapper and cannot fail or touch memory" + )] + // SAFETY: `geteuid` takes no arguments, returns a plain integer, and is + // always successful. + let euid = unsafe { libc::geteuid() }; + euid == 0 +} + /// Check if Docker is available. fn docker_available() -> bool { Command::new("docker") @@ -2429,6 +2478,18 @@ fn run_runner_with_cancel( /// Build bencher-init for the musl target and the runner CLI with `BENCHER_INIT_PATH`, /// then return the path to the runner binary. fn ensure_runner_bin() -> Result { + // The elevated run must not invoke cargo: doing so as root leaves the + // target directory and cargo's cache root-owned, which then breaks the + // unprivileged steps around it. CI builds first and points here. + if let Some(path) = std::env::var_os(RUNNER_BIN_ENV) { + let path = Utf8PathBuf::from(path.to_string_lossy().into_owned()); + if !path.exists() { + bail!("{RUNNER_BIN_ENV} is set to {path}, which does not exist"); + } + println!("Using pre-built runner from {RUNNER_BIN_ENV}: {path}"); + return Ok(path); + } + let workspace_root = super::workspace_root(); let target_triple = super::musl_target_triple()?; @@ -2443,7 +2504,7 @@ fn ensure_runner_bin() -> Result { bail!("cargo build -p bencher_init --target {target_triple} failed"); } - let init_path = workspace_root.join(format!("target/{target_triple}/debug/bencher-init")); + let init_path = target_dir().join(format!("{target_triple}/debug/bencher-init")); if !init_path.exists() { bail!("bencher-init binary not found at {init_path} after build"); } @@ -2460,7 +2521,7 @@ fn ensure_runner_bin() -> Result { bail!("cargo build -p bencher_runner_cli failed"); } - let runner_bin = workspace_root.join("target/debug/runner"); + let runner_bin = target_dir().join("debug/runner"); if !runner_bin.exists() { bail!("Runner binary not found at {runner_bin} after build"); } @@ -2472,6 +2533,9 @@ fn ensure_runner_bin() -> Result { // Jail confinement // --------------------------------------------------------------------------- +/// Environment variable naming a pre-built runner binary. +const RUNNER_BIN_ENV: &str = "BENCHER_RUNNER_BIN"; + /// How long to wait for the jailed VMM to appear before giving up. /// /// Generous: the runner pulls and unpacks the image and builds the rootfs From bd393c04eeccea6ea0b3a44d40c19e6dd3b11366 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 04:06:03 +0000 Subject: [PATCH 12/91] Address jail sockets through a descriptor, not a long path Every scenario but the two new ones failed in CI with a five second timeout that read as Firecracker's fault. It was the path. sockaddr_un.sun_path is 108 bytes and the limit applies to the string handed to bind and connect, before any resolution, so the host view of a jail under a deep state directory blows it at 144 bytes while the chroot view Firecracker uses stays short. The default state directory happens to fit at 91, which is 17 bytes of headroom on a limit nobody had declared. Socket paths are now a third view, built from a descriptor the runner holds open on the chroot: /proc/self/fd//api.sock is about thirty bytes and names the same inode however deep the jail is. The descriptor is O_PATH, held for the life of the job, and its lifetime is enforced by ownership rather than by discipline, because a closed and reused number would silently address a different directory. SocketPath checks every value against the limit at construction and names the limit, the length, and the offending path, so this can never again surface as a mystery timeout. The timeout was also swallowing the real error. An over-long path is rejected by the standard library before any syscall, and the readiness loop retried that for the full five seconds. It now retries only what a not-yet-listening VMM actually produces and fails immediately on anything describing the address itself. --- plus/bencher_runner/src/firecracker/client.rs | 45 ++- plus/bencher_runner/src/firecracker/error.rs | 12 + plus/bencher_runner/src/firecracker/mod.rs | 2 +- .../bencher_runner/src/firecracker/process.rs | 29 +- plus/bencher_runner/src/firecracker/vsock.rs | 73 ++--- plus/bencher_runner/src/jail/paths.rs | 270 ++++++++++++++---- 6 files changed, 325 insertions(+), 106 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/client.rs b/plus/bencher_runner/src/firecracker/client.rs index a9f04c838..1296b15b6 100644 --- a/plus/bencher_runner/src/firecracker/client.rs +++ b/plus/bencher_runner/src/firecracker/client.rs @@ -9,11 +9,9 @@ use std::io::{Read as _, Write as _}; use std::os::unix::net::UnixStream; use std::time::Duration; -use camino::Utf8Path; - use crate::firecracker::config::{Action, BootSource, Drive, MachineConfig, VsockConfig}; use crate::firecracker::error::FirecrackerError; -use crate::jail::HostPath; +use crate::jail::SocketPath; /// Client for the Firecracker REST API. pub struct FirecrackerClient { @@ -24,22 +22,29 @@ impl FirecrackerClient { /// Create a new client for the API socket. /// /// The runner reaches the socket from outside the chroot, so this is the - /// host view; the jailed VMM binds the chroot view of the same file. - pub fn new(socket_path: &HostPath) -> Self { + /// socket view; the jailed VMM binds the chroot view of the same file. + pub fn new(socket_path: &SocketPath) -> Self { Self { socket_path: socket_path.as_str().to_owned(), } } /// Wait for the Firecracker API socket to become ready. + /// + /// Only failures that a not-yet-listening VMM actually produces are + /// retried. Anything else fails immediately: an unusable path is not going + /// to become usable, and retrying it for the whole timeout turns a precise + /// error into a timeout that points at Firecracker instead of at the + /// cause. An over-long socket path is rejected by the standard library + /// before any syscall, which is exactly the case this distinction exists + /// to surface. pub fn wait_for_ready(&self, timeout: Duration) -> Result<(), FirecrackerError> { let start = std::time::Instant::now(); let poll_interval = Duration::from_millis(50); while start.elapsed() < timeout { - if Utf8Path::new(&self.socket_path).exists() { - // Try to connect - if let Ok(mut stream) = UnixStream::connect(&self.socket_path) { + match UnixStream::connect(&self.socket_path) { + Ok(mut stream) => { drop(stream.set_read_timeout(Some(Duration::from_secs(1)))); drop(stream.set_write_timeout(Some(Duration::from_secs(1)))); @@ -52,7 +57,14 @@ impl FirecrackerClient { return Ok(()); } } - } + }, + Err(e) if is_not_listening_yet(&e) => {}, + Err(e) => { + return Err(FirecrackerError::SocketUnusable { + path: self.socket_path.clone(), + source: e, + }); + }, } std::thread::sleep(poll_interval); } @@ -204,6 +216,21 @@ impl FirecrackerClient { } } +/// Whether an error means the VMM has simply not started listening yet. +/// +/// The socket file not existing, or existing with nothing accepting on it, is +/// the normal state during boot. Every other error describes the address +/// itself and will not change by waiting. +fn is_not_listening_yet(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::NotFound + | std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::WouldBlock + | std::io::ErrorKind::Interrupted + ) +} + /// Check if we have received a complete HTTP response. fn response_complete(data: &[u8]) -> bool { let header_end = find_header_end(data); diff --git a/plus/bencher_runner/src/firecracker/error.rs b/plus/bencher_runner/src/firecracker/error.rs index 63282c411..99013a023 100644 --- a/plus/bencher_runner/src/firecracker/error.rs +++ b/plus/bencher_runner/src/firecracker/error.rs @@ -30,6 +30,18 @@ pub enum FirecrackerError { #[error("Firecracker API socket not ready after {0:?}")] SocketNotReady(std::time::Duration), + /// The API socket address itself cannot be used. + /// + /// Distinct from [`Self::SocketNotReady`]: waiting will not help, so the + /// error names the path and the cause instead of a timeout. + #[error("Firecracker API socket {path} is unusable: {source}")] + SocketUnusable { + /// The socket path the runner tried to reach. + path: String, + /// Why it could not be reached. + source: std::io::Error, + }, + /// Failed to collect results via vsock. #[error("Vsock result collection failed: {0}")] VsockCollection(String), diff --git a/plus/bencher_runner/src/firecracker/mod.rs b/plus/bencher_runner/src/firecracker/mod.rs index 7fdf1197f..7ce0f048e 100644 --- a/plus/bencher_runner/src/firecracker/mod.rs +++ b/plus/bencher_runner/src/firecracker/mod.rs @@ -225,7 +225,7 @@ pub fn run_firecracker( // Step 3: Create vsock listeners (must be before boot) println!("Setting up vsock listeners..."); - let vsock_listener = VsockListener::new(jail.vsock().host())?; + let vsock_listener = VsockListener::new(jail.vsock().socket())?; // Firecracker connects out to these as the unprivileged jail user, so it // needs write access to the inodes. After bind and before InstanceStart. vsock_listener diff --git a/plus/bencher_runner/src/firecracker/process.rs b/plus/bencher_runner/src/firecracker/process.rs index 381683ba4..d94cc9364 100644 --- a/plus/bencher_runner/src/firecracker/process.rs +++ b/plus/bencher_runner/src/firecracker/process.rs @@ -11,7 +11,7 @@ use camino::Utf8Path; use crate::firecracker::client::FirecrackerClient; use crate::firecracker::config::{Action, ActionType}; use crate::firecracker::error::FirecrackerError; -use crate::jail::{HostPath, JailFile, JailUser}; +use crate::jail::{JailFile, JailUser, SocketPath}; /// Everything needed to spawn the VMM under the jailer. #[derive(Debug)] @@ -46,7 +46,7 @@ pub struct JailedSpawn<'a> { /// A running, jailed Firecracker process. pub struct FirecrackerProcess { child: Child, - api_socket_path: HostPath, + api_socket_path: SocketPath, stderr_thread: Option>, } @@ -141,7 +141,7 @@ impl FirecrackerProcess { let process = Self { child, - api_socket_path: api_socket.host().clone(), + api_socket_path: api_socket.socket().clone(), stderr_thread: Some(stderr_thread), }; @@ -196,7 +196,7 @@ impl FirecrackerProcess { /// The chroot itself is reclaimed wholesale by the jail teardown; this /// only keeps the socket from outliving the process within a job. pub fn cleanup(&self) { - drop(std::fs::remove_file(self.api_socket_path.as_path())); + drop(std::fs::remove_file(self.api_socket_path.as_str())); } /// Join the stderr reader thread if it exists. @@ -227,9 +227,9 @@ fn jailer_args(spawn: &JailedSpawn<'_>) -> Vec { "--exec-file".to_owned(), spawn.exec_file.to_string(), "--uid".to_owned(), - spawn.jail_user.uid.to_string(), + spawn.jail_user.uid().to_string(), "--gid".to_owned(), - spawn.jail_user.gid.to_string(), + spawn.jail_user.gid().to_string(), "--chroot-base-dir".to_owned(), spawn.chroot_base_dir.to_string(), "--netns".to_owned(), @@ -266,8 +266,6 @@ mod tests { use super::*; use crate::jail::JailPaths; - const JAIL_ROOT: &str = "/var/lib/bencher-runner/jail/firecracker/vm-1/root"; - fn spawn_for(jail: &JailPaths) -> JailedSpawn<'_> { JailedSpawn { jailer_bin: Utf8Path::new("/tmp/work/jailer"), @@ -284,10 +282,18 @@ mod tests { } fn args() -> Vec { - let jail = JailPaths::new(Utf8Path::new(JAIL_ROOT)); + let (_dir, jail) = jail_in_tmpdir(); jailer_args(&spawn_for(&jail)) } + /// The jail root has to exist: the paths hold a descriptor on it. + fn jail_in_tmpdir() -> (tempfile::TempDir, JailPaths) { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8Path::from_path(dir.path()).unwrap(); + let jail = JailPaths::new(root).unwrap(); + (dir, jail) + } + /// The value following `flag`, if the flag is present. fn value_of<'a>(args: &'a [String], flag: &str) -> Option<&'a str> { let index = args.iter().position(|arg| arg == flag)?; @@ -353,12 +359,13 @@ mod tests { // Firecracker binds the socket after it has been confined, so it must // receive the path as it will exist inside the chroot. The host view // names a directory the jailed process cannot reach. - let jail = JailPaths::new(Utf8Path::new(JAIL_ROOT)); + let (_dir, jail) = jail_in_tmpdir(); let args = jailer_args(&spawn_for(&jail)); assert_eq!(value_of(&args, "--api-sock"), Some("/api.sock")); + let jail_root = jail.root().as_str(); assert!( - !args.iter().any(|arg| arg.contains(JAIL_ROOT)), + !args.iter().any(|arg| arg.contains(jail_root)), "no host-side jail path may reach the jailed process: {args:?}" ); } diff --git a/plus/bencher_runner/src/firecracker/vsock.rs b/plus/bencher_runner/src/firecracker/vsock.rs index 79c1e435f..591fbcb6f 100644 --- a/plus/bencher_runner/src/firecracker/vsock.rs +++ b/plus/bencher_runner/src/firecracker/vsock.rs @@ -21,7 +21,7 @@ use nix::poll::{PollFd, PollFlags, PollTimeout, poll}; use crate::firecracker::error::FirecrackerError; use crate::jail::chroot::chown_to_jail; -use crate::jail::{HostPath, JailUser}; +use crate::jail::{JailUser, SocketPath}; /// Poll timeout for vsock listeners (50ms). /// @@ -74,12 +74,12 @@ impl VsockListener { /// Creates Unix listeners at `{vsock_uds_path}_{port}` for each port, /// using the host view of the path: the runner binds them from outside /// the chroot. These must be created before the VM boots. - pub fn new(vsock_uds_path: &HostPath) -> Result { + pub fn new(vsock_uds_path: &SocketPath) -> Result { + let stdout_path = vsock_uds_path.with_suffix(&format!("_{}", ports::STDOUT)); + let stderr_path = vsock_uds_path.with_suffix(&format!("_{}", ports::STDERR)); + let exit_code_path = vsock_uds_path.with_suffix(&format!("_{}", ports::EXIT_CODE)); + let output_files_path = vsock_uds_path.with_suffix(&format!("_{}", ports::OUTPUT_FILES)); let vsock_uds_path = vsock_uds_path.as_str(); - let stdout_path = format!("{vsock_uds_path}_{}", ports::STDOUT); - let stderr_path = format!("{vsock_uds_path}_{}", ports::STDERR); - let exit_code_path = format!("{vsock_uds_path}_{}", ports::EXIT_CODE); - let output_files_path = format!("{vsock_uds_path}_{}", ports::OUTPUT_FILES); // Remove stale socket files for path in [ @@ -349,6 +349,7 @@ fn try_accept_and_read(listener: &UnixListener, max_data_size: usize) -> Option< )] mod tests { use super::*; + use crate::jail::JailPaths; use std::io::Write as _; use std::os::unix::net::UnixStream; @@ -358,23 +359,24 @@ mod tests { /// Short grace period for tests to avoid slowing down the test suite. const TEST_GRACE_PERIOD: Duration = Duration::from_millis(50); - /// Helper: resolve the jail paths for a temp directory standing in for a - /// chroot root, so the tests bind the same host paths production does. - fn jail_paths(dir: &tempfile::TempDir) -> crate::jail::JailPaths { + /// Helper: a jail whose descriptor stays open for the whole test. + /// + /// The socket view names an open descriptor by number, so the `JailPaths` + /// has to outlive every path derived from it. Dropping it early leaves the + /// paths addressing whatever the kernel hands that number to next, which + /// is exactly the failure this binding prevents. + fn jail_in_tmpdir() -> (tempfile::TempDir, JailPaths) { + let dir = tempfile::tempdir().unwrap(); let root = Utf8Path::from_path(dir.path()).unwrap(); - crate::jail::JailPaths::new(root) + let jail = JailPaths::new(root).unwrap(); + (dir, jail) } - /// Helper: the host path of the vsock base in a temp directory. - fn vsock_base(dir: &tempfile::TempDir) -> String { - jail_paths(dir).vsock().host().as_str().to_owned() - } - - /// Helper: create a `VsockListener` in a temp directory. - fn listener_in_tmpdir() -> (tempfile::TempDir, VsockListener) { - let dir = tempfile::tempdir().unwrap(); - let listener = VsockListener::new(jail_paths(&dir).vsock().host()).unwrap(); - (dir, listener) + /// Helper: create a `VsockListener` on a jail that stays alive. + fn listener_in_tmpdir() -> (tempfile::TempDir, JailPaths, VsockListener) { + let (dir, jail) = jail_in_tmpdir(); + let listener = VsockListener::new(jail.vsock().socket()).unwrap(); + (dir, jail, listener) } /// Helper: connect to a vsock port and write data. @@ -387,9 +389,8 @@ mod tests { #[test] fn vsock_listener_creates_socket_files() { - let dir = tempfile::tempdir().unwrap(); - let base = vsock_base(&dir); - let _listener = VsockListener::new(jail_paths(&dir).vsock().host()).unwrap(); + let (_dir, jail, _listener) = listener_in_tmpdir(); + let base = jail.vsock().socket().as_str().to_owned(); for port in [5000, 5001, 5002, 5005] { let path = format!("{base}_{port}"); @@ -402,11 +403,11 @@ mod tests { #[test] fn vsock_listener_cleanup_removes_files() { - let dir = tempfile::tempdir().unwrap(); - let base = vsock_base(&dir); + let (_dir, jail) = jail_in_tmpdir(); + let base = jail.vsock().socket().as_str().to_owned(); { - let _listener = VsockListener::new(jail_paths(&dir).vsock().host()).unwrap(); + let _listener = VsockListener::new(jail.vsock().socket()).unwrap(); // listener drops here } @@ -421,8 +422,8 @@ mod tests { #[test] fn collect_all_ports() { - let (dir, listener) = listener_in_tmpdir(); - let base = vsock_base(&dir); + let (_dir, jail, listener) = listener_in_tmpdir(); + let base = jail.vsock().socket().as_str().to_owned(); // Build protocol-encoded data: 1 file, path="out.bin", content=\x00\x01\x02 let mut encoded = Vec::new(); @@ -463,8 +464,8 @@ mod tests { #[test] fn collect_exit_code_only() { - let (dir, listener) = listener_in_tmpdir(); - let base = vsock_base(&dir); + let (_dir, jail, listener) = listener_in_tmpdir(); + let base = jail.vsock().socket().as_str().to_owned(); let base_clone = base.clone(); let sender = std::thread::spawn(move || { @@ -490,7 +491,7 @@ mod tests { #[test] fn collect_timeout_returns_error() { - let (_dir, listener) = listener_in_tmpdir(); + let (_dir, _jail, listener) = listener_in_tmpdir(); // No data sent — should timeout with an error let result = listener.collect_results( @@ -509,8 +510,8 @@ mod tests { #[test] fn collect_non_utf8_stdout() { - let (dir, listener) = listener_in_tmpdir(); - let base = vsock_base(&dir); + let (_dir, jail, listener) = listener_in_tmpdir(); + let base = jail.vsock().socket().as_str().to_owned(); let base_clone = base.clone(); let sender = std::thread::spawn(move || { @@ -538,8 +539,8 @@ mod tests { #[test] fn collect_exit_code_triggers_final_pass() { - let (dir, listener) = listener_in_tmpdir(); - let base = vsock_base(&dir); + let (_dir, jail, listener) = listener_in_tmpdir(); + let base = jail.vsock().socket().as_str().to_owned(); let base_clone = base.clone(); let sender = std::thread::spawn(move || { @@ -641,7 +642,7 @@ mod tests { #[test] fn collect_cancelled_returns_error() { - let (_dir, listener) = listener_in_tmpdir(); + let (_dir, _jail, listener) = listener_in_tmpdir(); // Set the cancel flag before collecting let cancel_flag = Arc::new(AtomicBool::new(true)); diff --git a/plus/bencher_runner/src/jail/paths.rs b/plus/bencher_runner/src/jail/paths.rs index a4db233cc..0a3e450e4 100644 --- a/plus/bencher_runner/src/jail/paths.rs +++ b/plus/bencher_runner/src/jail/paths.rs @@ -1,14 +1,44 @@ -//! The two views of every file inside the jail chroot. +//! The views of every file inside the jail chroot. //! //! Once Firecracker is jailed, every path it receives resolves inside the -//! chroot, while the runner reaches the same file from outside. The two views -//! are different types rather than two strings, so handing Firecracker a host -//! path (or the runner a chroot path) is a compile error instead of a boot -//! that hangs waiting for a socket that will never appear. +//! chroot, while the runner reaches the same file from outside. The views are +//! different types rather than strings, so handing Firecracker a host path (or +//! the runner a chroot path) is a compile error instead of a boot that hangs +//! waiting for a socket that will never appear. +//! +//! There is a third view because of a hard kernel limit. `sockaddr_un.sun_path` +//! is 108 bytes, and the limit applies to the string handed to `bind` and +//! `connect`, before any resolution. A jail deep under an operator's +//! `--state-dir` blows that limit long before it comes near `PATH_MAX`, so the +//! runner addresses the jail's sockets through a descriptor it holds open on +//! the chroot. Resolution happens after the length check, so a short string +//! naming a long directory is exactly what is needed. + +use std::fs::File; +use std::os::fd::AsRawFd as _; +use std::os::unix::fs::OpenOptionsExt as _; use camino::{Utf8Path, Utf8PathBuf}; use serde::Serialize; +use crate::error::JailError; + +/// Size of `sockaddr_un.sun_path` on Linux. +const SUN_PATH_LEN: usize = 108; + +/// Longest path a socket name may occupy, leaving room for the NUL. +/// +/// Linux accepts a full 108 unterminated bytes when `addrlen` says so, but +/// `unix(7)` warns against relying on it and the standard library rejects +/// anything that does not leave room for the terminator. +const MAX_SOCKET_PATH: usize = SUN_PATH_LEN - 1; + +/// Room reserved after the vsock base path for its `_` suffix. +/// +/// Sized for the widest port a `u32` can print rather than for the ports in +/// use, so adding a port can never silently eat the margin. +const VSOCK_SUFFIX_RESERVE: usize = "_4294967295".len(); + /// A path as the runner sees it: the host filesystem, outside the chroot. #[derive(Debug, Clone, PartialEq, Eq)] pub struct HostPath(Utf8PathBuf); @@ -55,24 +85,64 @@ impl std::fmt::Display for ChrootPath { } } -/// One file in the jail, in both views. +/// A path the runner may hand to `bind` or `connect`. +/// +/// The type makes the length limit unforgeable: every value has been checked +/// against `sun_path`, so a path that would not fit is reported when the jail +/// is built, naming the limit and the offending string, rather than surfacing +/// later as a socket that never becomes ready. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SocketPath(String); + +impl SocketPath { + /// Check a path against the `sun_path` limit. + /// + /// `reserve` is the longest suffix that will later be appended, so a base + /// that fits only without its suffix is still rejected. + fn new(path: String, reserve: usize) -> Result { + let length = path.len() + reserve; + if length > MAX_SOCKET_PATH { + return Err(JailError::SocketPathTooLong { + path, + length, + limit: MAX_SOCKET_PATH, + }); + } + Ok(Self(path)) + } + + /// The path as a string. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + /// This path with a suffix appended. + /// + /// The suffix was reserved when the base was checked, so the result is + /// within the limit by construction. + #[must_use] + pub fn with_suffix(&self, suffix: &str) -> String { + format!("{}{suffix}", self.0) + } +} + +impl std::fmt::Display for SocketPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +/// One file in the jail, in every view that reaches it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct JailFile { host: HostPath, chroot: ChrootPath, + socket: SocketPath, } impl JailFile { - /// Build both views of `name` directly under the chroot root. - #[must_use] - pub fn new(jail_root: &Utf8Path, name: &str) -> Self { - Self { - host: HostPath(jail_root.join(name)), - chroot: ChrootPath(Utf8Path::new("/").join(name)), - } - } - - /// The path the runner uses. + /// The path the runner uses to create, read, and own the file. #[must_use] pub fn host(&self) -> &HostPath { &self.host @@ -83,12 +153,25 @@ impl JailFile { pub fn chroot(&self) -> &ChrootPath { &self.chroot } + + /// The path the runner binds or connects to. + #[must_use] + pub fn socket(&self) -> &SocketPath { + &self.socket + } } /// Every file the runner places in, or reaches inside, a jail chroot. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct JailPaths { root: Utf8PathBuf, + /// Held open for the life of the job. + /// + /// The socket views name this descriptor by number, so closing it would + /// leave them addressing whatever directory the kernel hands that number + /// to next. `O_PATH` because the runner never reads or writes through it: + /// it exists only to be named. + _dir: File, api_socket: JailFile, kernel: JailFile, rootfs: JailFile, @@ -96,17 +179,40 @@ pub struct JailPaths { } impl JailPaths { - /// Resolve both views of every jail file for a chroot rooted at - /// `jail_root`. - #[must_use] - pub fn new(jail_root: &Utf8Path) -> Self { - Self { + /// Resolve every view of every jail file for a chroot that already exists. + pub fn new(jail_root: &Utf8Path) -> Result { + let dir = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_PATH | libc::O_DIRECTORY) + .open(jail_root) + .map_err(|e| JailError::OpenJailRoot { + path: jail_root.to_owned(), + source: e, + })?; + + // Naming the directory by descriptor keeps the string short however + // deep the operator's state directory is. The descriptor also pins the + // directory's identity for the whole job. + let dir_path = format!("/proc/self/fd/{}", dir.as_raw_fd()); + + let file = |name: &str, reserve: usize| -> Result { + Ok(JailFile { + host: HostPath(jail_root.join(name)), + chroot: ChrootPath(Utf8Path::new("/").join(name)), + socket: SocketPath::new(format!("{dir_path}/{name}"), reserve)?, + }) + }; + + Ok(Self { root: jail_root.to_owned(), - api_socket: JailFile::new(jail_root, "api.sock"), - kernel: JailFile::new(jail_root, "vmlinux"), - rootfs: JailFile::new(jail_root, "rootfs.ext4"), - vsock: JailFile::new(jail_root, "v.sock"), - } + api_socket: file("api.sock", 0)?, + kernel: file("vmlinux", 0)?, + rootfs: file("rootfs.ext4", 0)?, + // The runner binds `{base}_{port}` for each vsock port, so the + // base has to leave room for the longest of those suffixes. + vsock: file("v.sock", VSOCK_SUFFIX_RESERVE)?, + _dir: dir, + }) } /// The chroot root on the host, which becomes `/` inside the jail. @@ -144,15 +250,16 @@ impl JailPaths { mod tests { use super::*; - const JAIL_ROOT: &str = "/var/lib/bencher-runner/jail/firecracker/vm-1/root"; - - fn paths() -> JailPaths { - JailPaths::new(Utf8Path::new(JAIL_ROOT)) + fn jail_in_tmpdir() -> (tempfile::TempDir, JailPaths) { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8Path::from_path(dir.path()).unwrap(); + let paths = JailPaths::new(root).unwrap(); + (dir, paths) } #[test] fn chroot_view_is_rooted_at_the_chroot() { - let paths = paths(); + let (_dir, paths) = jail_in_tmpdir(); assert_eq!(paths.api_socket().chroot().as_str(), "/api.sock"); assert_eq!(paths.kernel().chroot().as_str(), "/vmlinux"); assert_eq!(paths.rootfs().chroot().as_str(), "/rootfs.ext4"); @@ -161,25 +268,17 @@ mod tests { #[test] fn host_view_is_under_the_jail_root() { - let paths = paths(); - assert_eq!( - paths.api_socket().host().as_str(), - format!("{JAIL_ROOT}/api.sock") - ); - assert_eq!( - paths.kernel().host().as_str(), - format!("{JAIL_ROOT}/vmlinux") - ); - assert_eq!( - paths.rootfs().host().as_str(), - format!("{JAIL_ROOT}/rootfs.ext4") - ); - assert_eq!(paths.vsock().host().as_str(), format!("{JAIL_ROOT}/v.sock")); + let (_dir, paths) = jail_in_tmpdir(); + let root = paths.root().to_owned(); + assert_eq!(paths.api_socket().host().as_path(), root.join("api.sock")); + assert_eq!(paths.kernel().host().as_path(), root.join("vmlinux")); + assert_eq!(paths.rootfs().host().as_path(), root.join("rootfs.ext4")); + assert_eq!(paths.vsock().host().as_path(), root.join("v.sock")); } #[test] - fn the_two_views_round_trip_through_the_jail_root() { - let paths = paths(); + fn the_chroot_and_host_views_round_trip_through_the_jail_root() { + let (_dir, paths) = jail_in_tmpdir(); for file in [ paths.api_socket(), paths.kernel(), @@ -197,9 +296,82 @@ mod tests { } } + #[test] + fn the_socket_view_resolves_to_the_same_file_as_the_host_view() { + // The whole point: a short string naming the same inode. If this ever + // stops holding, the runner and Firecracker stop meeting. + let (_dir, paths) = jail_in_tmpdir(); + std::fs::write(paths.rootfs().host().as_path(), b"guest").unwrap(); + + let through_socket_view = std::fs::read(paths.rootfs().socket().as_str()).unwrap(); + + assert_eq!(through_socket_view, b"guest"); + } + + #[test] + fn every_socket_view_fits_the_sun_path_limit() { + let (_dir, paths) = jail_in_tmpdir(); + for file in [paths.api_socket(), paths.vsock()] { + assert!( + file.socket().as_str().len() <= MAX_SOCKET_PATH, + "{} must fit sun_path", + file.socket() + ); + } + // And the longest name the vsock base ever grows into still fits. + let longest = paths.vsock().socket().with_suffix("_4294967295"); + assert!(longest.len() <= MAX_SOCKET_PATH, "{longest} must fit"); + } + + #[test] + fn the_socket_view_survives_a_jail_root_far_past_the_limit() { + // A deep state directory is exactly the case that produced a five + // second timeout pointing at Firecracker instead of at the path. + let dir = tempfile::tempdir().unwrap(); + let mut root = Utf8Path::from_path(dir.path()).unwrap().to_owned(); + for _ in 0..8 { + root = root.join("a-fairly-long-directory-name"); + } + std::fs::create_dir_all(&root).unwrap(); + assert!( + root.as_str().len() > MAX_SOCKET_PATH, + "the host path has to be over the limit for this to prove anything" + ); + + let paths = JailPaths::new(&root).unwrap(); + + assert!(paths.api_socket().socket().as_str().len() <= MAX_SOCKET_PATH); + } + + #[test] + fn an_oversized_socket_path_names_the_limit_and_the_length() { + let err = SocketPath::new("/x".repeat(80), 0).unwrap_err(); + let message = err.to_string(); + + assert!(message.contains("107"), "the limit is named: {message}"); + assert!(message.contains("160"), "the length is named: {message}"); + } + + #[test] + fn a_reserved_suffix_counts_against_the_limit() { + let base = "/a".repeat(52); + assert_eq!(base.len(), 104); + + SocketPath::new(base.clone(), 0).unwrap(); + SocketPath::new(base, 8).unwrap_err(); + } + + #[test] + fn a_missing_jail_root_is_reported() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8Path::from_path(dir.path()).unwrap().join("absent"); + + JailPaths::new(&root).unwrap_err(); + } + #[test] fn chroot_paths_serialize_as_bare_strings() { - let paths = paths(); + let (_dir, paths) = jail_in_tmpdir(); assert_eq!( serde_json::to_string(paths.rootfs().chroot()).unwrap(), "\"/rootfs.ext4\"" From 0a53f1c61f452af751f916fec90269acc8eca49c Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 04:06:17 +0000 Subject: [PATCH 13/91] Make the jail scenarios fail when the product is broken Both new scenarios reported PASSED in CI while every other sandboxed scenario failed to boot a VM at all. The confinement probe checks the VMM's uid, its cgroup, and its root inode, all of which hold whether or not the guest ever runs, so the scenario stayed green through a broken product. That is the worst failure mode a confinement test has. Both now assert the job succeeded, exit code and guest output, before asserting anything about confinement. jail_teardown_on_cancel is replaced by jail_sweep_reclaims_orphan, which tests the mechanism that actually covers exits that never unwind. SIGTERM to the one-shot runner takes the default disposition, since signal handlers are installed only by the daemon, so Drop never ran and the scenario was asserting teardown that could not have happened; it passed only because the run failed fast. The replacement kills the runner once its VMM is up, proves the chroot survived, and then proves the next job swept it. It reaps the orphaned VMM itself: the sweep reclaims the chroot but nothing reaps an orphaned VMM or its cgroup, so a stray Firecracker would otherwise burn benchmark cores for the rest of the suite. The confinement probe no longer treats a VMM caught mid-flight as a violation. The jailer pivot_roots before it drops privilege, so there is a window where the process root already matches the jail while the process is still root; that is now not-ready-yet, and the timeout is what catches a VMM that never drops. A cgroup that cannot be read says so out loud rather than passing silently. The scenarios also pass --no-tuning. Elevating them turned real host tuning on for all twenty-five: unprivileged every knob failed with EPERM and warned, but as root they apply, and offlining SMT siblings on a two-vCPU hosted runner would change the core count mid-suite. --- tasks/test_runner/src/task/scenarios.rs | 289 +++++++++++++++++++++--- 1 file changed, 261 insertions(+), 28 deletions(-) diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index 275c989ab..e834bf58a 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -58,6 +58,12 @@ struct Scenario { sandboxed: bool, /// If set, a host-side check run while the runner is executing. probe: Option, + /// Kill the runner once its VMM is up so nothing unwinds, then run the + /// image again and report the second run. + /// + /// SIGKILL is the point: it is the exit that never unwinds, so `Drop` + /// cannot reclaim the chroot and only the sweep can. + orphan_then_rerun: bool, validate: fn(&ScenarioOutput) -> Result<()>, } @@ -222,7 +228,12 @@ fn run_scenario(scenario: &Scenario, runner_bin: &Utf8Path) -> Result<()> { drop(fs::remove_dir_all(&state_dir)); // Prepend --sandbox firecracker for sandboxed scenarios - let mut args: Vec<&str> = vec!["--state-dir", state_dir.as_str()]; + // --no-tuning matters now that the scenarios run as root. Unprivileged, + // every tuning knob failed with EPERM and warned; elevated they actually + // apply, and offlining SMT siblings on a two-vCPU hosted runner would + // change the core count mid-suite. The scenarios exercise job execution, + // not tuning, so this costs no coverage. + let mut args: Vec<&str> = vec!["--state-dir", state_dir.as_str(), "--no-tuning"]; if scenario.sandboxed { args.extend(["--sandbox", "firecracker"]); } @@ -231,6 +242,8 @@ fn run_scenario(scenario: &Scenario, runner_bin: &Utf8Path) -> Result<()> { // Run the runner (with optional cancellation or host-side probe) let output = if let Some(secs) = scenario.cancel_after_secs { run_runner_with_cancel(&image_path, &args, Duration::from_secs(secs), runner_bin) + } else if scenario.orphan_then_rerun { + run_runner_after_orphan(&image_path, &args, &state_dir, runner_bin) } else if let Some(probe) = scenario.probe { run_runner_with_probe(&image_path, &args, probe, &state_dir, runner_bin) } else { @@ -264,6 +277,7 @@ fn all_scenarios() -> Vec { CMD ["echo", "hello from vm"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -282,6 +296,7 @@ ENV MY_VAR=test_value CMD ["sh", "-c", "echo $MY_VAR"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -305,6 +320,7 @@ WORKDIR /myapp CMD ["pwd"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -322,6 +338,7 @@ CMD ["pwd"]"#, CMD ["sh", "-c", "echo '{\"result\": 42}' > /tmp/output.json && cat /tmp/output.json"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--output", "/tmp/output.json"], validate: |output| { @@ -339,6 +356,7 @@ CMD ["sh", "-c", "echo '{\"result\": 42}' > /tmp/output.json && cat /tmp/output. CMD ["sh", "-c", "exit 42"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -357,6 +375,7 @@ CMD ["sh", "-c", "exit 42"]"#, CMD ["sleep", "3600"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { @@ -375,6 +394,7 @@ CMD ["sleep", "3600"]"#, CMD ["sh", "-c", "echo test > /data.txt && cat /data.txt"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -395,6 +415,7 @@ CMD ["sh", "-c", "echo test > /data.txt && cat /data.txt"]"#, CMD ["sh", "-c", "echo stdout && echo stderr >&2"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -413,6 +434,7 @@ CMD ["sh", "-c", "echo stdout && echo stderr >&2"]"#, CMD ["sh", "-c", "cat /proc/cpuinfo | grep processor | wc -l"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "10", "--vcpus", "4"], validate: |output| { @@ -441,6 +463,7 @@ ENTRYPOINT ["echo"] CMD ["hello", "world"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -458,6 +481,7 @@ CMD ["hello", "world"]"#, CMD ["sh", "-c", "ping -c 1 -W 1 8.8.8.8 2>&1 || echo no_network"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -483,6 +507,7 @@ CMD ["sh", "-c", "ping -c 1 -W 1 8.8.8.8 2>&1 || echo no_network"]"#, CMD ["sh", "-c", "dd if=/dev/zero bs=1M count=20 2>/dev/null | tr '\\0' 'A' && echo DONE"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "120", "--max-output-size", "10485760"], validate: |output| { @@ -506,6 +531,7 @@ CMD ["sh", "-c", "dd if=/dev/zero bs=1M count=20 2>/dev/null | tr '\\0' 'A' && e CMD ["sh", "-c", "trap '' TERM INT; echo started; while true; do sleep 1; done"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { @@ -538,6 +564,7 @@ CMD ["sh", "-c", "trap '' TERM INT; echo started; while true; do sleep 1; done"] CMD ["id"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -565,6 +592,7 @@ CMD ["id"]"#, CMD ["echo", "kvm_test_ok"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -588,6 +616,7 @@ CMD ["echo", "kvm_test_ok"]"#, CMD ["cat", "/proc/version"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -613,6 +642,7 @@ CMD ["cat", "/proc/version"]"#, CMD ["sh", "-c", "touch /tmp/write_test && echo write_ok"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -639,6 +669,7 @@ CMD ["sh", "-c", "touch /tmp/write_test && echo write_ok"]"#, CMD ["sh", "-c", "echo partial_output_marker && sleep 3600"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "10"], validate: |output| { @@ -666,6 +697,7 @@ CMD ["sh", "-c", "echo partial_output_marker && sleep 3600"]"#, CMD ["sleep", "3600"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { @@ -700,6 +732,7 @@ COPY --from=build /test_iopl /test_iopl CMD ["/test_iopl"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -730,6 +763,7 @@ CMD ["/test_iopl"]"#, CMD ["echo", "UNIQUE_VM_OUTPUT_a7f3b2c9"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -759,6 +793,7 @@ CMD ["echo", "UNIQUE_VM_OUTPUT_a7f3b2c9"]"#, CMD ["sh", "-c", "ls /proc | grep -E '^[0-9]+$' | wc -l"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -792,6 +827,7 @@ CMD ["sh", "-c", "ls /proc | grep -E '^[0-9]+$' | wc -l"]"#, CMD ["sh", "-c", "cat /proc/version && echo PID1=$(cat /proc/1/cmdline | tr '\\0' ' ')"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -820,6 +856,7 @@ CMD ["sh", "-c", "cat /proc/version && echo PID1=$(cat /proc/1/cmdline | tr '\\0 CMD ["echo", "metrics_test"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -843,6 +880,7 @@ CMD ["echo", "metrics_test"]"#, CMD ["echo", "fast_benchmark"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -881,6 +919,7 @@ CMD ["echo", "fast_benchmark"]"#, CMD ["sleep", "3600"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { @@ -920,6 +959,7 @@ CMD ["sleep", "3600"]"#, CMD ["echo", "hmac_test_output"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -953,6 +993,7 @@ CMD ["echo", "hmac_test_output"]"#, CMD ["echo", "transport_test"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -988,6 +1029,7 @@ CMD ["echo", "transport_test"]"#, CMD ["sh", "-c", "echo started && sleep 3600"]"#, cancel_after_secs: Some(5), probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { @@ -1015,6 +1057,7 @@ CMD ["sh", "-c", "echo started && sleep 3600"]"#, CMD ["sh", "-c", "echo error_output >&2"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1036,6 +1079,7 @@ CMD ["sh", "-c", "echo error_output >&2"]"#, CMD ["true"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1059,6 +1103,7 @@ CMD ["true"]"#, CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1094,6 +1139,7 @@ CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, dockerfile: "FROM busybox\nCMD echo shell_form_works", cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1118,6 +1164,7 @@ CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, ENTRYPOINT ["echo", "entrypoint_only_works"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1141,6 +1188,7 @@ ENTRYPOINT ["echo", "entrypoint_only_works"]"#, dockerfile: "FROM busybox\nENTRYPOINT echo shell_entrypoint_works", cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1171,6 +1219,7 @@ ENTRYPOINT echo ep_marker CMD ["cmd_arg"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1201,6 +1250,7 @@ CMD ["cmd_arg"]"#, RUN echo "no command set""#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "30"], validate: |output| { @@ -1229,6 +1279,7 @@ RUN echo "no command set""#, CMD ["mock"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { @@ -1273,6 +1324,7 @@ COPY --from=builder /tmp/hello /usr/bin/hello CMD ["/usr/bin/hello"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { @@ -1312,6 +1364,7 @@ CMD ["/usr/bin/hello"]"#, CMD ["echo", "rapid_exit_marker"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1346,6 +1399,7 @@ CMD ["echo", "rapid_exit_marker"]"#, CMD ["sh", "-c", "exit 137"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1379,6 +1433,7 @@ ENV LARGE_VALUE=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA CMD ["sh", "-c", "echo A1=$A1 B10=$B10 LARGE_LEN=${#LARGE_VALUE}"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1412,6 +1467,7 @@ CMD ["sh", "-c", "echo A1=$A1 B10=$B10 LARGE_LEN=${#LARGE_VALUE}"]"#, CMD ["echo", "no file written"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--output", "/nonexistent/path.json"], validate: |output| { @@ -1431,6 +1487,7 @@ CMD ["echo", "no file written"]"#, CMD ["sh", "-c", "dd if=/dev/urandom bs=1024 count=2048 2>/dev/null | base64 > /tmp/output.json && echo done"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--output", "/tmp/output.json"], validate: |output| { @@ -1448,6 +1505,7 @@ CMD ["sh", "-c", "dd if=/dev/urandom bs=1024 count=2048 2>/dev/null | base64 > / CMD ["sh", "-c", "echo stdout_marker && echo stderr_marker >&2 && echo '{\"data\":true}' > /tmp/out.json"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--output", "/tmp/out.json"], validate: |output| { @@ -1471,6 +1529,7 @@ CMD ["sh", "-c", "echo stdout_marker && echo stderr_marker >&2 && echo '{\"data\ CMD ["sh", "-c", "echo '{\"result\": 1}' > /tmp/a.json && echo '{\"result\": 2}' > /tmp/b.json && echo done"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &[ "--timeout", @@ -1501,6 +1560,7 @@ RUN echo "c" > /var/file_c.txt CMD ["sh", "-c", "cat /tmp/file_a.txt /opt/file_b.txt /var/file_c.txt"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1526,6 +1586,7 @@ RUN echo "target" > /tmp/target.txt && ln -s /tmp/target.txt /tmp/link.txt CMD ["cat", "/tmp/link.txt"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1553,6 +1614,7 @@ CMD ["cat", "/tmp/link.txt"]"#, CMD ["sh", "-c", "echo partial_stdout && echo partial_stderr >&2 && exit 1"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1573,6 +1635,7 @@ CMD ["sh", "-c", "echo partial_stdout && echo partial_stderr >&2 && exit 1"]"#, CMD ["sleep", "3600"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "1"], validate: |output| { @@ -1590,6 +1653,7 @@ CMD ["sleep", "3600"]"#, CMD ["sh", "-c", "dd if=/dev/zero bs=1024 count=50 2>/dev/null | tr '\\0' 'X'"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--max-output-size", "1024"], validate: |output| { @@ -1614,6 +1678,7 @@ ENV SAFE_VAR=safe_value CMD ["sh", "-c", "echo LD_PRELOAD=$LD_PRELOAD LD_LIBRARY_PATH=$LD_LIBRARY_PATH SAFE=$SAFE_VAR"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1654,6 +1719,7 @@ CMD ["sh", "-c", "echo LD_PRELOAD=$LD_PRELOAD LD_LIBRARY_PATH=$LD_LIBRARY_PATH S CMD ["free", "-m"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--memory", "64", "--timeout", "60"], validate: |output| { @@ -1682,6 +1748,7 @@ CMD ["free", "-m"]"#, CMD ["sh", "-c", "df -m / | tail -1 | awk '{print $2}'"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--disk", "64", "--timeout", "60"], validate: |output| { @@ -1710,6 +1777,7 @@ CMD ["sh", "-c", "df -m / | tail -1 | awk '{print $2}'"]"#, CMD ["sh", "-c", "df -m / | tail -1 | awk '{print \"TOTAL_MB=\" $2}'"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--disk", "64", "--timeout", "60"], validate: |output| { @@ -1743,6 +1811,7 @@ CMD ["sh", "-c", "df -m / | tail -1 | awk '{print \"TOTAL_MB=\" $2}'"]"#, CMD ["nproc"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1769,6 +1838,7 @@ CMD ["nproc"]"#, CMD ["sh", "-c", "wget -q -O /dev/null http://detectportal.firefox.com/success.txt && echo net_ok || echo net_fail"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "30", "--network"], validate: |output| { @@ -1800,6 +1870,7 @@ RUN mkdir -p /data && echo "content_ok" > /data/file.txt CMD ["cat", "/data/file.txt"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1824,6 +1895,7 @@ RUN mkdir -p /data && printf '#!/bin/sh\necho hello' > /data/test.sh && chmod +x CMD ["sh", "-c", "test -x /data/test.sh && echo perm_ok"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1851,6 +1923,7 @@ RUN mkdir -p /data/restricted && chmod 750 /data/restricted CMD ["stat", "-c", "%a", "/data/restricted"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1878,6 +1951,7 @@ CMD ["stat", "-c", "%a", "/data/restricted"]"#, dockerfile: "FROM busybox\nENV SPACED=\"hello world\" WITH_EQ=\"key=value\"\nCMD [\"sh\", \"-c\", \"echo SPACED=$SPACED EQ=$WITH_EQ\"]", cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { @@ -1908,6 +1982,7 @@ ENTRYPOINT ["echo", "image_ep"] CMD ["image_cmd"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--entrypoint", "echo", "cli_ep"], validate: |output| { @@ -1946,6 +2021,7 @@ ENTRYPOINT ["echo"] CMD ["image_cmd"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--cmd", "cli_cmd"], validate: |output| { @@ -1977,6 +2053,7 @@ ENTRYPOINT ["echo", "image_ep"] CMD ["image_cmd"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &[ "--timeout", @@ -2015,6 +2092,7 @@ ENV MY_VAR=image_value CMD ["sh", "-c", "echo MY_VAR=$MY_VAR"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--env", "MY_VAR=cli_value"], validate: |output| { @@ -2041,6 +2119,7 @@ ENV EXISTING=from_image CMD ["sh", "-c", "echo EXISTING=$EXISTING NEW=$NEW_VAR"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--env", "NEW_VAR=from_cli"], validate: |output| { @@ -2070,6 +2149,7 @@ CMD ["sh", "-c", "echo EXISTING=$EXISTING NEW=$NEW_VAR"]"#, CMD ["sh", "-c", "echo A=$A B=$B"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--env", "A=one", "--env", "B=two"], validate: |output| { @@ -2093,6 +2173,7 @@ CMD ["sh", "-c", "echo A=$A B=$B"]"#, CMD ["hello", "world"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--entrypoint", "echo"], validate: |output| { @@ -2119,6 +2200,7 @@ CMD ["hello", "world"]"#, CMD ["echo", "iter_output"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--iter", "3"], validate: |output| { @@ -2140,6 +2222,7 @@ CMD ["echo", "iter_output"]"#, CMD ["echo", "should_not_appear"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--iter", "0"], validate: |output| { @@ -2159,6 +2242,7 @@ CMD ["echo", "should_not_appear"]"#, CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--iter", "3"], validate: |output| { @@ -2186,6 +2270,7 @@ CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: true, extra_args: &["--timeout", "60", "--iter", "3", "--allow-failure"], validate: |output| { @@ -2213,6 +2298,10 @@ CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, /// /// These test the `local_execute` code path (no Firecracker VM). /// The OCI image is unpacked and the command runs directly on the host. +#[expect( + clippy::too_many_lines, + reason = "Each scenario needs its configuration" +)] fn nosandbox_scenarios() -> Vec { vec![ Scenario { @@ -2222,6 +2311,7 @@ fn nosandbox_scenarios() -> Vec { CMD ["echo", "hello from host"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2244,6 +2334,7 @@ ENV MY_VAR=host_test_value CMD ["sh", "-c", "echo $MY_VAR"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2267,6 +2358,7 @@ CMD ["sh", "-c", "echo $MY_VAR"]"#, CMD ["echo", "local_metrics_test"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2300,6 +2392,7 @@ CMD ["echo", "local_metrics_test"]"#, CMD ["sh", "-c", "exit 42"]"#, cancel_after_secs: None, probe: None, + orphan_then_rerun: false, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2560,34 +2653,69 @@ fn jail_scenarios() -> Vec { vec![ Scenario { name: "jail_confinement", - description: "VMM runs unprivileged in its cgroup, and its chroot is reclaimed", + description: "A jailed job succeeds with the VMM unprivileged and in its cgroup", + // The guest sleeps so the VMM is alive long enough to be observed + // by a probe that polls every 100ms. dockerfile: r#"FROM busybox -CMD ["echo", "jailed"]"#, +CMD ["sh", "-c", "echo jailed && sleep 5"]"#, cancel_after_secs: None, probe: Some(probe_confinement), + orphan_then_rerun: false, sandboxed: true, - extra_args: &["--timeout", "60"], + extra_args: &["--timeout", "120"], validate: |output| { - if !output.stdout.contains("jailed") { - bail!("Expected 'jailed' in output, got: {}", output.stdout); - } + // The job has to have actually run before anything the probe + // saw means anything. Every confinement property the probe + // checks is equally true of a VMM that started and then never + // booted a guest, so without this the scenario stays green + // while the product is broken. + assert_job_succeeded(output, "jailed")?; assert_no_chroot_remains(&scenario_state_dir()) }, }, Scenario { - name: "jail_teardown_on_cancel", - description: "A cancelled job leaves no chroot behind", + name: "jail_sweep_reclaims_orphan", + description: "A chroot orphaned by a runner that never unwound is swept by the next job", dockerfile: r#"FROM busybox -CMD ["sleep", "300"]"#, - cancel_after_secs: Some(20), +CMD ["sh", "-c", "echo swept && sleep 10"]"#, + cancel_after_secs: None, probe: None, + orphan_then_rerun: true, sandboxed: true, - extra_args: &["--timeout", "300"], - validate: |_output| assert_no_chroot_remains(&scenario_state_dir()), + extra_args: &["--timeout", "120"], + validate: |output| { + assert_job_succeeded(output, "swept")?; + assert_no_chroot_remains(&scenario_state_dir()) + }, }, ] } +/// Assert the runner actually completed the job. +/// +/// A confinement scenario that asserts only confinement passes vacuously when +/// the VM never boots: the VMM process exists, is unprivileged, and is in its +/// cgroup either way. Success of the job itself is the precondition for any of +/// that meaning anything. +fn assert_job_succeeded(output: &ScenarioOutput, marker: &str) -> Result<()> { + if output.exit_code != 0 { + bail!( + "Expected the job to succeed, got exit code {}.\nstdout: {}\nstderr: {}", + output.exit_code, + output.stdout, + output.stderr + ); + } + if !output.stdout.contains(marker) { + bail!( + "Expected '{marker}' in the guest output, so the VM booted and ran.\nstdout: {}\nstderr: {}", + output.stdout, + output.stderr + ); + } + Ok(()) +} + /// Assert every chroot has been reclaimed. /// /// The jailer cleans up nothing by design, so a leftover here means the @@ -2624,7 +2752,12 @@ fn probe_confinement(state_dir: &Utf8Path) -> Result { return Ok(false); }; - check_unprivileged(pid, &jail_root)?; + if !check_unprivileged(pid, &jail_root)? { + return Ok(false); + } + // Placement happens in `pre_exec`, before the jailer itself starts, so + // membership already holds the first time the process is observable. + // There is no not-ready-yet window for it. check_cgroup_membership(&vm_id, pid)?; Ok(true) @@ -2674,7 +2807,14 @@ fn find_jailed_vmm(jail_root: &Utf8Path) -> Option { } /// Check the VMM dropped root and runs as the user the jail was handed to. -fn check_unprivileged(pid: u32, jail_root: &Utf8Path) -> Result<()> { +/// +/// Returns `Ok(false)` while the observation is premature rather than wrong. +/// The jailer `pivot_root`s before it drops privilege, so there is a window in +/// which the process root already matches the jail while the process is still +/// root and the jail root is still root-owned. Treating that as a violation +/// would fail the run for catching the jailer mid-flight; the probe's timeout +/// is what catches a VMM that genuinely never drops. +fn check_unprivileged(pid: u32, jail_root: &Utf8Path) -> Result { let status = fs::read_to_string(format!("/proc/{pid}/status")) .with_context(|| format!("Failed to read the status of the VMM (pid {pid})"))?; let uid_line = status @@ -2689,31 +2829,31 @@ fn check_unprivileged(pid: u32, jail_root: &Utf8Path) -> Result<()> { .context("Unparsable uid in the VMM's /proc status")?; if vmm_uid == 0 { - bail!("The VMM (pid {pid}) is running as root; the jailer did not drop privilege"); + return Ok(false); } // The jailer chowns the chroot root to the jail uid, so the two must // agree: a VMM running as some other unprivileged user would not be // confined to the jail it was given. - let jail_uid = jail_root_uid(jail_root)?; + let Some(jail_uid) = jail_root_uid(jail_root) else { + return Ok(false); + }; if vmm_uid != jail_uid { bail!("The VMM (pid {pid}) runs as uid {vmm_uid} but its jail is owned by uid {jail_uid}"); } - Ok(()) + Ok(true) } -/// The uid the jailer handed the chroot root to. -fn jail_root_uid(jail_root: &Utf8Path) -> Result { +/// The uid the jailer handed the chroot root to, once it has handed it over. +/// +/// `None` while the root is still owned by root, which is the same +/// not-ready-yet window `check_unprivileged` documents. +fn jail_root_uid(jail_root: &Utf8Path) -> Option { use std::os::unix::fs::MetadataExt as _; - let uid = fs::metadata(jail_root) - .with_context(|| format!("Failed to stat the jail root {jail_root}"))? - .uid(); - if uid == 0 { - bail!("The jail root {jail_root} is still owned by root"); - } - Ok(uid) + let uid = fs::metadata(jail_root).ok()?.uid(); + (uid != 0).then_some(uid) } /// Check the VMM is in its cgroup. @@ -2723,8 +2863,13 @@ fn jail_root_uid(jail_root: &Utf8Path) -> Result { fn check_cgroup_membership(vm_id: &str, pid: u32) -> Result<()> { let procs_path = format!("/sys/fs/cgroup/bencher/{vm_id}/cgroup.procs"); // No cgroup means no isolation was possible on this host, which is a - // declared limitation rather than a confinement failure. + // declared limitation rather than a confinement failure. Say so out loud: + // a confinement check that quietly asserts nothing is exactly the kind of + // green that must never be invisible. let Ok(procs) = fs::read_to_string(&procs_path) else { + println!( + " NOTE: {procs_path} is unreadable, so cgroup placement was NOT verified for this run" + ); return Ok(()); }; if procs.lines().any(|line| line.trim() == pid.to_string()) { @@ -2734,6 +2879,94 @@ fn check_cgroup_membership(vm_id: &str, pid: u32) -> Result<()> { } } +/// Orphan a jail by killing the runner, then prove the next job sweeps it. +/// +/// SIGKILL rather than SIGTERM, because SIGTERM on the one-shot path takes the +/// default disposition too (signal handlers are installed only by the daemon), +/// and either way nothing unwinds. That is the point: `Drop` cannot reclaim +/// the chroot, so if the next job finds a clean tree it can only be because +/// the sweep reclaimed it. +fn run_runner_after_orphan( + image_path: &Utf8Path, + args: &[&str], + state_dir: &Utf8Path, + runner_bin: &Utf8Path, +) -> Result { + let parent = jail_parent(state_dir); + + let mut child = Command::new(runner_bin.as_str()) + .arg("run") + .arg("--image") + .arg(image_path.as_str()) + .args(args) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn()?; + + // Wait for a real orphan: a chroot with a VMM running in it, not just an + // empty directory created microseconds before the kill. + let deadline = std::time::Instant::now() + PROBE_TIMEOUT; + let orphan = loop { + if let Some((vm_id, jail_root)) = find_jail(&parent) + && let Some(pid) = find_jailed_vmm(&jail_root) + { + break Some((vm_id, jail_root, pid)); + } + if child.try_wait()?.is_some() || std::time::Instant::now() >= deadline { + break None; + } + std::thread::sleep(PROBE_INTERVAL); + }; + + let Some((vm_id, jail_root, vmm_pid)) = orphan else { + let output = child.wait_with_output()?; + bail!( + "No jailed VMM appeared within {PROBE_TIMEOUT:?}, so nothing was orphaned and the sweep is untested.\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + }; + + kill_pid(child.id(), libc::SIGKILL); + drop(child.wait()); + + if !jail_root.exists() { + bail!( + "The chroot {jail_root} was reclaimed despite the runner being killed without unwinding, so the sweep is untested" + ); + } + + // Reap the VMM the killed runner left behind. The sweep reclaims the + // chroot but nothing reaps an orphaned VMM or its cgroup, so without this + // a stray Firecracker would burn benchmark cores for the rest of the + // suite. Compensating for a known gap, not hiding one. + kill_pid(vmm_pid, libc::SIGKILL); + + println!(" orphaned jail {vm_id} (VMM pid {vmm_pid}), running a second job..."); + + let output = run_runner(image_path, args, runner_bin)?; + + if jail_root.exists() { + bail!("The orphaned chroot {jail_root} survived the next job, so it was never swept"); + } + + Ok(output) +} + +/// Send a signal to a process, ignoring the result. +fn kill_pid(pid: u32, signal: libc::c_int) { + #[expect( + unsafe_code, + clippy::cast_possible_wrap, + reason = "libc::kill requires unsafe; PID fits in i32" + )] + // SAFETY: `kill` takes plain integers and touches no memory. A signal to a + // pid that has already exited fails harmlessly with ESRCH. + unsafe { + libc::kill(pid as i32, signal); + } +} + /// Run the runner while checking a host-side invariant. fn run_runner_with_probe( image_path: &Utf8Path, From f0862d13376377b6d0aaf02e32c39e7c17dd305b Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 04:06:31 +0000 Subject: [PATCH 14/91] Prepare the host on demand rather than at daemon startup runner up required root just to start. prepare_host ran at startup, and unprivileged it cannot create the state directory, cannot create /run/netns, and cannot unshare, so the daemon died after preflight and never reached polling. That broke the API smoke tests and contradicts documented behavior: a Runner serving only non-sandboxed Specs is a supported configuration, and the daemon cannot know its Specs at startup because it learns them from the server. Preparation now happens immediately before the first job that builds a jail, on both entry points, which also retires the special case the one-shot path carried for the same reason. It stays fatal, since a sandboxed job that cannot be confined must not run, and it is not remembered on failure, so a transient permission problem is retried by the next job rather than needing a restart. The sweep still runs before any jail exists in the process, which is what its purpose requires. Verified unprivileged on Linux: the daemon reaches Connecting to channel, and the state directory is not created. Ordering matters and is deliberate: preparation takes the jail lock and releases it before the job takes it. flock is per open file description, so nesting the two would block on itself; that is now spelled out on the lock. The sweep reports what it reclaimed. Each leftover held a VMM binary and a full guest rootfs image, and an operator never heard about any of it. --- plus/bencher_runner/src/jail/mod.rs | 157 +++++++++++++++--- plus/bencher_runner/src/run.rs | 7 - plus/bencher_runner/src/up/job.rs | 9 +- plus/bencher_runner/src/up/mod.rs | 11 +- plus/bencher_runner/src/vm.rs | 16 +- .../docs-reference/runner/de/runner-run.mdx | 2 +- .../docs-reference/runner/de/runner-up.mdx | 2 +- .../docs-reference/runner/en/runner-run.mdx | 2 +- .../docs-reference/runner/en/runner-up.mdx | 2 +- .../docs-reference/runner/es/runner-run.mdx | 2 +- .../docs-reference/runner/es/runner-up.mdx | 2 +- .../docs-reference/runner/fr/runner-run.mdx | 2 +- .../docs-reference/runner/fr/runner-up.mdx | 2 +- .../docs-reference/runner/ja/runner-run.mdx | 2 +- .../docs-reference/runner/ja/runner-up.mdx | 2 +- .../docs-reference/runner/ko/runner-run.mdx | 2 +- .../docs-reference/runner/ko/runner-up.mdx | 2 +- .../docs-reference/runner/pt/runner-run.mdx | 2 +- .../docs-reference/runner/pt/runner-up.mdx | 2 +- .../docs-reference/runner/ru/runner-run.mdx | 2 +- .../docs-reference/runner/ru/runner-up.mdx | 2 +- .../docs-reference/runner/zh/runner-run.mdx | 2 +- .../docs-reference/runner/zh/runner-up.mdx | 2 +- 23 files changed, 173 insertions(+), 63 deletions(-) diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 4456d0043..d5478d938 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -28,10 +28,13 @@ pub use chroot::JailDir; #[cfg(target_os = "linux")] pub use lock::JailLock; #[cfg(target_os = "linux")] -pub use paths::{ChrootPath, HostPath, JailFile, JailPaths}; +pub use paths::{ChrootPath, HostPath, JailFile, JailPaths, SocketPath}; #[cfg(target_os = "linux")] pub use state::StateDir; +#[cfg(target_os = "linux")] +use std::sync::atomic::{AtomicBool, Ordering}; + use serde::{Deserialize, Serialize}; /// Default location of the runner's persistent state directory. @@ -61,17 +64,46 @@ pub const DEFAULT_JAIL_UID: u32 = 61016; /// See [`DEFAULT_JAIL_UID`]. pub const DEFAULT_JAIL_GID: u32 = 61016; -/// The uid and gid the jailed Firecracker VMM drops to. +/// The unprivileged uid and gid the jailed Firecracker VMM drops to. /// /// A host process owning this uid can signal the VMM and, depending on the /// `ptrace` scope, trace it, so it must not be an id the host allocates to /// anything else. +/// +/// The fields are private because `0` must never reach them. The whole +/// sandbox is built by dropping privilege, so a jail user of root is not a +/// weaker jail, it is no jail at all: untrusted code would run against a root +/// VMM, which is the one thing the confinement exists to prevent. An operator +/// hitting a permission error is exactly the person most likely to try it. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct JailUser { + uid: u32, + gid: u32, +} + +impl JailUser { + /// Build a jail user, rejecting root. + pub fn new(uid: u32, gid: u32) -> Result { + if uid == 0 { + return Err(crate::error::JailError::PrivilegedJailUser { field: "uid" }); + } + if gid == 0 { + return Err(crate::error::JailError::PrivilegedJailUser { field: "gid" }); + } + Ok(Self { uid, gid }) + } + /// The uid the VMM drops to. - pub uid: u32, + #[must_use] + pub fn uid(self) -> u32 { + self.uid + } + /// The gid the VMM drops to. - pub gid: u32, + #[must_use] + pub fn gid(self) -> u32 { + self.gid + } } impl Default for JailUser { @@ -83,23 +115,50 @@ impl Default for JailUser { } } -/// Prepare the host for jailed execution. +/// Set once the host has been prepared, so it happens at most once per +/// runner process. +#[cfg(target_os = "linux")] +static HOST_PREPARED: AtomicBool = AtomicBool::new(false); + +/// Prepare the host for jailed execution, once per runner process. /// -/// Idempotent, and called from every entry point that can reach the VM -/// executor: the `up` daemon has a startup hook, the one-shot `run` CLI does -/// not, so the work lives here rather than in daemon startup. +/// Called on demand, immediately before the first job builds a jail, never at +/// startup. The daemon learns which Specs it serves from the server, so at +/// startup it cannot know whether it will ever need a jail, and a Runner that +/// serves only non-sandboxed Specs is a supported configuration that must come +/// up on a host where the runner is not root. Preparing eagerly would make +/// `runner up` require root just to start. /// /// Failure is fatal. Untrusted code never runs with silently degraded -/// confinement, so a host that cannot be prepared does not execute a job. +/// confinement, so a host that cannot be prepared does not execute a job. A +/// failure is not remembered, so a transient permission problem is retried by +/// the next job rather than requiring a restart. +#[cfg(target_os = "linux")] +pub fn prepare_host_once( + state_dir: &camino::Utf8Path, + jail_user: JailUser, +) -> Result<(), crate::error::JailError> { + if HOST_PREPARED.load(Ordering::SeqCst) { + return Ok(()); + } + prepare_host(state_dir, jail_user)?; + HOST_PREPARED.store(true, Ordering::SeqCst); + Ok(()) +} + +/// Create the state directory and reclaim what a previous runner left behind. /// -/// The sweep and the network namespace handle are both taken under the jail -/// lock. The sweep removes every chroot it finds on the reasoning that jobs -/// are serial, so it must not run while another runner has one in flight, and -/// two processes rebinding the namespace handle at once can stack mounts on -/// it. Holding the lock makes serialization a constraint rather than an -/// assumption. +/// The sweep is taken under the jail lock: it removes every chroot it finds on +/// the reasoning that jobs are serial, so it must not run while another runner +/// has one in flight. Running before any jail exists in this process is what +/// the sweep's purpose actually requires. +/// +/// The network namespace is deliberately not built here. It is a process- +/// global object on a tmpfs, so it is rebuilt per job rather than once per +/// daemon lifetime. #[cfg(target_os = "linux")] -pub fn prepare_host( +#[expect(clippy::print_stdout, reason = "host preparation reports what it did")] +fn prepare_host( state_dir: &camino::Utf8Path, jail_user: JailUser, ) -> Result<(), crate::error::JailError> { @@ -109,8 +168,12 @@ pub fn prepare_host( warn_on_named_account(jail_user); let _lock = JailLock::acquire(state.path())?; - state::sweep_jails(&state.jail_parent()); - netns::ensure()?; + let swept = state::sweep_jails(&state.jail_parent()); + if swept > 0 { + // Each one held a copy of the VMM binary and a full guest rootfs + // image, so an operator should hear about it. + println!(" Reclaimed {swept} stale jail(s) from {state_dir}"); + } Ok(()) } @@ -124,7 +187,7 @@ pub fn prepare_host( #[cfg(target_os = "linux")] #[expect(clippy::print_stderr, reason = "host preparation prints diagnostics")] fn warn_on_named_account(jail_user: JailUser) { - let JailUser { uid, gid } = jail_user; + let (uid, gid) = (jail_user.uid(), jail_user.gid()); if let Some(name) = passwd_name(uid) { eprintln!( "Warning: jail uid {uid} belongs to the existing account '{name}'. That account can signal the jailed VMM; pass --jail-uid to pick an unallocated id." @@ -139,9 +202,13 @@ fn warn_on_named_account(jail_user: JailUser) { /// The account name for a uid, read from `/etc/passwd`. /// -/// Deliberately not a `getpwuid` call: the runner ships as a self-contained -/// binary and pulling in NSS would make it depend on the host's resolver -/// configuration. A local account is what matters here, and that is the file. +/// Best effort, and blind to anything the local files do not know about: a +/// host backed by LDAP, Active Directory, or SSSD allocates ids that never +/// appear here, and those are the hosts most likely to allocate in this range +/// at all. Deliberately not a `getpwuid` call even so, because the runner +/// ships as a self-contained binary and NSS would make it depend on the host's +/// resolver configuration. This catches the cheap case; it is not a guarantee +/// that the id is unallocated. #[cfg(target_os = "linux")] fn passwd_name(uid: u32) -> Option { lookup_name("/etc/passwd", uid) @@ -178,7 +245,7 @@ fn lookup_name_in(database: &str, id: u32) -> Option { /// /// The jail is Linux-only, as is the VM executor it protects. #[cfg(not(target_os = "linux"))] -pub fn prepare_host( +pub fn prepare_host_once( _state_dir: &camino::Utf8Path, _jail_user: JailUser, ) -> Result<(), crate::error::JailError> { @@ -293,6 +360,50 @@ mod tests { } } + #[cfg(target_os = "linux")] + #[test] + fn preparation_is_lazy_and_happens_at_most_once() { + // A daemon that prepared at startup would need root just to come up, + // which breaks a Runner serving only non-sandboxed Specs. Nothing may + // touch the state directory until a job actually builds a jail. + let dir = tempfile::tempdir().unwrap(); + let root = camino::Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let state_dir = root.join("state"); + assert!(!state_dir.exists(), "startup has not prepared anything"); + + prepare_host_once(&state_dir, JailUser::default()).unwrap(); + assert!(state_dir.join("jail").is_dir(), "the first job prepares"); + + // A second job must not redo it: proven by removing the tree and + // seeing that it is not rebuilt. + std::fs::remove_dir_all(&state_dir).unwrap(); + prepare_host_once(&state_dir, JailUser::default()).unwrap(); + assert!(!state_dir.exists(), "preparation happens at most once"); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_jail_user_rejects_root() { + // Untrusted code against a root VMM is the one thing the confinement + // exists to prevent, so this must not be reachable by a typo. + JailUser::new(0, DEFAULT_JAIL_GID).unwrap_err(); + JailUser::new(DEFAULT_JAIL_UID, 0).unwrap_err(); + JailUser::new(0, 0).unwrap_err(); + + let user = JailUser::new(1234, 5678).unwrap(); + assert_eq!(user.uid(), 1234); + assert_eq!(user.gid(), 5678); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_default_jail_user_is_unprivileged() { + let default = JailUser::default(); + assert_eq!(default.uid(), DEFAULT_JAIL_UID); + assert_eq!(default.gid(), DEFAULT_JAIL_GID); + JailUser::new(default.uid(), default.gid()).unwrap(); + } + #[cfg(target_os = "linux")] #[test] fn a_named_account_is_found_by_id() { diff --git a/plus/bencher_runner/src/run.rs b/plus/bencher_runner/src/run.rs index b648daac4..f9a33cb19 100644 --- a/plus/bencher_runner/src/run.rs +++ b/plus/bencher_runner/src/run.rs @@ -156,13 +156,6 @@ pub fn run_with_args(args: &RunArgs) -> Result<(), RunnerError> { let mut config = build_config_from_run_args(args)?; - // Prepare the host only for the sandboxed path: the one-shot CLI can - // execute on the host without a jail, and that path has no business - // requiring the runner's state directory. - if config.sandbox.is_some() { - crate::jail::prepare_host(&config.state_dir, config.jail_user)?; - } - // Detect the CPU layout after tuning (disabling SMT changes the core // count), steer kernel work off the benchmark cores, and pin the run // to them. Mirrors the `runner up` path. Core pinning is core runner diff --git a/plus/bencher_runner/src/up/job.rs b/plus/bencher_runner/src/up/job.rs index ec462b87e..d26f9c463 100644 --- a/plus/bencher_runner/src/up/job.rs +++ b/plus/bencher_runner/src/up/job.rs @@ -850,16 +850,13 @@ mod tests { // reach the job, or the VMM shares a uid with a local account that can // signal it. let mut up_config = test_up_config(); - up_config.jail_user = crate::jail::JailUser { - uid: 4242, - gid: 4243, - }; + up_config.jail_user = crate::jail::JailUser::new(4242, 4243).unwrap(); let job = test_job(1, mib_to_bytes(512), mib_to_bytes(1024), 300, false); let result = build_config_from_job(&up_config, &job).unwrap(); - assert_eq!(result.jail_user.uid, 4242); - assert_eq!(result.jail_user.gid, 4243); + assert_eq!(result.jail_user.uid(), 4242); + assert_eq!(result.jail_user.gid(), 4243); } #[test] diff --git a/plus/bencher_runner/src/up/mod.rs b/plus/bencher_runner/src/up/mod.rs index 6d0b140fc..a84e65649 100644 --- a/plus/bencher_runner/src/up/mod.rs +++ b/plus/bencher_runner/src/up/mod.rs @@ -108,12 +108,11 @@ impl Up { // Warn about host conditions that limit benchmark accuracy (Linux only) preflight::print_host_warnings(); - // Create the state directory, sweep chroots left behind by a runner - // that exited without unwinding, and ensure the empty network - // namespace. The daemon claims sandboxed jobs, so this is required - // before the first one arrives, not on demand. - crate::jail::prepare_host(&self.config.state_dir, self.config.jail_user) - .map_err(crate::RunnerError::from)?; + // The host is deliberately NOT prepared here. A Runner that serves + // only non-sandboxed Specs is a supported configuration and must come + // up without root, and the daemon learns its Specs from the server, so + // it cannot know at startup whether it will ever build a jail. + // Preparation happens on demand, before the first job that does. println!(" State directory: {}", self.config.state_dir); // Serialize host-global tuning across runner processes. Declared diff --git a/plus/bencher_runner/src/vm.rs b/plus/bencher_runner/src/vm.rs index d5d2c1efd..593365043 100644 --- a/plus/bencher_runner/src/vm.rs +++ b/plus/bencher_runner/src/vm.rs @@ -30,18 +30,27 @@ pub fn vm_execute( let state_dir = StateDir::new(config.state_dir.clone()); + // Prepare the host on demand, before the first jail this process builds. + // Must come before the lock is taken: preparation takes the same lock, and + // `flock` is per open file description, so nesting would block on itself. + crate::jail::prepare_host_once(state_dir.path(), config.jail_user)?; + // Held for the whole job. Another runner's sweep removes every chroot it // finds, so it must not run while this one is live. Declared before the // jail guard so the lock outlives the teardown it protects. let _lock = JailLock::acquire(state_dir.path())?; + // Rebuilt per job rather than once per daemon lifetime: the handle lives + // on a tmpfs and is operator visible, so it has to be self-healing. + let netns = netns::ensure()?; + // The jail root is a function of the VM id, and the job's artifacts are // built inside it rather than copied in afterwards, so the id is minted // before any of them exist. Dropping this guard removes the chroot tree, // which is what the workspace temp directory used to cover. let vm_id = uuid::Uuid::new_v4().to_string(); let jail_dir = JailDir::create(&state_dir, &vm_id)?; - let jail = JailPaths::new(jail_dir.root()); + let jail = JailPaths::new(jail_dir.root())?; println!(" Jail: {}", jail.root()); let workspace = prepare_oci_workspace(config)?; @@ -96,7 +105,7 @@ pub fn vm_execute( chroot::chown_to_jail(kernel_dest, config.jail_user)?; // Step 7-8: Build Firecracker config and run the microVM - let fc_config = build_firecracker_config(config, work_dir, vm_id, &state_dir, jail)?; + let fc_config = build_firecracker_config(config, work_dir, vm_id, &state_dir, jail, netns)?; let run_output = run_firecracker(&fc_config, cancel_flag)?; @@ -110,6 +119,7 @@ fn build_firecracker_config( vm_id: String, state_dir: &StateDir, jail: JailPaths, + netns: Utf8PathBuf, ) -> Result { // The jailer copies `--exec-file` into the chroot itself and rejects a // multiply linked file, so Firecracker is staged outside the jail and is @@ -155,7 +165,7 @@ fn build_firecracker_config( jail, jail_user: config.jail_user, chroot_base_dir: state_dir.chroot_base(), - netns: netns::handle_path(), + netns, vcpus, memory_mib, boot_args: config.kernel_cmdline.clone(), diff --git a/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx index 70f755de5..949b052f0 100644 --- a/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx @@ -36,7 +36,7 @@ Standardmäßig wird `300` verwendet. Das persistente Zustandsverzeichnis für den Runner. Das Jail, das die Sandbox einschließt, wird unter diesem Verzeichnis erstellt, -und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor der Ausführung eines Jobs daraus entfernt. +und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor dem ersten Sandbox-Job daraus entfernt. Standardmäßig wird `/var/lib/bencher-runner` verwendet. Kann auch mit der Umgebungsvariable `BENCHER_STATE_DIR` gesetzt werden. diff --git a/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx index 1d3c1f5d0..e0e0bfc9a 100644 --- a/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx @@ -39,7 +39,7 @@ Standardmäßig wird `55` verwendet. Das persistente Zustandsverzeichnis für den Runner. Das Jail, das die Sandbox einschließt, wird unter diesem Verzeichnis erstellt, -und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor der Ausführung eines Jobs daraus entfernt. +und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor dem ersten Sandbox-Job daraus entfernt. Standardmäßig wird `/var/lib/bencher-runner` verwendet. Kann auch mit der Umgebungsvariable `BENCHER_STATE_DIR` gesetzt werden. diff --git a/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx index c16060ed4..4addb2cc8 100644 --- a/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx @@ -36,7 +36,7 @@ By default, `300` is used. The persistent state directory for the Runner. The jail that confines the sandbox is built under this directory, -and jails left behind by an unclean exit are swept from it before a Job runs. +and jails left behind by an unclean exit are swept from it before the first sandboxed Job runs. By default, `/var/lib/bencher-runner` is used. Can also be set with the `BENCHER_STATE_DIR` environment variable. diff --git a/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx index 313387328..310463bee 100644 --- a/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx @@ -39,7 +39,7 @@ By default, `55` is used. The persistent state directory for the Runner. The jail that confines the sandbox is built under this directory, -and jails left behind by an unclean exit are swept from it before a Job runs. +and jails left behind by an unclean exit are swept from it before the first sandboxed Job runs. By default, `/var/lib/bencher-runner` is used. Can also be set with the `BENCHER_STATE_DIR` environment variable. diff --git a/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx index cbe53278b..a02fc9067 100644 --- a/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx @@ -36,7 +36,7 @@ Por defecto, se usa `300`. El directorio de estado persistente del Runner. La jaula que confina el sandbox se crea dentro de este directorio, -y las jaulas que quedaron tras una salida no limpia se eliminan de él antes de ejecutar un Job. +y las jaulas que quedaron tras una salida no limpia se eliminan de él antes del primer Job con sandbox. Por defecto, se usa `/var/lib/bencher-runner`. También se puede establecer con la variable de entorno `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx index 1f1442d8d..874610834 100644 --- a/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx @@ -39,7 +39,7 @@ Por defecto, se usa `55`. El directorio de estado persistente del Runner. La jaula que confina el sandbox se crea dentro de este directorio, -y las jaulas que quedaron tras una salida no limpia se eliminan de él antes de ejecutar un Job. +y las jaulas que quedaron tras una salida no limpia se eliminan de él antes del primer Job con sandbox. Por defecto, se usa `/var/lib/bencher-runner`. También se puede establecer con la variable de entorno `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx index cdee6d8ab..9148efa52 100644 --- a/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx @@ -36,7 +36,7 @@ Par défaut, `300` est utilisé. Le répertoire d'état persistant du Runner. La prison qui confine le bac à sable est créée dans ce répertoire, -et les prisons laissées par un arrêt brutal en sont supprimées avant l'exécution d'un Job. +et les prisons laissées par un arrêt brutal en sont supprimées avant le premier Job avec bac à sable. Par défaut, `/var/lib/bencher-runner` est utilisé. Peut également être défini avec la variable d'environnement `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx index e3aaea32a..f0608d490 100644 --- a/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx @@ -39,7 +39,7 @@ Par défaut, `55` est utilisé. Le répertoire d'état persistant du Runner. La prison qui confine le bac à sable est créée dans ce répertoire, -et les prisons laissées par un arrêt brutal en sont supprimées avant l'exécution d'un Job. +et les prisons laissées par un arrêt brutal en sont supprimées avant le premier Job avec bac à sable. Par défaut, `/var/lib/bencher-runner` est utilisé. Peut également être défini avec la variable d'environnement `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx index 9d7de68a6..10cc5cfb6 100644 --- a/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx @@ -36,7 +36,7 @@ Firecracker microVM には `firecracker` を使用します (Linux のみ)。 Runner の永続的な状態ディレクトリ。 サンドボックスを閉じ込める jail はこのディレクトリの下に作成され、 -異常終了で残った jail は Job の実行前にここから一掃されます。 +異常終了で残った jail は、最初のサンドボックス Job の実行前にここから一掃されます。 デフォルトでは `/var/lib/bencher-runner` が使用されます。 環境変数 `BENCHER_STATE_DIR` でも設定できます。 diff --git a/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx index abd2ee11c..9f9b4a0ad 100644 --- a/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx @@ -38,7 +38,7 @@ Job を待機する際のロングポーリングのタイムアウト (秒)。` Runner の永続的な状態ディレクトリ。 サンドボックスを閉じ込める jail はこのディレクトリの下に作成され、 -異常終了で残った jail は Job の実行前にここから一掃されます。 +異常終了で残った jail は、最初のサンドボックス Job の実行前にここから一掃されます。 デフォルトでは `/var/lib/bencher-runner` が使用されます。 環境変数 `BENCHER_STATE_DIR` でも設定できます。 diff --git a/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx index 40525fd5a..d2636ddd7 100644 --- a/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx @@ -36,7 +36,7 @@ Firecracker microVM(Linux 전용)을 사용하려면 `firecracker`를 사용하 Runner의 영구 상태 디렉터리입니다. 샌드박스를 가두는 jail이 이 디렉터리 아래에 생성되며, -비정상 종료로 남겨진 jail은 Job 실행 전에 이곳에서 정리됩니다. +비정상 종료로 남겨진 jail은 첫 번째 샌드박스 Job 실행 전에 이곳에서 정리됩니다. 기본적으로 `/var/lib/bencher-runner`가 사용됩니다. `BENCHER_STATE_DIR` 환경 변수로도 설정할 수 있습니다. diff --git a/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx index bc801e149..be67c7ed1 100644 --- a/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx @@ -39,7 +39,7 @@ Job을 기다리는 동안의 롱 폴(long-poll) 타임아웃(초)으로, `1`에 Runner의 영구 상태 디렉터리입니다. 샌드박스를 가두는 jail이 이 디렉터리 아래에 생성되며, -비정상 종료로 남겨진 jail은 Job 실행 전에 이곳에서 정리됩니다. +비정상 종료로 남겨진 jail은 첫 번째 샌드박스 Job 실행 전에 이곳에서 정리됩니다. 기본적으로 `/var/lib/bencher-runner`가 사용됩니다. `BENCHER_STATE_DIR` 환경 변수로도 설정할 수 있습니다. diff --git a/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx index d7a1ce6e7..7be345481 100644 --- a/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx @@ -36,7 +36,7 @@ Por padrão, `300` é usado. O diretório de estado persistente do Runner. A jail que confina o sandbox é criada sob este diretório, -e as jails deixadas por um encerramento não limpo são removidas dele antes de um Job ser executado. +e as jails deixadas por um encerramento não limpo são removidas dele antes do primeiro Job com sandbox. Por padrão, `/var/lib/bencher-runner` é usado. Também pode ser definido com a variável de ambiente `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx index 214442783..d568f8bc1 100644 --- a/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx @@ -39,7 +39,7 @@ Por padrão, `55` é usado. O diretório de estado persistente do Runner. A jail que confina o sandbox é criada sob este diretório, -e as jails deixadas por um encerramento não limpo são removidas dele antes de um Job ser executado. +e as jails deixadas por um encerramento não limpo são removidas dele antes do primeiro Job com sandbox. Por padrão, `/var/lib/bencher-runner` é usado. Também pode ser definido com a variável de ambiente `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx index 7f0b1920b..9257dd18d 100644 --- a/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx @@ -36,7 +36,7 @@ JWT-токен для аутентификации в реестре при за Постоянный каталог состояния Runner. Изолирующая среда (jail), ограничивающая песочницу, создаётся внутри этого каталога, -а среды, оставшиеся после аварийного завершения, удаляются из него перед запуском Job. +а среды, оставшиеся после аварийного завершения, удаляются из него перед запуском первого Job с песочницей. По умолчанию используется `/var/lib/bencher-runner`. Также может быть задан переменной окружения `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx index f7e983924..dfe388977 100644 --- a/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx @@ -39,7 +39,7 @@ UUID или slug Runner, от имени которого работать. Постоянный каталог состояния Runner. Изолирующая среда (jail), ограничивающая песочницу, создаётся внутри этого каталога, -а среды, оставшиеся после аварийного завершения, удаляются из него перед запуском Job. +а среды, оставшиеся после аварийного завершения, удаляются из него перед запуском первого Job с песочницей. По умолчанию используется `/var/lib/bencher-runner`. Также может быть задан переменной окружения `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx index 85b05cf95..2f6bef16d 100644 --- a/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx @@ -36,7 +36,7 @@ runner run --image [OPTIONS] Runner 的持久状态目录。 限制沙箱的 jail 在该目录下创建， -非正常退出遗留的 jail 会在 Job 运行前从中清除。 +非正常退出遗留的 jail 会在第一个沙箱 Job 运行前从中清除。 默认使用 `/var/lib/bencher-runner`。 也可以通过 `BENCHER_STATE_DIR` 环境变量设置。 diff --git a/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx index 3ae2067d2..3425c651b 100644 --- a/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx @@ -38,7 +38,7 @@ runner up [OPTIONS] Runner 的持久状态目录。 限制沙箱的 jail 在该目录下创建， -非正常退出遗留的 jail 会在 Job 运行前从中清除。 +非正常退出遗留的 jail 会在第一个沙箱 Job 运行前从中清除。 默认使用 `/var/lib/bencher-runner`。 也可以通过 `BENCHER_STATE_DIR` 环境变量设置。 From 9b1bbbf17f6d57615d4aa438ef1eec2ca0a77f9b Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 04:06:47 +0000 Subject: [PATCH 15/91] Reject a root jail user, and harden the namespace and state directory --jail-uid 0 was accepted and silently defeated the whole jail: the sandbox is built by dropping privilege, so a jail user of root is not a weaker jail but no jail at all, and untrusted code would run against a root VMM. It is a plausible typo and an even more plausible fix for an operator hitting a permission error. JailUser now validates and carries private fields, so 0 cannot reach the jailer through the flag, the environment variable, or the library. The flags carry a range parser as well. The network namespace is rebuilt rather than reused. Proving a handle is a namespace and is not the runner's own does not prove it is empty: a bencher-jail left by an operator experimenting with ip netns could hold interfaces, and the VMM would silently regain the host network reach this exists to remove. Recreating is cheaper and stronger than asserting a namespace holds nothing but a down lo. It is also rebuilt per job rather than once per lifetime, since the handle lives on a tmpfs and is operator visible, and it takes its own lock. The jail lock is scoped to a state directory while the namespace is process global, so two runners started with different --state-dir values held different locks and could still stack mounts on the same handle, which is the race the lock was added to close. StateDir::create refuses a root that already exists, is not empty, and carries nothing the runner put there. It applies 0700 on every call so an older runner's laxer directory is tightened, which pointed at --state-dir /var/lib would have chmodded that directory and taken the host down. The named-account warning no longer implies more than it delivers: it reads the local files, so it is blind to the LDAP, Active Directory, and SSSD hosts most likely to allocate in this range. --- plus/bencher_runner/build.rs | 2 +- plus/bencher_runner/src/error.rs | 47 +++++++++++++ plus/bencher_runner/src/jail/chroot.rs | 3 +- plus/bencher_runner/src/jail/lock.rs | 14 +++- plus/bencher_runner/src/jail/netns.rs | 69 +++++++++++++++++-- plus/bencher_runner/src/jail/state.rs | 91 +++++++++++++++++++++++++- services/runner/src/parser/mod.rs | 14 +++- services/runner/src/parser/up.rs | 14 +++- services/runner/src/runner/run.rs | 7 +- services/runner/src/runner/up.rs | 7 +- 10 files changed, 244 insertions(+), 24 deletions(-) diff --git a/plus/bencher_runner/build.rs b/plus/bencher_runner/build.rs index 2f103aa4b..3ef0f678c 100644 --- a/plus/bencher_runner/build.rs +++ b/plus/bencher_runner/build.rs @@ -22,7 +22,7 @@ //! //! - `BENCHER_INIT_PATH` — path to a pre-built bencher-init binary //! - `BENCHER_FIRECRACKER_PATH` — path to a pre-built firecracker binary -//! - `BENCHER_JAILER_PATH` — path to a pre-built jailer binary +//! - `BENCHER_JAILER_PATH`: path to a pre-built jailer binary //! - `BENCHER_KERNEL_PATH` — path to a pre-built vmlinux kernel #![expect( diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index e06fd0f2f..57e628922 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -59,6 +59,16 @@ pub enum JailError { #[error("Cpuset partition mode '{mode}' rejected by the kernel: {state}")] PartitionInvalid { mode: String, state: String }, + #[error( + "The jail {field} must not be 0: the sandbox is built by dropping privilege, so a jail user of root is no jail at all" + )] + PrivilegedJailUser { field: &'static str }, + + #[error( + "The state directory {path} already exists, is not empty, and was not created by the runner. Point --state-dir at a directory the runner owns." + )] + ForeignStateDir { path: Utf8PathBuf }, + #[error("Failed to create runner state directory {path}: {source}")] CreateStateDir { path: Utf8PathBuf, @@ -108,6 +118,43 @@ pub enum JailError { source: std::io::Error, }, + #[cfg(target_os = "linux")] + #[error("Failed to open the network namespace lock {path}: {source}")] + OpenNetnsLock { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to take the network namespace lock {path}: {source}")] + NetnsLock { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error( + "The network namespace handle {path} is not a namespace distinct from the runner's own" + )] + NetnsNotDistinct { path: Utf8PathBuf }, + + #[cfg(target_os = "linux")] + #[error("Failed to open the jail chroot {path}: {source}")] + OpenJailRoot { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error( + "The socket path {path} is {length} bytes, over the {limit} byte sun_path limit for a Unix domain socket" + )] + SocketPathTooLong { + path: String, + length: usize, + limit: usize, + }, + #[cfg(target_os = "linux")] #[error("Failed to create jail chroot {path}: {source}")] CreateJail { diff --git a/plus/bencher_runner/src/jail/chroot.rs b/plus/bencher_runner/src/jail/chroot.rs index 10f2d1606..d61244f2a 100644 --- a/plus/bencher_runner/src/jail/chroot.rs +++ b/plus/bencher_runner/src/jail/chroot.rs @@ -77,8 +77,7 @@ impl Drop for JailDir { /// has to be handed over explicitly, and getting it wrong produces an opaque /// boot failure, so each one is checked. pub fn chown_to_jail(path: &Utf8Path, jail_user: JailUser) -> Result<(), JailError> { - let JailUser { uid, gid } = jail_user; - chown(path, Some(uid), Some(gid)).map_err(|e| JailError::ChownJail { + chown(path, Some(jail_user.uid()), Some(jail_user.gid())).map_err(|e| JailError::ChownJail { path: path.to_owned(), source: e, }) diff --git a/plus/bencher_runner/src/jail/lock.rs b/plus/bencher_runner/src/jail/lock.rs index aa0cb523b..c0b641970 100644 --- a/plus/bencher_runner/src/jail/lock.rs +++ b/plus/bencher_runner/src/jail/lock.rs @@ -32,6 +32,13 @@ const LOCK_FILE: &str = ".lock"; /// /// The kernel releases a `flock` when the holder exits or dies, so a crashed /// runner cannot wedge future runs. +/// +/// `flock` is per open file description, not per process, so a second +/// `acquire` on the same path from a process that already holds it opens a new +/// description and blocks on itself forever. Nothing nests today: host +/// preparation takes and releases this lock before a job takes it, and the +/// network namespace uses a different lock file. Any new caller has to keep it +/// that way. #[derive(Debug)] pub struct JailLock { /// The locked file, held only for its `flock`. @@ -62,7 +69,7 @@ impl JailLock { } println!(" Waiting for another bencher runner to release {path}..."); - flock(&file, libc::LOCK_EX).map_err(|e| JailError::JailLock { + flock_exclusive(&file).map_err(|e| JailError::JailLock { path: path.clone(), source: e, })?; @@ -71,6 +78,11 @@ impl JailLock { } } +/// Take an exclusive `flock`, waiting for whichever holder has it. +pub(super) fn flock_exclusive(file: &File) -> std::io::Result<()> { + flock(file, libc::LOCK_EX) +} + /// Apply `flock` to a file, retrying if a signal interrupts the wait. fn flock(file: &File, operation: libc::c_int) -> std::io::Result<()> { loop { diff --git a/plus/bencher_runner/src/jail/netns.rs b/plus/bencher_runner/src/jail/netns.rs index e511d8fb1..085b01066 100644 --- a/plus/bencher_runner/src/jail/netns.rs +++ b/plus/bencher_runner/src/jail/netns.rs @@ -14,6 +14,7 @@ use nix::mount::{MntFlags, MsFlags, mount, umount2}; use nix::sched::{CloneFlags, unshare}; use crate::error::JailError; +use crate::jail::lock::flock_exclusive; /// Directory holding named network namespace handles. /// @@ -37,6 +38,9 @@ const SELF_NETNS: &str = "/proc/self/ns/net"; /// reports the real state either way. const MAX_STACKED_MOUNTS: usize = 32; +/// Lock file serializing access to the global network namespace handle. +const NETNS_LOCK_PATH: &str = "/run/bencher_runner_netns.lock"; + /// The calling *thread's* network namespace. /// /// `/proc/self` resolves through the thread group leader, so it must not be @@ -50,11 +54,24 @@ pub fn handle_path() -> Utf8PathBuf { Utf8Path::new(NETNS_DIR).join(NETNS_NAME) } -/// Ensure the empty network namespace exists, returning its handle path. +/// Build a fresh empty network namespace, returning its handle path. +/// +/// The handle is always cleared and recreated rather than reused. Proving a +/// handle is a namespace and is not the runner's own does not prove it is +/// empty: a `bencher-jail` left by an operator experimenting with `ip netns`, +/// or by a name collision, could hold interfaces, and the VMM would silently +/// regain the host network reach this module exists to remove. Recreating is +/// both cheaper and stronger than trying to assert a namespace holds nothing +/// but a down `lo`. /// -/// Idempotent: a handle that is already a live namespace distinct from the -/// runner's own is reused. Anything else at the path (a leftover placeholder -/// file, or a handle whose mount is gone) is cleared and recreated. +/// Called per job rather than once per daemon lifetime. `/run` is a tmpfs and +/// the handle is a shared, operator-visible object, so an `ip netns del` or a +/// remount would otherwise break every subsequent job until a restart. +/// +/// The namespace is process-global while the jail lock is per state directory, +/// so this takes its own lock: two runners started with different +/// `--state-dir` values hold different jail locks and would otherwise clear +/// and rebind the same handle concurrently. pub fn ensure() -> Result { let handle = handle_path(); @@ -63,9 +80,7 @@ pub fn ensure() -> Result { source: e, })?; - if is_live_netns(&handle) { - return Ok(handle); - } + let _lock = NetnsLock::acquire()?; clear(&handle)?; @@ -80,6 +95,15 @@ pub fn ensure() -> Result { return Err(e); } + // The namespace has to be a real one and not the runner's own, or the VMM + // would keep host network reach. Cheap, and the whole point of the module. + if !is_live_netns(&handle) { + drop(fs::remove_file(&handle)); + return Err(JailError::NetnsNotDistinct { + path: handle.clone(), + }); + } + Ok(handle) } @@ -110,6 +134,37 @@ fn clear(handle: &Utf8Path) -> Result<(), JailError> { } } +/// Advisory lock over the process-global network namespace handle. +/// +/// Separate from the jail lock, which is scoped to a state directory: the +/// handle is a single global object and two runners with different state +/// directories must still not rebind it at the same time. It lives beside the +/// host tuning lock, in root-writable tmpfs that clears on reboot. +struct NetnsLock { + _file: fs::File, +} + +impl NetnsLock { + /// Take the lock, waiting for whichever runner holds it. + fn acquire() -> Result { + let path = Utf8Path::new(NETNS_LOCK_PATH); + let file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(path) + .map_err(|e| JailError::OpenNetnsLock { + path: path.to_owned(), + source: e, + })?; + flock_exclusive(&file).map_err(|e| JailError::NetnsLock { + path: path.to_owned(), + source: e, + })?; + Ok(Self { _file: file }) + } +} + /// Whether `handle` is a live network namespace other than the runner's own. /// /// Every namespace inode lives on the single kernel `nsfs`, so sharing a diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index 9f02479ef..a76855c2e 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -16,6 +16,12 @@ use crate::error::JailError; /// Subdirectory of the state directory used as the jailer's chroot base. const CHROOT_BASE: &str = "jail"; +/// The jail lock file, which lives beside the chroot base. +/// +/// Named here as well as in the lock module so the state directory knows +/// which of its entries it created. +const LOCK_FILE: &str = ".lock"; + /// The `--exec-file` base name the jailer derives the chroot layout from. /// /// The jailer builds `///root`, so the @@ -45,6 +51,34 @@ impl StateDir { &self.root } + /// Refuse a state directory that belongs to the host rather than to us. + /// + /// A path that does not exist, or exists and is empty, is ours to take. A + /// populated one is only ours if it already carries something the runner + /// put there. Without this, `--state-dir /var/lib` would be chmodded to + /// 0700 and take the host down with it. + fn check_root_is_ours(&self) -> Result<(), JailError> { + let Ok(mut entries) = fs::read_dir(&self.root) else { + // Missing, or unreadable: creating it is the next step and will + // report the real error. + return Ok(()); + }; + let mut populated = false; + for entry in entries.by_ref().flatten() { + populated = true; + let name = entry.file_name(); + if RUNNER_ENTRIES.iter().any(|ours| name == *ours) { + return Ok(()); + } + } + if populated { + return Err(JailError::ForeignStateDir { + path: self.root.clone(), + }); + } + Ok(()) + } + /// The jailer's `--chroot-base-dir`. #[must_use] pub fn chroot_base(&self) -> Utf8PathBuf { @@ -74,8 +108,12 @@ impl StateDir { /// Create the state directory tree at mode 0700. /// /// Idempotent. The mode is applied on every call so a directory created - /// with a laxer mode by an older runner is tightened on upgrade. + /// with a laxer mode by an older runner is tightened on upgrade. That + /// tightening is why the root has to be one the runner owns: pointed at a + /// populated system directory it would otherwise chmod that directory to + /// 0700 and break the host. pub fn create(&self) -> Result<(), JailError> { + self.check_root_is_ours()?; for dir in [&self.root, &self.chroot_base(), &self.jail_parent()] { fs::create_dir_all(dir).map_err(|e| JailError::CreateStateDir { path: dir.clone(), @@ -92,6 +130,12 @@ impl StateDir { } } +/// Entries the runner creates directly in its state directory. +/// +/// Their presence is what distinguishes a directory the runner has used from +/// one that belongs to the host. +const RUNNER_ENTRIES: [&str; 2] = [CHROOT_BASE, LOCK_FILE]; + /// Remove every jail directory under `jail_parent`, returning how many were /// reclaimed. /// @@ -185,6 +229,51 @@ mod tests { assert_eq!(mode & 0o777, 0o700); } + #[test] + fn a_populated_foreign_directory_is_refused() { + // The mode tightening would otherwise chmod a system directory to + // 0700: `--state-dir /var/lib` must not take the host down. + let (_dir, root) = temp_root(); + let foreign = root.join("var-lib"); + fs::create_dir_all(foreign.join("dpkg")).unwrap(); + fs::create_dir_all(foreign.join("systemd")).unwrap(); + + StateDir::new(foreign.clone()).create().unwrap_err(); + + let mode = fs::metadata(&foreign).unwrap().permissions().mode(); + assert_ne!(mode & 0o777, 0o700, "a refused root must not be chmodded"); + } + + #[test] + fn an_empty_directory_is_ours_to_take() { + let (_dir, root) = temp_root(); + let empty = root.join("empty"); + fs::create_dir_all(&empty).unwrap(); + + StateDir::new(empty).create().unwrap(); + } + + #[test] + fn a_directory_the_runner_already_used_is_ours() { + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")); + state.create().unwrap(); + // Something the host put there afterwards does not disown it. + fs::write(state.path().join("notes.txt"), b"operator note").unwrap(); + + state.create().unwrap(); + } + + #[test] + fn a_directory_holding_only_the_lock_is_ours() { + let (_dir, root) = temp_root(); + let state = root.join("state"); + fs::create_dir_all(&state).unwrap(); + fs::write(state.join(".lock"), b"").unwrap(); + + StateDir::new(state).create().unwrap(); + } + #[test] fn sweep_removes_stale_jails() { let (_dir, root) = temp_root(); diff --git a/services/runner/src/parser/mod.rs b/services/runner/src/parser/mod.rs index 102969e39..023171cb8 100644 --- a/services/runner/src/parser/mod.rs +++ b/services/runner/src/parser/mod.rs @@ -67,11 +67,21 @@ pub struct CliRun { pub state_dir: Utf8PathBuf, /// Unprivileged uid the jailed sandbox process drops to. - #[arg(long, env = "BENCHER_JAIL_UID", default_value_t = bencher_runner::DEFAULT_JAIL_UID)] + #[arg( + long, + env = "BENCHER_JAIL_UID", + default_value_t = bencher_runner::DEFAULT_JAIL_UID, + value_parser = clap::value_parser!(u32).range(1..), + )] pub jail_uid: u32, /// Unprivileged gid the jailed sandbox process drops to. - #[arg(long, env = "BENCHER_JAIL_GID", default_value_t = bencher_runner::DEFAULT_JAIL_GID)] + #[arg( + long, + env = "BENCHER_JAIL_GID", + default_value_t = bencher_runner::DEFAULT_JAIL_GID, + value_parser = clap::value_parser!(u32).range(1..), + )] pub jail_gid: u32, /// Output file paths inside guest (may be repeated). diff --git a/services/runner/src/parser/up.rs b/services/runner/src/parser/up.rs index 066fff16a..84b700004 100644 --- a/services/runner/src/parser/up.rs +++ b/services/runner/src/parser/up.rs @@ -28,11 +28,21 @@ pub struct CliUp { pub state_dir: Utf8PathBuf, /// Unprivileged uid the jailed sandbox process drops to. - #[arg(long, env = "BENCHER_JAIL_UID", default_value_t = bencher_runner::DEFAULT_JAIL_UID)] + #[arg( + long, + env = "BENCHER_JAIL_UID", + default_value_t = bencher_runner::DEFAULT_JAIL_UID, + value_parser = clap::value_parser!(u32).range(1..), + )] pub jail_uid: u32, /// Unprivileged gid the jailed sandbox process drops to. - #[arg(long, env = "BENCHER_JAIL_GID", default_value_t = bencher_runner::DEFAULT_JAIL_GID)] + #[arg( + long, + env = "BENCHER_JAIL_GID", + default_value_t = bencher_runner::DEFAULT_JAIL_GID, + value_parser = clap::value_parser!(u32).range(1..), + )] pub jail_gid: u32, #[command(flatten)] diff --git a/services/runner/src/runner/run.rs b/services/runner/src/runner/run.rs index ad5e3a87d..38cc950e7 100644 --- a/services/runner/src/runner/run.rs +++ b/services/runner/src/runner/run.rs @@ -13,6 +13,8 @@ impl TryFrom for Run { fn try_from(task: CliRun) -> Result { let tuning = task.tuning.try_into()?; + let jail_user = bencher_runner::JailUser::new(task.jail_uid, task.jail_gid) + .map_err(bencher_runner::RunnerError::from)?; let vcpus = task.vcpus.map(bencher_runner::Cpu::try_from).transpose()?; let memory = task @@ -55,10 +57,7 @@ impl TryFrom for Run { sandbox_log_level: task.sandbox_log_level, sandbox: task.sandbox, state_dir: task.state_dir, - jail_user: bencher_runner::JailUser { - uid: task.jail_uid, - gid: task.jail_gid, - }, + jail_user, }, }) } diff --git a/services/runner/src/runner/up.rs b/services/runner/src/runner/up.rs index cfcd43274..eafdea8fa 100644 --- a/services/runner/src/runner/up.rs +++ b/services/runner/src/runner/up.rs @@ -13,6 +13,8 @@ impl TryFrom for Up { fn try_from(task: CliUp) -> Result { let tuning = task.tuning.try_into()?; + let jail_user = bencher_runner::JailUser::new(task.jail_uid, task.jail_gid) + .map_err(bencher_runner::RunnerError::from)?; Ok(Self { config: UpConfig { @@ -32,10 +34,7 @@ impl TryFrom for Up { update_channel: task.update_channel, max_download_size: task.max_download_size, state_dir: task.state_dir, - jail_user: bencher_runner::JailUser { - uid: task.jail_uid, - gid: task.jail_gid, - }, + jail_user, }, }) } From dc015376fb85be17a7dec8e1ab52523fc4bff840 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 19:00:40 +0000 Subject: [PATCH 16/91] Use a marker the runner cannot print The check written to stop vacuous passes was itself vacuous in the scenario it was written to protect. ScenarioOutput captures the runner's stdout, not the guest's, and the runner prints "Launching jailed Firecracker microVM..." on its way to starting a VM. Matching on "jailed" therefore matched the runner announcing its intent, so the marker check passed while the guest never ran, leaving a non-zero exit code as the only real guard where two were designed. Both scenarios now use tokens the runner's own output cannot contain, following the convention already in this file. "swept" does not collide today but sits one refactor away from the sweep's own reporting. --- tasks/test_runner/src/task/scenarios.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index e834bf58a..f8abcf2f9 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -2656,8 +2656,13 @@ fn jail_scenarios() -> Vec { description: "A jailed job succeeds with the VMM unprivileged and in its cgroup", // The guest sleeps so the VMM is alive long enough to be observed // by a probe that polls every 100ms. + // + // The marker is a token the runner's own output cannot contain. + // "jailed" collided with the runner announcing "Launching jailed + // Firecracker microVM...", so the check passed on the runner + // saying it was about to start a VM that then never booted. dockerfile: r#"FROM busybox -CMD ["sh", "-c", "echo jailed && sleep 5"]"#, +CMD ["sh", "-c", "echo JAIL_CONFINEMENT_a7f3b2c9 && sleep 5"]"#, cancel_after_secs: None, probe: Some(probe_confinement), orphan_then_rerun: false, @@ -2669,22 +2674,24 @@ CMD ["sh", "-c", "echo jailed && sleep 5"]"#, // checks is equally true of a VMM that started and then never // booted a guest, so without this the scenario stays green // while the product is broken. - assert_job_succeeded(output, "jailed")?; + assert_job_succeeded(output, "JAIL_CONFINEMENT_a7f3b2c9")?; assert_no_chroot_remains(&scenario_state_dir()) }, }, Scenario { name: "jail_sweep_reclaims_orphan", description: "A chroot orphaned by a runner that never unwound is swept by the next job", + // Likewise a token the runner cannot print: "swept" sits one + // refactor away from colliding with the sweep's own reporting. dockerfile: r#"FROM busybox -CMD ["sh", "-c", "echo swept && sleep 10"]"#, +CMD ["sh", "-c", "echo JAIL_SWEEP_a7f3b2c9 && sleep 10"]"#, cancel_after_secs: None, probe: None, orphan_then_rerun: true, sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { - assert_job_succeeded(output, "swept")?; + assert_job_succeeded(output, "JAIL_SWEEP_a7f3b2c9")?; assert_no_chroot_remains(&scenario_state_dir()) }, }, @@ -2697,6 +2704,10 @@ CMD ["sh", "-c", "echo swept && sleep 10"]"#, /// the VM never boots: the VMM process exists, is unprivileged, and is in its /// cgroup either way. Success of the job itself is the precondition for any of /// that meaning anything. +/// +/// `marker` has to be a token the runner's own progress output cannot contain. +/// This captures the runner's stdout, not the guest's, and the runner prints +/// plenty about jails and sweeps on its way to launching a VM. fn assert_job_succeeded(output: &ScenarioOutput, marker: &str) -> Result<()> { if output.exit_code != 0 { bail!( From e44e79d64c58f3634c452d43add2e458d1aae40d Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 19:00:56 +0000 Subject: [PATCH 17/91] Reap the VMM and cgroup a killed runner leaves behind The sweep reclaimed the chroot and left the more damaging half. A runner that is SIGKILLed, crashes, or execs itself during a self-update does not signal its jailed VMM, so the VMM is reparented and keeps running, holding the exclusive benchmark CPUs through a cgroup nothing removes. The consequence is not leakage, it is wrong numbers that look right. The next job's cpuset write is rejected by the cgroup still owning those CPUs, and that failure was swallowed twice over: apply_cpuset returned Ok on every internal failure and the caller only warned on top of it. Every subsequent run would report success while measuring somewhere other than where it claimed, until someone rebooted. A half-applied fidelity mechanism is a confinement-grade failure, so a cpuset that cannot be applied to a cgroup that exists now aborts the job. Failing to create the cgroup at all still degrades, because a declared absence of isolation is not a lie about it. Killing a process the runner does not own is a new destructive capability, so the target is identified as narrowly as it can be: only a process whose root directory is the chroot being swept, compared by device and inode. Verified on a real kernel against a live jailed VMM: exactly one of 125 processes matched. Not "any process owned by the jail uid", which on a shared host may legitimately own something else. The pid is pinned with a pidfd before the signal. A pid found by scanning /proc can exit and have its number recycled before the signal lands, and this runs as root. Holding the descriptor keeps the number from being reused, which turns the identity check into a guarantee rather than a narrow window. Ordering is forced: reap, then remove the tree, then remove the cgroup. Removing the tree first pulls the rootfs from under a process still running, and rmdir on a cgroup that still holds one fails. A cgroup that survives anyway is reported loudly, because a surviving isolated cpuset is exactly the silent degradation this exists to prevent. --- plus/bencher_runner/src/firecracker/error.rs | 7 + plus/bencher_runner/src/firecracker/mod.rs | 23 +- plus/bencher_runner/src/jail/cgroup.rs | 68 ++++-- plus/bencher_runner/src/jail/mod.rs | 2 + plus/bencher_runner/src/jail/reap.rs | 233 +++++++++++++++++++ plus/bencher_runner/src/jail/state.rs | 26 ++- 6 files changed, 321 insertions(+), 38 deletions(-) create mode 100644 plus/bencher_runner/src/jail/reap.rs diff --git a/plus/bencher_runner/src/firecracker/error.rs b/plus/bencher_runner/src/firecracker/error.rs index 99013a023..721725144 100644 --- a/plus/bencher_runner/src/firecracker/error.rs +++ b/plus/bencher_runner/src/firecracker/error.rs @@ -66,6 +66,13 @@ pub enum FirecrackerError { cgroup: camino::Utf8PathBuf, }, + /// The cgroup exists but its cpuset could not be applied. + /// + /// Boxed because [`crate::error::RunnerError`] contains this type, and it + /// in turn contains a `RunnerError`. + #[error("Failed to confine Firecracker to the benchmark cores: {0}")] + CpusetFailed(#[source] Box), + /// A jail artifact could not be handed to the jail uid and gid. #[error("Jail ownership failed: {0}")] Chown(#[source] crate::error::JailError), diff --git a/plus/bencher_runner/src/firecracker/mod.rs b/plus/bencher_runner/src/firecracker/mod.rs index 7ce0f048e..0dcbe76b9 100644 --- a/plus/bencher_runner/src/firecracker/mod.rs +++ b/plus/bencher_runner/src/firecracker/mod.rs @@ -133,15 +133,18 @@ pub fn run_firecracker( if layout.has_isolation() { match CgroupManager::new(vm_id) { Ok(cg) => { - // Apply cpuset to pin Firecracker to benchmark cores - if let Err(e) = cg.apply_cpuset(layout) { - eprintln!("Warning: failed to apply cpuset: {e}"); - } else { - println!( - "CPU isolation: Firecracker pinned to cores {}", - layout.benchmark_cpuset() - ); - } + // A cgroup that exists but has no cpuset does not confine + // the VMM to the benchmark cores, so the run would report + // a number measured somewhere other than where it claims. + // Fatal, unlike failing to create the cgroup at all, which + // is a declared absence of isolation rather than a lie + // about it. + cg.apply_cpuset(layout) + .map_err(|e| FirecrackerError::CpusetFailed(Box::new(e)))?; + println!( + "CPU isolation: Firecracker pinned to cores {}", + layout.benchmark_cpuset() + ); // Keep VM memory resident: swap adds run-to-run variance if let Err(e) = cg.disable_swap() { eprintln!("Warning: failed to disable swap for VM cgroup: {e}"); @@ -225,7 +228,7 @@ pub fn run_firecracker( // Step 3: Create vsock listeners (must be before boot) println!("Setting up vsock listeners..."); - let vsock_listener = VsockListener::new(jail.vsock().socket())?; + let vsock_listener = VsockListener::new(jail.vsock())?; // Firecracker connects out to these as the unprivileged jail user, so it // needs write access to the inodes. After bind and before InstanceStart. vsock_listener diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index c1c577a5b..0da4df166 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -152,8 +152,12 @@ impl CgroupManager { /// /// # Errors /// - /// Returns Ok even if cpuset is not available (logs a warning). - /// CPU pinning is best-effort for isolation but not required for correctness. + /// Returns an error when the cgroup exists but the cpuset cannot be + /// applied to it. A cgroup with no cpuset does not confine the VMM to the + /// benchmark cores, so the run would report a number measured somewhere + /// other than where it claims. A half-applied fidelity mechanism is a + /// confinement-grade failure; a host that cannot isolate at all is handled + /// earlier, by not creating a cgroup in the first place. pub fn apply_cpuset(&self, layout: &CpuLayout) -> Result<(), RunnerError> { if !layout.has_isolation() { // No meaningful isolation possible (single core or overlapping sets) @@ -165,26 +169,24 @@ impl CgroupManager { return Ok(()); } - // Try to write cpuset.cpus - may fail if cpuset controller is not available let path = self.cgroup_path.join("cpuset.cpus"); - if let Err(e) = fs::write(&path, &cpuset) { - // Log warning but don't fail - cpuset is optional for isolation - eprintln!( - "Warning: failed to set cpuset.cpus to '{cpuset}' (cpuset controller may not be available): {e}" - ); - } else { - // Also need to set cpuset.mems for cpuset to work. Use the - // parent's effective memory nodes so multi-node NUMA hosts - // are not forced onto node 0. - let mems = self - .cgroup_path - .parent() - .map_or_else(|| "0".to_owned(), effective_mems); - let mems_path = self.cgroup_path.join("cpuset.mems"); - if let Err(e) = fs::write(&mems_path, &mems) { - eprintln!("Warning: failed to set cpuset.mems: {e}"); - } - } + fs::write(&path, &cpuset).map_err(|e| JailError::WriteCgroup { + path: path.clone(), + source: e, + })?; + + // Also need to set cpuset.mems for cpuset to work. Use the parent's + // effective memory nodes so multi-node NUMA hosts are not forced onto + // node 0. Applied cpus without mems is the half-applied case. + let mems = self + .cgroup_path + .parent() + .map_or_else(|| "0".to_owned(), effective_mems); + let mems_path = self.cgroup_path.join("cpuset.mems"); + fs::write(&mems_path, &mems).map_err(|e| JailError::WriteCgroup { + path: mems_path, + source: e, + })?; Ok(()) } @@ -352,6 +354,30 @@ pub(crate) fn effective_mems(cgroup: &Utf8Path) -> String { } } +/// Remove the cgroup a swept jail left behind. +/// +/// The cgroup and the chroot are named by the same VM id by construction, so +/// the id read off the chroot directory names the cgroup exactly. +/// +/// Must run after the VMM in it has been reaped: `rmdir` on a cgroup that +/// still holds a process fails, which is what forces that ordering. A cgroup +/// that survives is worth shouting about, because it holds the exclusive +/// benchmark CPUs and the next job's cpuset will be rejected because of it. +pub(crate) fn remove_stale_cgroup(vm_id: &str) { + let path = Utf8PathBuf::from(CGROUP_ROOT) + .join(BENCHER_CGROUP_BASE) + .join(vm_id); + if !path.exists() { + return; + } + match fs::remove_dir(&path) { + Ok(()) => eprintln!("Warning: removed stale cgroup {path} left by a previous runner"), + Err(e) => eprintln!( + "Warning: failed to remove stale cgroup {path}: {e}. It still holds the benchmark CPUs, so the next run's CPU isolation will be rejected." + ), + } +} + /// Whether a `cgroup.procs` listing contains `pid`. /// /// Matches whole lines: pid `7` must not be satisfied by pid `70`. diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index d5478d938..68bdc59ca 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -17,6 +17,8 @@ pub mod netns; #[cfg(target_os = "linux")] pub mod paths; #[cfg(target_os = "linux")] +pub mod reap; +#[cfg(target_os = "linux")] pub mod state; #[cfg(target_os = "linux")] diff --git a/plus/bencher_runner/src/jail/reap.rs b/plus/bencher_runner/src/jail/reap.rs new file mode 100644 index 000000000..41c597f9b --- /dev/null +++ b/plus/bencher_runner/src/jail/reap.rs @@ -0,0 +1,233 @@ +//! Reaping the VMM a runner left behind. +//! +//! The sweep reclaims chroots because `Drop` does not run when a runner is +//! `SIGKILL`ed, crashes, or `exec`s itself during a self-update. The same exits +//! strand the jailed VMM: it is not signalled when its parent dies, so it is +//! reparented and keeps running, holding the benchmark cores through its +//! cgroup. Reclaiming only the disk leaves the more damaging half behind. +//! +//! The consequence is not leakage but wrong numbers that look right. An +//! orphaned cgroup still owns the exclusive benchmark CPUs, so the next job's +//! cpuset write is rejected and, before this, the job reported success anyway. +//! +//! Killing a process the runner does not own is a destructive capability, so +//! the target is identified as narrowly as possible: only a process whose root +//! directory *is* the chroot being swept, compared by device and inode. A +//! process merely owned by the jail uid is not a target, because on a shared +//! host that uid may legitimately own something else. + +#![expect(clippy::print_stderr, reason = "reaping prints diagnostics")] + +use std::fs; +use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd}; +use std::os::unix::fs::MetadataExt as _; +use std::time::{Duration, Instant}; + +use camino::Utf8Path; + +/// How long to wait for a killed VMM to disappear. +const REAP_TIMEOUT: Duration = Duration::from_secs(5); + +/// How often to check whether it has. +const REAP_INTERVAL: Duration = Duration::from_millis(20); + +/// Kill the VMM confined to `jail_root`, if one is still running. +/// +/// Returns the pid that was reaped. Best effort: a VMM that cannot be +/// identified or killed is reported and left alone, because the alternative +/// to leaving an unidentified process alone is killing the wrong one. +pub fn reap_jailed_vmm(jail_root: &Utf8Path) -> Option { + let pid = find_jailed_vmm(jail_root)?; + + // Pin the pid before signalling it. A pid found by scanning `/proc` can + // exit and have its number recycled before the signal lands, and this runs + // as root, so the signal would go to whatever inherited the number. A + // pidfd refers to one process for as long as it is open and keeps the + // number from being reused, which turns the check below into a guarantee + // rather than a narrow window. + let pidfd = pidfd_open(pid)?; + + // Re-check now that the pid cannot change underneath us. + if !is_jailed_vmm(pid, jail_root) { + return None; + } + + if let Err(e) = pidfd_kill(&pidfd) { + eprintln!("Warning: failed to kill orphaned VMM (pid {pid}) in {jail_root}: {e}"); + return None; + } + + if wait_for_exit(pid) { + eprintln!("Warning: reaped orphaned VMM (pid {pid}) left behind in {jail_root}"); + Some(pid) + } else { + eprintln!( + "Warning: orphaned VMM (pid {pid}) in {jail_root} did not exit within {} seconds", + REAP_TIMEOUT.as_secs() + ); + None + } +} + +/// Find the pid of the VMM whose root directory is `jail_root`. +/// +/// The jailer `chroot`s before exec, so the confined process's root *is* the +/// chroot. Comparing device and inode rather than the path is what makes this +/// exact: the jailer pivots into a private mount namespace, so the path reads +/// back as `/`, while the identity is preserved. +fn find_jailed_vmm(jail_root: &Utf8Path) -> Option { + let jail = fs::metadata(jail_root).ok()?; + for entry in fs::read_dir("/proc").ok()?.flatten() { + let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { + continue; + }; + if matches_jail(pid, &jail) { + return Some(pid); + } + } + None +} + +/// Whether a process's root directory is `jail_root`. +fn is_jailed_vmm(pid: u32, jail_root: &Utf8Path) -> bool { + fs::metadata(jail_root).is_ok_and(|jail| matches_jail(pid, &jail)) +} + +/// Whether a process's root directory is the same inode as `jail`. +fn matches_jail(pid: u32, jail: &fs::Metadata) -> bool { + // Following this magic symlink crosses into the process's own mount + // namespace, which a privileged reader is allowed to do. + fs::metadata(format!("/proc/{pid}/root")) + .is_ok_and(|root| root.dev() == jail.dev() && root.ino() == jail.ino()) +} + +/// Open a descriptor pinned to a process. +/// +/// `None` when the process is already gone, which is the common case and not +/// an error: something else reaped it first. +fn pidfd_open(pid: u32) -> Option { + #[expect( + unsafe_code, + reason = "pidfd_open has no std wrapper; it takes plain integers" + )] + // SAFETY: `pidfd_open` takes a pid and a flag word and touches no memory. + // It returns a new descriptor or -1, and the descriptor is handed straight + // to `OwnedFd` so it is closed exactly once. + let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, libc::pid_t::try_from(pid).ok()?, 0) }; + let raw = libc::c_int::try_from(raw).ok()?; + if raw < 0 { + return None; + } + #[expect( + unsafe_code, + reason = "taking ownership of a descriptor this call just created" + )] + // SAFETY: `raw` is a fresh descriptor returned by the syscall above and is + // not owned by anything else. + let fd = unsafe { OwnedFd::from_raw_fd(raw) }; + Some(fd) +} + +/// SIGKILL the process a descriptor is pinned to. +/// +/// SIGKILL rather than a graceful shutdown: the job that owned this VMM is +/// already gone, so there is nothing left to shut down cleanly for. +fn pidfd_kill(pidfd: &OwnedFd) -> std::io::Result<()> { + #[expect( + unsafe_code, + reason = "pidfd_send_signal has no std wrapper; the fd is owned and valid" + )] + // SAFETY: `pidfd` is an open, owned descriptor for the duration of the + // call. A null `siginfo` pointer is the documented way to ask the kernel + // to synthesize one, and the final argument is a reserved flag word. + let ret = unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + pidfd.as_raw_fd(), + libc::SIGKILL, + std::ptr::null::(), + 0, + ) + }; + if ret == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + +/// Wait for a killed process to disappear. +fn wait_for_exit(pid: u32) -> bool { + let deadline = Instant::now() + REAP_TIMEOUT; + while Instant::now() < deadline { + if !Utf8Path::new(&format!("/proc/{pid}")).exists() { + return true; + } + std::thread::sleep(REAP_INTERVAL); + } + false +} + +#[cfg(test)] +mod tests { + use camino::Utf8PathBuf; + + use super::*; + + #[test] + fn no_process_is_rooted_at_an_ordinary_directory() { + // The identification has to be narrow enough that a directory no + // process is chrooted into matches nothing at all. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + assert_eq!(find_jailed_vmm(&root), None); + } + + #[test] + fn a_missing_jail_matches_nothing() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + assert_eq!(find_jailed_vmm(&root.join("absent")), None); + assert!(!is_jailed_vmm(std::process::id(), &root.join("absent"))); + } + + #[test] + fn the_runners_own_root_is_matched_by_identity_not_by_path() { + // The runner is rooted at `/`, so it is found when `/` is the jail and + // not otherwise. This is the mechanism the reap depends on: a jailed + // VMM's root path reads back as `/` too, so only the inode separates + // them. + // + // It also pins the subtler half. `/proc//root` is a magic + // symlink, and the comparison has to follow it: `metadata` does, + // `symlink_metadata` would return the procfs link's own inode and + // match nothing ever. This assertion fails if that changes. + let root = Utf8Path::new("/"); + + assert!(is_jailed_vmm(std::process::id(), root)); + + let dir = tempfile::tempdir().unwrap(); + let other = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + assert!(!is_jailed_vmm(std::process::id(), &other)); + } + + #[test] + fn a_pidfd_pins_a_live_process_and_refuses_a_dead_one() { + pidfd_open(std::process::id()).expect("this process is alive"); + + // Pid 0 is never a process: the syscall rejects it rather than + // signalling the caller's process group, which is what the plain + // `kill` interface would have done. + assert!(pidfd_open(0).is_none()); + } + + #[test] + fn reaping_an_unjailed_directory_kills_nothing() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + assert_eq!(reap_jailed_vmm(&root), None); + } +} diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index a76855c2e..67d946cb1 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -16,6 +16,9 @@ use crate::error::JailError; /// Subdirectory of the state directory used as the jailer's chroot base. const CHROOT_BASE: &str = "jail"; +/// The chroot directory inside a jail, which the jailer makes `/`. +const JAIL_ROOT: &str = "root"; + /// The jail lock file, which lives beside the chroot base. /// /// Named here as well as in the lock module so the state directory knows @@ -102,7 +105,7 @@ impl StateDir { /// The chroot root for a VM, which becomes `/` inside the jail. #[must_use] pub fn jail_root(&self, vm_id: &str) -> Utf8PathBuf { - self.jail_dir(vm_id).join("root") + self.jail_dir(vm_id).join(JAIL_ROOT) } /// Create the state directory tree at mode 0700. @@ -154,17 +157,26 @@ pub fn sweep_jails(jail_parent: &Utf8Path) -> usize { let mut swept = 0; for entry in entries.flatten() { - let path = entry.path(); if !entry.file_type().is_ok_and(|file_type| file_type.is_dir()) { continue; } - match fs::remove_dir_all(&path) { + let vm_id = entry.file_name().to_string_lossy().into_owned(); + let jail_dir = jail_parent.join(&vm_id); + + // Reap before removing. Pulling the rootfs out from under a process + // that is still running leaves it running anyway, so the process goes + // first and the directory second. + super::reap::reap_jailed_vmm(&jail_dir.join(JAIL_ROOT)); + + match fs::remove_dir_all(&jail_dir) { Ok(()) => swept += 1, - Err(e) => eprintln!( - "Warning: failed to sweep stale jail {}: {e}", - path.display() - ), + Err(e) => eprintln!("Warning: failed to sweep stale jail {jail_dir}: {e}"), } + + // The cgroup is the half that actually corrupts later runs: it holds + // the exclusive benchmark CPUs, so leaving it makes the next job's + // cpuset write fail. It shares the chroot's name by construction. + super::cgroup::remove_stale_cgroup(&vm_id); } swept } From a1808ae0f9e4b676ee554b6effd020b0ac964daf Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 19:01:07 +0000 Subject: [PATCH 18/91] Unlink jail sockets through the host view Both cleanups named the socket view, which is a descriptor number. Unlinking has no sun_path limit, so that view bought nothing and cost a dependency on a descriptor still being open. Both run from Drop, where a future reordering could close it first, and where the failure would not be an error: the number is reused immediately, so the identical string resolves to a different directory and the unlink deletes whatever file inherited it. Reserving the socket view for bind and connect makes that unrepresentable, and a test pins it rather than leaving it a convention: drop the paths, claim the released number with another directory, and assert the same string no longer names the jail. --- .../bencher_runner/src/firecracker/process.rs | 17 ++++-- plus/bencher_runner/src/firecracker/vsock.rs | 54 +++++++++++-------- plus/bencher_runner/src/jail/paths.rs | 38 +++++++++++++ 3 files changed, 82 insertions(+), 27 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/process.rs b/plus/bencher_runner/src/firecracker/process.rs index d94cc9364..f0da6aa6a 100644 --- a/plus/bencher_runner/src/firecracker/process.rs +++ b/plus/bencher_runner/src/firecracker/process.rs @@ -11,7 +11,7 @@ use camino::Utf8Path; use crate::firecracker::client::FirecrackerClient; use crate::firecracker::config::{Action, ActionType}; use crate::firecracker::error::FirecrackerError; -use crate::jail::{JailFile, JailUser, SocketPath}; +use crate::jail::{JailFile, JailUser}; /// Everything needed to spawn the VMM under the jailer. #[derive(Debug)] @@ -46,7 +46,7 @@ pub struct JailedSpawn<'a> { /// A running, jailed Firecracker process. pub struct FirecrackerProcess { child: Child, - api_socket_path: SocketPath, + api_socket: JailFile, stderr_thread: Option>, } @@ -141,7 +141,7 @@ impl FirecrackerProcess { let process = Self { child, - api_socket_path: api_socket.socket().clone(), + api_socket: api_socket.clone(), stderr_thread: Some(stderr_thread), }; @@ -153,7 +153,7 @@ impl FirecrackerProcess { /// Get a client for the Firecracker REST API. pub fn client(&self) -> FirecrackerClient { - FirecrackerClient::new(&self.api_socket_path) + FirecrackerClient::new(self.api_socket.socket()) } /// Get the PID of the Firecracker process. @@ -195,8 +195,15 @@ impl FirecrackerProcess { /// /// The chroot itself is reclaimed wholesale by the jail teardown; this /// only keeps the socket from outliving the process within a job. + /// + /// Unlinks through the host view. Unlinking has no `sun_path` limit, so + /// the socket view buys nothing here and costs a dependency on a + /// descriptor still being open. This runs from `Drop`, where a future + /// reordering could close that descriptor first, and where the failure + /// would not be an error but the deletion of whatever file inherited the + /// number. pub fn cleanup(&self) { - drop(std::fs::remove_file(self.api_socket_path.as_str())); + drop(std::fs::remove_file(self.api_socket.host().as_path())); } /// Join the stderr reader thread if it exists. diff --git a/plus/bencher_runner/src/firecracker/vsock.rs b/plus/bencher_runner/src/firecracker/vsock.rs index 591fbcb6f..e8c0b976c 100644 --- a/plus/bencher_runner/src/firecracker/vsock.rs +++ b/plus/bencher_runner/src/firecracker/vsock.rs @@ -21,7 +21,7 @@ use nix::poll::{PollFd, PollFlags, PollTimeout, poll}; use crate::firecracker::error::FirecrackerError; use crate::jail::chroot::chown_to_jail; -use crate::jail::{JailUser, SocketPath}; +use crate::jail::{JailFile, JailUser}; /// Poll timeout for vsock listeners (50ms). /// @@ -59,8 +59,12 @@ pub struct VsockResults { /// Host-side vsock listener that accepts connections from Firecracker. pub struct VsockListener { - /// Base path for the vsock UDS. - vsock_uds_path: String, + /// Both views of the vsock base path. + /// + /// Binding needs the socket view, because of the `sun_path` limit. + /// Everything else uses the host view, which carries no dependency on a + /// descriptor staying open. + vsock: JailFile, /// Listeners for each port. stdout_listener: UnixListener, stderr_listener: UnixListener, @@ -74,12 +78,12 @@ impl VsockListener { /// Creates Unix listeners at `{vsock_uds_path}_{port}` for each port, /// using the host view of the path: the runner binds them from outside /// the chroot. These must be created before the VM boots. - pub fn new(vsock_uds_path: &SocketPath) -> Result { - let stdout_path = vsock_uds_path.with_suffix(&format!("_{}", ports::STDOUT)); - let stderr_path = vsock_uds_path.with_suffix(&format!("_{}", ports::STDERR)); - let exit_code_path = vsock_uds_path.with_suffix(&format!("_{}", ports::EXIT_CODE)); - let output_files_path = vsock_uds_path.with_suffix(&format!("_{}", ports::OUTPUT_FILES)); - let vsock_uds_path = vsock_uds_path.as_str(); + pub fn new(vsock: &JailFile) -> Result { + let socket = vsock.socket(); + let stdout_path = socket.with_suffix(&format!("_{}", ports::STDOUT)); + let stderr_path = socket.with_suffix(&format!("_{}", ports::STDERR)); + let exit_code_path = socket.with_suffix(&format!("_{}", ports::EXIT_CODE)); + let output_files_path = socket.with_suffix(&format!("_{}", ports::OUTPUT_FILES)); // Remove stale socket files for path in [ @@ -115,7 +119,7 @@ impl VsockListener { })?; Ok(Self { - vsock_uds_path: vsock_uds_path.to_owned(), + vsock: vsock.clone(), stdout_listener, stderr_listener, exit_code_listener, @@ -284,21 +288,27 @@ impl VsockListener { /// and before `InstanceStart`. pub fn chown_to_jail(&self, jail_user: JailUser) -> Result<(), crate::error::JailError> { for port in ports::ALL { - chown_to_jail(Utf8Path::new(&self.socket_path(port)), jail_user)?; + chown_to_jail(Utf8Path::new(&self.host_path(port)), jail_user)?; } Ok(()) } /// Remove all socket files created by this listener. + /// + /// Unlinks through the host view, for the same reason as + /// [`crate::firecracker::process::FirecrackerProcess::cleanup`]: unlinking + /// has no `sun_path` limit, and this runs from `Drop`, where naming a + /// descriptor that may already be closed would delete an unrelated file + /// rather than fail. pub fn cleanup(&self) { for port in ports::ALL { - drop(std::fs::remove_file(self.socket_path(port))); + drop(std::fs::remove_file(self.host_path(port))); } } /// The host path of the listener socket for a port. - fn socket_path(&self, port: u32) -> String { - format!("{}_{port}", self.vsock_uds_path) + fn host_path(&self, port: u32) -> String { + format!("{}_{port}", self.vsock.host()) } } @@ -375,7 +385,7 @@ mod tests { /// Helper: create a `VsockListener` on a jail that stays alive. fn listener_in_tmpdir() -> (tempfile::TempDir, JailPaths, VsockListener) { let (dir, jail) = jail_in_tmpdir(); - let listener = VsockListener::new(jail.vsock().socket()).unwrap(); + let listener = VsockListener::new(jail.vsock()).unwrap(); (dir, jail, listener) } @@ -390,7 +400,7 @@ mod tests { #[test] fn vsock_listener_creates_socket_files() { let (_dir, jail, _listener) = listener_in_tmpdir(); - let base = jail.vsock().socket().as_str().to_owned(); + let base = jail.vsock().host().to_string(); for port in [5000, 5001, 5002, 5005] { let path = format!("{base}_{port}"); @@ -404,10 +414,10 @@ mod tests { #[test] fn vsock_listener_cleanup_removes_files() { let (_dir, jail) = jail_in_tmpdir(); - let base = jail.vsock().socket().as_str().to_owned(); + let base = jail.vsock().host().to_string(); { - let _listener = VsockListener::new(jail.vsock().socket()).unwrap(); + let _listener = VsockListener::new(jail.vsock()).unwrap(); // listener drops here } @@ -423,7 +433,7 @@ mod tests { #[test] fn collect_all_ports() { let (_dir, jail, listener) = listener_in_tmpdir(); - let base = jail.vsock().socket().as_str().to_owned(); + let base = jail.vsock().host().to_string(); // Build protocol-encoded data: 1 file, path="out.bin", content=\x00\x01\x02 let mut encoded = Vec::new(); @@ -465,7 +475,7 @@ mod tests { #[test] fn collect_exit_code_only() { let (_dir, jail, listener) = listener_in_tmpdir(); - let base = jail.vsock().socket().as_str().to_owned(); + let base = jail.vsock().host().to_string(); let base_clone = base.clone(); let sender = std::thread::spawn(move || { @@ -511,7 +521,7 @@ mod tests { #[test] fn collect_non_utf8_stdout() { let (_dir, jail, listener) = listener_in_tmpdir(); - let base = jail.vsock().socket().as_str().to_owned(); + let base = jail.vsock().host().to_string(); let base_clone = base.clone(); let sender = std::thread::spawn(move || { @@ -540,7 +550,7 @@ mod tests { #[test] fn collect_exit_code_triggers_final_pass() { let (_dir, jail, listener) = listener_in_tmpdir(); - let base = jail.vsock().socket().as_str().to_owned(); + let base = jail.vsock().host().to_string(); let base_clone = base.clone(); let sender = std::thread::spawn(move || { diff --git a/plus/bencher_runner/src/jail/paths.rs b/plus/bencher_runner/src/jail/paths.rs index 0a3e450e4..646c9af8d 100644 --- a/plus/bencher_runner/src/jail/paths.rs +++ b/plus/bencher_runner/src/jail/paths.rs @@ -308,6 +308,44 @@ mod tests { assert_eq!(through_socket_view, b"guest"); } + #[test] + fn the_socket_view_stops_naming_the_jail_once_the_paths_are_dropped() { + // This is why `bind` and `connect` are the only callers of the socket + // view: the descriptor number is reused the moment it is released, and + // the identical string then resolves to a different directory with no + // error at all. Anything that only needs a path, unlinking above all, + // uses the host view, which cannot go stale. + let jail = tempfile::tempdir().unwrap(); + let jail_root = Utf8Path::from_path(jail.path()).unwrap(); + std::fs::write(jail_root.join("rootfs.ext4"), b"the jail").unwrap(); + + let socket_view = { + let paths = JailPaths::new(jail_root).unwrap(); + let view = paths.rootfs().socket().as_str().to_owned(); + assert_eq!(std::fs::read(&view).unwrap(), b"the jail"); + view + }; + + // Claim the number the jail's descriptor just released. + let impostor = tempfile::tempdir().unwrap(); + let impostor_root = Utf8Path::from_path(impostor.path()).unwrap(); + std::fs::write(impostor_root.join("rootfs.ext4"), b"somewhere else").unwrap(); + let _claim = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_PATH | libc::O_DIRECTORY) + .open(impostor_root) + .unwrap(); + + // The same string no longer names the jail. It either names the + // impostor or fails; what it must never do is still work. + let stale = std::fs::read(&socket_view); + assert_ne!( + stale.unwrap_or_default(), + b"the jail", + "a dropped descriptor must not leave the socket view pointing at the jail" + ); + } + #[test] fn every_socket_view_fits_the_sun_path_limit() { let (_dir, paths) = jail_in_tmpdir(); From 3b65040a2763c8e63f9a875b1da5e76b69cdb615 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 19:13:16 +0000 Subject: [PATCH 19/91] Elevate the smoke test's sandboxed runner A sandboxed Job is a jailed Job, and the jailer needs root, so the smoke test's Firecracker runner could no longer come up as the unprivileged CI user. Only that one process is elevated. Cargo and everything else stay as the invoking user, and the no-sandbox runner in the same test stays unprivileged, which proves the coupling holds in both directions in a single run. The already-built binary is run under sudo directly rather than through cargo, so nothing root-owned lands in the target directory. Teardown signals the process group rather than the handle. Verified on Linux with sudo 1.9.15p5 that sudo forks rather than execing in place: the handle is sudo (pid N) and the runner is a separate process in the same group. Killing only the handle left a root runner daemon running, which would have held the jail lock for the rest of the test; killing the group leaves nothing. The kill is itself elevated, because the unprivileged test process cannot signal a root daemon. Missing passwordless sudo now fails immediately and says why, instead of surfacing thirty seconds later as a readiness timeout with nothing pointing at the cause. --- tasks/test_api/src/task/plus/runner.rs | 80 +++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/tasks/test_api/src/task/plus/runner.rs b/tasks/test_api/src/task/plus/runner.rs index 3961f4907..0d9fd112a 100644 --- a/tasks/test_api/src/task/plus/runner.rs +++ b/tasks/test_api/src/task/plus/runner.rs @@ -3,6 +3,8 @@ use std::process::Command; use assert_cmd::cargo::CommandCargoExt as _; use bencher_json::{JsonProjectKeyCreated, JsonUserKeyCreated, Jwt, Url}; use pretty_assertions::assert_eq; +#[cfg(unix)] +use std::os::unix::process::CommandExt as _; use crate::parser::TaskRunner; use crate::task::test::seed_test::{ @@ -174,11 +176,29 @@ impl RunnerTest { anyhow::ensure!(build_status.success(), "Failed to build bencher CLI"); } - // Start the Firecracker runner daemon only when KVM is available + // Start the Firecracker runner daemon only when KVM is available. + // + // Elevated, because a sandboxed Job is a jailed Job: the jailer creates + // the chroot's device nodes with mknod, chowns the tree to the jail + // user, pivot_roots, and joins a network namespace. Only this one + // process is elevated. Everything else in this test, cargo included, + // stays as the invoking user, and the no-sandbox runner below stays + // unprivileged, which proves the coupling holds in both directions in + // the same run. let runner_child_and_handle = if has_kvm { - println!("Starting runner daemon..."); - let mut runner_child = Command::cargo_bin("runner")?; - let mut runner_child = runner_child + println!("Starting runner daemon (elevated)..."); + ensure_passwordless_sudo()?; + + // Resolve the already-built binary and run it under sudo directly, + // so cargo is never invoked as root and cannot leave root-owned + // artifacts in the target directory. + let runner_cmd = Command::cargo_bin("runner")?; + let runner_bin = runner_cmd.get_program().to_owned(); + + let mut runner_child = Command::new("sudo"); + let runner_child = runner_child + .args(["-n", "--"]) + .arg(&runner_bin) .args([ "up", HOST_ARG, @@ -189,8 +209,13 @@ impl RunnerTest { "test-runner", ]) .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::inherit()) - .spawn()?; + .stderr(std::process::Stdio::inherit()); + // Its own process group, so teardown can signal the runner even + // though the child handle is sudo, which may exec the runner in + // place or fork it depending on version and configuration. + #[cfg(unix)] + runner_child.process_group(0); + let mut runner_child = runner_child.spawn()?; let reader_handle = wait_for_stdout_ready( &mut runner_child, @@ -288,8 +313,7 @@ impl RunnerTest { // Always kill runner daemons, even if the test failed if let Some((mut runner_child, reader_handle)) = runner_child_and_handle { - let _kill = runner_child.kill(); - let _wait = runner_child.wait(); + kill_elevated_runner(&mut runner_child); let _join = reader_handle.join(); } let _kill = no_sandbox_child.kill(); @@ -309,6 +333,46 @@ impl RunnerTest { } } +/// Fail early when the sandboxed runner cannot be elevated. +/// +/// Without this the runner would start, fail to build its first jail, and the +/// readiness wait would time out after thirty seconds with nothing pointing at +/// the cause. +fn ensure_passwordless_sudo() -> anyhow::Result<()> { + let status = Command::new("sudo") + .args(["-n", "true"]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status()?; + anyhow::ensure!( + status.success(), + "A Runner executing sandboxed Jobs must run as root, and passwordless sudo is not available. \ + The jailer creates the chroot's device nodes with mknod, chowns the tree to the jail user, \ + pivot_roots into it, and joins a network namespace, none of which an unprivileged process can do." + ); + Ok(()) +} + +/// Stop the elevated runner daemon and anything it spawned. +/// +/// The runner runs as root, so the unprivileged test process cannot signal it, +/// and the handle is sudo rather than the runner itself. Signalling the whole +/// process group covers both cases; the group exists because the spawn put the +/// child in its own. Killing sudo alone would leave a root runner daemon +/// holding the jail lock for the rest of the run. +fn kill_elevated_runner(child: &mut std::process::Child) { + let pid = child.id(); + let _status = Command::new("sudo") + .args(["-n", "sh", "-c"]) + .arg(format!( + "kill -KILL -{pid} 2>/dev/null || kill -KILL {pid} 2>/dev/null || true" + )) + .status(); + // Reap the handle whatever the signal did. + let _kill = child.kill(); + let _wait = child.wait(); +} + /// Check whether Docker is available. pub fn docker_available() -> bool { Command::new("docker") From ae7d8c31b604b099f9efe6323d64c86bad9b6bb0 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 19:13:29 +0000 Subject: [PATCH 20/91] Document that sandboxed Jobs now require a root Runner A Runner executing sandboxed Jobs must run as root where before it did not, so it is called out where an operator actually looks. The self-hosted Runner intro said Firecracker sandboxing requires Linux with KVM enabled, which is now only half the requirement, and the start-the-Runner page showed bare runner up commands that read as unprivileged. Both are corrected in all nine locales. The changelog entry names the capabilities rather than asserting the requirement: mknod for the chroot's device nodes, chown to hand the guest images to the jail user, pivot_root, and setns to join the network namespace. A world-readable /dev/kvm is enough to use KVM unprivileged but not to build the jail around it, which is exactly the assumption an operator will have. It points anyone who cannot run as root at --danger-allow-no-sandbox while being explicit that this trades away the microVM itself and not just the jail. --- .../chunks/docs-explanation/self-hosted-runners/de/intro.mdx | 5 ++++- .../chunks/docs-explanation/self-hosted-runners/de/run.mdx | 4 ++++ .../chunks/docs-explanation/self-hosted-runners/en/intro.mdx | 5 ++++- .../chunks/docs-explanation/self-hosted-runners/en/run.mdx | 4 ++++ .../chunks/docs-explanation/self-hosted-runners/es/intro.mdx | 5 ++++- .../chunks/docs-explanation/self-hosted-runners/es/run.mdx | 4 ++++ .../chunks/docs-explanation/self-hosted-runners/fr/intro.mdx | 5 ++++- .../chunks/docs-explanation/self-hosted-runners/fr/run.mdx | 4 ++++ .../chunks/docs-explanation/self-hosted-runners/ja/intro.mdx | 5 ++++- .../chunks/docs-explanation/self-hosted-runners/ja/run.mdx | 4 ++++ .../chunks/docs-explanation/self-hosted-runners/ko/intro.mdx | 5 ++++- .../chunks/docs-explanation/self-hosted-runners/ko/run.mdx | 4 ++++ .../chunks/docs-explanation/self-hosted-runners/pt/intro.mdx | 5 ++++- .../chunks/docs-explanation/self-hosted-runners/pt/run.mdx | 4 ++++ .../chunks/docs-explanation/self-hosted-runners/ru/intro.mdx | 5 ++++- .../chunks/docs-explanation/self-hosted-runners/ru/run.mdx | 4 ++++ .../chunks/docs-explanation/self-hosted-runners/zh/intro.mdx | 5 ++++- .../chunks/docs-explanation/self-hosted-runners/zh/run.mdx | 4 ++++ .../src/chunks/docs-reference/changelog/en/changelog.mdx | 1 + 19 files changed, 73 insertions(+), 9 deletions(-) diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/de/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/de/intro.mdx index 68736ef2d..2d212555d 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/de/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/de/intro.mdx @@ -14,7 +14,10 @@ das Verknüpfen beider und das Starten des `runner`-Binary. Einen konzeptionellen Überblick darüber, wie Runner, Specs, Sandboxes und Jobs zusammenpassen, finden Sie in der [Bare Metal-Übersicht][bare metal]. -> 🐰 Firecracker-Sandboxing erfordert Linux mit aktiviertem [KVM][kvm]. +> 🐰 Firecracker-Sandboxing erfordert Linux mit aktiviertem [KVM][kvm], +> und der Runner muss als `root` laufen. +> Jeder Sandbox-Job wird in einem Chroot als unprivilegierter Benutzer eingesperrt, +> und der Aufbau dieser Isolierung benötigt Rechte, die das Aktivieren von KVM allein nicht gewährt. > Ein Runner, der nur Specs ohne Sandbox bedient, kann auf jedem unterstützten Host laufen. [bare metal]: /de/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/de/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/de/run.mdx index fcdd95ad5..a5147f4c4 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/de/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/de/run.mdx @@ -29,6 +29,10 @@ Standardmäßig folgt der Runner dem `stable` [Update-Kanal][update channel] und aktualisiert nur auf versionierte Releases. Der `canary`-Kanal folgt stattdessen dem rollierenden Canary-Build von Bencher Cloud. +Ein Runner, der Sandbox-Jobs ausführt, muss als `root` laufen, +die obigen Befehle werden also so gezeigt, wie der Benutzer `root` sie ausführen würde, +oder unter einem Dienstmanager, der den Runner als `root` startet. + Standardmäßig lehnt ein Runner jeden Job ab, dessen Spec keine [Sandbox][sandbox] hat. Um einem Runner zu erlauben, Jobs ohne Sandbox direkt auf dem Host auszuführen, starten Sie ihn mit [dem `--danger-allow-no-sandbox`-Flag][danger allow no sandbox]. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/en/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/en/intro.mdx index 93fc96b08..cab2f6556 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/en/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/en/intro.mdx @@ -14,7 +14,10 @@ linking the two together, and starting the `runner` binary. For a conceptual overview of how Runners, Specs, Sandboxes, and Jobs fit together, see the [Bare Metal Overview][bare metal]. -> 🐰 Firecracker sandboxing requires Linux with [KVM][kvm] enabled. +> 🐰 Firecracker sandboxing requires Linux with [KVM][kvm] enabled, +> and the Runner must run as `root`. +> Each sandboxed Job is confined to a chroot as an unprivileged user, +> and building that confinement needs privileges that enabling KVM alone does not grant. > A Runner that only serves non-sandboxed Specs can run on any supported host. [bare metal]: /docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/en/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/en/run.mdx index 862400697..e3a87163f 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/en/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/en/run.mdx @@ -29,6 +29,10 @@ By default the Runner follows the `stable` [update channel][update channel], updating only to versioned releases. The `canary` channel instead tracks the rolling canary build deployed to Bencher Cloud. +A Runner that executes sandboxed Jobs must run as `root`, +so the commands above are shown as they would be run by the `root` user +or under a service manager that starts the Runner as `root`. + By default a Runner rejects any Job whose Spec has no [Sandbox][sandbox]. To let a Runner execute non-sandboxed Jobs directly on the host, start it with [the `--danger-allow-no-sandbox` flag][danger allow no sandbox]. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/es/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/es/intro.mdx index 68faedfc5..184a78e3c 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/es/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/es/intro.mdx @@ -14,7 +14,10 @@ la vinculación de ambos, y el inicio del binario `runner`. Para una visión general conceptual de cómo encajan Runners, Specs, Sandboxes y Jobs, consulta la [Visión General de Bare Metal][bare metal]. -> 🐰 El sandboxing con Firecracker requiere Linux con [KVM][kvm] habilitado. +> 🐰 El sandboxing con Firecracker requiere Linux con [KVM][kvm] habilitado, +> y el Runner debe ejecutarse como `root`. +> Cada Job con sandbox queda confinado en un chroot como usuario sin privilegios, +> y construir ese confinamiento necesita permisos que habilitar KVM por sí solo no concede. > Un Runner que solo sirve Specs sin sandbox puede ejecutarse en cualquier host compatible. [bare metal]: /es/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/es/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/es/run.mdx index 02ac1e068..dede154de 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/es/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/es/run.mdx @@ -29,6 +29,10 @@ Por defecto, el Runner sigue el [canal de actualización][update channel] `stabl y solo se actualiza a releases versionadas. El canal `canary` sigue en cambio la compilación canary continua desplegada en Bencher Cloud. +Un Runner que ejecuta Jobs con sandbox debe correr como `root`, +así que los comandos anteriores se muestran tal como los ejecutaría el usuario `root`, +o bajo un gestor de servicios que inicie el Runner como `root`. + Por defecto, un Runner rechaza cualquier Job cuyo Spec no tenga [Sandbox][sandbox]. Para permitir que un Runner ejecute Jobs sin sandbox directamente en el host, inícialo con [el flag `--danger-allow-no-sandbox`][danger allow no sandbox]. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/intro.mdx index 6f8812d28..5b61e4edf 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/intro.mdx @@ -14,7 +14,10 @@ la liaison des deux ensemble, et le démarrage du binaire `runner`. Pour un aperçu conceptuel de la façon dont les Runners, Specs, Sandboxes et Jobs s'articulent, consultez l'[Aperçu Bare Metal][bare metal]. -> 🐰 Le sandboxing Firecracker nécessite Linux avec [KVM][kvm] activé. +> 🐰 Le sandboxing Firecracker nécessite Linux avec [KVM][kvm] activé, +> et le Runner doit s'exécuter en tant que `root`. +> Chaque Job en bac à sable est confiné dans un chroot en tant qu'utilisateur non privilégié, +> et construire ce confinement demande des droits que l'activation de KVM seule n'accorde pas. > Un Runner qui ne sert que des Specs sans sandbox peut s'exécuter sur n'importe quel hôte pris en charge. [bare metal]: /fr/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/run.mdx index a73295cba..66cd3c677 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/run.mdx @@ -29,6 +29,10 @@ Par défaut, le Runner suit le [canal de mise à jour][update channel] `stable` et ne se met à jour que vers les versions publiées. Le canal `canary` suit quant à lui le build canary continu déployé sur Bencher Cloud. +Un Runner qui exécute des Jobs en bac à sable doit s'exécuter en tant que `root`, +les commandes ci-dessus sont donc présentées telles que l'utilisateur `root` les lancerait, +ou sous un gestionnaire de services qui démarre le Runner en tant que `root`. + Par défaut, un Runner rejette tout Job dont le Spec n'a pas de [Sandbox][sandbox]. Pour permettre à un Runner d'exécuter directement des Jobs sans sandbox sur l'hôte, démarrez-le avec [le drapeau `--danger-allow-no-sandbox`][danger allow no sandbox]. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/intro.mdx index 282b8e682..4b47a69c0 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/intro.mdx @@ -14,7 +14,10 @@ Runner のセルフホストは、次のような場合に適しています。 Runner、Spec、Sandbox、Job がどのように組み合わさるかの概念的な概要については、 [Bare Metal の概要][bare metal] を参照してください。 -> 🐰 Firecracker のサンドボックス化には、[KVM][kvm] が有効になった Linux が必要です。 +> 🐰 Firecracker のサンドボックス化には、[KVM][kvm] が有効になった Linux が必要で、 +> Runner は `root` として実行する必要があります。 +> サンドボックス化された各 Job は非特権ユーザーとして chroot に閉じ込められ、 +> その閉じ込めの構築には、KVM を有効にするだけでは得られない権限が必要です。 > サンドボックスなしの Spec のみを提供する Runner は、サポートされている任意のホスト上で実行できます。 [bare metal]: /ja/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/run.mdx index 9b949a4e1..cab805713 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/run.mdx @@ -29,6 +29,10 @@ runner up バージョン付きリリースへのみ更新します。 `canary` チャネルは、Bencher Cloud にデプロイされるローリング canary ビルドを追跡します。 +サンドボックス化された Job を実行する Runner は `root` として動かす必要があるため、 +上記のコマンドは `root` ユーザーが実行する場合、 +あるいは Runner を `root` で起動するサービスマネージャー配下での実行を想定しています。 + デフォルトでは、Runner は [Sandbox][sandbox] のない Spec の Job をすべて拒否します。 Runner にサンドボックスなしの Job をホスト上で直接実行させるには、 [`--danger-allow-no-sandbox` フラグ][danger allow no sandbox] を付けて起動します。 diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/intro.mdx index 61fb36c1d..90875fd97 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/intro.mdx @@ -14,7 +14,10 @@ Runner를 셀프 호스팅하는 것은 다음과 같은 경우에 적합합니 Runner, Spec, Sandbox, Job이 어떻게 맞물리는지에 대한 개념적 개요는 [Bare Metal 개요][bare metal]를 참고하세요. -> 🐰 Firecracker 샌드박싱에는 [KVM][kvm]이 활성화된 Linux가 필요합니다. +> 🐰 Firecracker 샌드박싱에는 [KVM][kvm]이 활성화된 Linux가 필요하며, +> Runner는 `root`로 실행해야 합니다. +> 샌드박스 Job은 각각 비특권 사용자로서 chroot에 가두어지며, +> 그 격리를 구성하려면 KVM 활성화만으로는 얻을 수 없는 권한이 필요합니다. > 샌드박스가 없는 Spec만 제공하는 Runner는 지원되는 모든 호스트에서 실행할 수 있습니다. [bare metal]: /ko/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/run.mdx index a43310e76..1ad2e5059 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/run.mdx @@ -29,6 +29,10 @@ runner up 버전이 지정된 릴리스로만 업데이트합니다. `canary` 채널은 Bencher Cloud에 배포되는 롤링 canary 빌드를 추적합니다. +샌드박스 Job을 실행하는 Runner는 `root`로 실행해야 하므로, +위 명령은 `root` 사용자가 실행하거나 +Runner를 `root`로 시작하는 서비스 관리자 아래에서 실행하는 것을 전제로 합니다. + 기본적으로 Runner는 [Sandbox][sandbox]가 없는 Spec의 Job을 거부합니다. Runner가 샌드박스 없는 Job을 호스트에서 직접 실행하도록 하려면, [`--danger-allow-no-sandbox` 플래그][danger allow no sandbox]로 시작하세요. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/intro.mdx index 72f2be6c8..e15970b74 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/intro.mdx @@ -14,7 +14,10 @@ vincular os dois e iniciar o binário `runner`. Para uma visão conceitual de como Runners, Specs, Sandboxes e Jobs se encaixam, consulte a [Visão Geral de Bare Metal][bare metal]. -> 🐰 O sandboxing com Firecracker requer Linux com [KVM][kvm] habilitado. +> 🐰 O sandboxing com Firecracker requer Linux com [KVM][kvm] habilitado, +> e o Runner precisa ser executado como `root`. +> Cada Job com sandbox fica confinado em um chroot como usuário sem privilégios, +> e construir esse confinamento exige permissões que habilitar o KVM sozinho não concede. > Um Runner que atende apenas Specs sem sandbox pode rodar em qualquer host suportado. [bare metal]: /pt/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/run.mdx index 8c32f8924..b977148a6 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/run.mdx @@ -29,6 +29,10 @@ Por padrão, o Runner segue o [canal de atualização][update channel] `stable` e só atualiza para releases versionadas. O canal `canary`, por sua vez, acompanha o build canary contínuo implantado no Bencher Cloud. +Um Runner que executa Jobs com sandbox precisa rodar como `root`, +então os comandos acima são mostrados como o usuário `root` os executaria, +ou sob um gerenciador de serviços que inicie o Runner como `root`. + Por padrão, um Runner rejeita qualquer Job cuja Spec não tenha [Sandbox][sandbox]. Para permitir que um Runner execute Jobs sem sandbox diretamente no host, inicie-o com [a flag `--danger-allow-no-sandbox`][danger allow no sandbox]. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/intro.mdx index c52993fcc..c8ea1a490 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/intro.mdx @@ -14,7 +14,10 @@ Self-Hosted Runner — это [Bare Metal Runner][bare metal], который в Концептуальный обзор того, как Runner, Spec, Sandbox и Job сочетаются друг с другом, смотрите в [обзоре Bare Metal][bare metal]. -> 🐰 Изоляция через Firecracker требует Linux с включённым [KVM][kvm]. +> 🐰 Изоляция через Firecracker требует Linux с включённым [KVM][kvm], +> а сам Runner должен работать от имени `root`. +> Каждый Job в песочнице запирается в chroot от имени непривилегированного пользователя, +> и построение этой изоляции требует прав, которых одно только включение KVM не даёт. > Runner, обслуживающий только не изолированные Spec, может работать на любом поддерживаемом хосте. [bare metal]: /ru/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/run.mdx index 9f5b27809..d083596e5 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/run.mdx @@ -29,6 +29,10 @@ runner up и обновляется только до версионированных релизов. Канал `canary` вместо этого отслеживает скользящую canary-сборку, развёрнутую в Bencher Cloud. +Runner, выполняющий Job в песочнице, должен работать от имени `root`, +поэтому команды выше приведены так, как их запустил бы пользователь `root`, +или под менеджером служб, который запускает Runner от имени `root`. + По умолчанию Runner отклоняет любую Job, чей Spec не имеет [Sandbox][sandbox]. Чтобы разрешить Runner выполнять не изолированные Job напрямую на хосте, запустите его с [флагом `--danger-allow-no-sandbox`][danger allow no sandbox]. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/intro.mdx index 46a7a352b..15b06c418 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/intro.mdx @@ -14,7 +14,10 @@ Self-Hosted Runner 是一个由你自己运行和管理的 [Bare Metal Runner][b 关于 Runner、Spec、Sandbox 和 Job 如何协同工作的概念性概述， 请参阅 [Bare Metal 概述][bare metal]。 -> 🐰 Firecracker 沙箱需要启用了 [KVM][kvm] 的 Linux。 +> 🐰 Firecracker 沙箱需要启用了 [KVM][kvm] 的 Linux， +> 并且 Runner 必须以 `root` 身份运行。 +> 每个沙箱 Job 都会以非特权用户的身份被限制在 chroot 中， +> 而构建这种限制所需的权限，仅仅启用 KVM 是无法获得的。 > 仅服务于非沙箱化 Spec 的 Runner 可以在任何受支持的主机上运行。 [bare metal]: /zh/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/run.mdx index 067b89b3d..90451c8ff 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/run.mdx @@ -29,6 +29,10 @@ runner up 仅更新到版本化的发布版本。 `canary` 通道则跟踪部署到 Bencher Cloud 的滚动 canary 构建。 +执行沙箱 Job 的 Runner 必须以 `root` 身份运行， +因此上面的命令是按 `root` 用户执行的方式给出的， +或者在以 `root` 启动 Runner 的服务管理器下运行。 + 默认情况下，Runner 会拒绝任何其 Spec 没有 [Sandbox][sandbox] 的 Job。 要让 Runner 直接在主机上执行非沙箱化的 Job， 请使用 [`--danger-allow-no-sandbox` 标志][danger allow no sandbox] 启动它。 diff --git a/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx b/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx index 3bfc3a0e3..1d6fed892 100644 --- a/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx +++ b/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx @@ -1,4 +1,5 @@ ## `v0.6.11` +- **BREAKING CHANGE** Confine the Firecracker VMM on Bare Metal Runners with the [Firecracker jailer](https://github.com/firecracker-microvm/firecracker/blob/main/docs/jailer.md): each sandboxed Job now runs in a chroot as a dedicated unprivileged user (`61016` by default, settable with `--jail-uid`/`--jail-gid`), in an empty network namespace, and is placed in its cgroup before it starts. A Runner that executes sandboxed Jobs must now run as **root**, where before it did not: building the jail requires `mknod` for the chroot's `/dev/kvm`, `chown` to hand the guest images to the jail user, `pivot_root`, and `setns` to join the network namespace. A world-readable `/dev/kvm` is enough to *use* KVM unprivileged but not to build the jail around it. Run `runner up` as root, or, if you cannot, start it with `--danger-allow-no-sandbox` and assign it only Specs with no Sandbox; be aware that this trades away the microVM itself and not just the jail, so those Jobs execute directly on the host. Runners also gain a persistent state directory (`--state-dir`, default `/var/lib/bencher-runner`) holding each Job's chroot, which is swept of anything a previous Runner left behind - Fix the percent difference shown in the report table when a metric drops to zero; a value of `0.00` against a positive baseline now shows `-100.00%` instead of `0.00%` in both the Console and the CI comment (Thank you [@OmarTawfik](https://github.com/OmarTawfik)) - **BREAKING CHANGE** Name the GitHub Check after the Project (ie `Bencher Report ()`) so runs for different Projects on the same commit no longer overwrite each other; `--ci-id` takes the Project name's place when set, the in-progress Check falls back to `Bencher Report` until the results are posted when `--project` is not set, and the name is truncated to 255 bytes. Update any required status check named `Bencher Report` in branch protection to the new name, and note that renaming a Project changes the Check name too (Thank you [@OmarTawfik](https://github.com/OmarTawfik)) From 82b3f6f37d55fede45c07abe6def75ea1417cb63 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 20:17:15 +0000 Subject: [PATCH 21/91] Degrade when the cpuset controller is absent, fail when it refuses Making a failed cpuset.cpus write fatal was too broad and would have broken hosts that worked before. enable_controllers falls back as far as +cpu +memory +pids, and only those three are required, so a host that does not delegate cpuset (a containerized runner, or a cgroup namespace without it in subtree_control) creates its cgroup successfully and then has no cpuset.cpus to write at all. Every sandboxed job on such a host would have started failing where it previously ran with a warning, and the doc comment claimed such hosts were handled earlier by not creating a cgroup, which is not what the code does. The line the spec actually draws is between an absent mechanism and a half-applied one. A controller that is not delegated is a declared absence of isolation: the cgroup is dropped and the job runs without one, exactly as on a host where the cgroup could not be created. A controller that is present and rejects the write is a cgroup claiming an isolation it does not have, which stays fatal. The local execution path takes the same distinction but keeps both branches best effort, since a non-sandboxed run makes no confinement claim to falsify. --- plus/bencher_runner/src/firecracker/mod.rs | 51 +++++---- plus/bencher_runner/src/jail/cgroup.rs | 118 +++++++++++++++------ plus/bencher_runner/src/local_isolation.rs | 15 ++- 3 files changed, 132 insertions(+), 52 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/mod.rs b/plus/bencher_runner/src/firecracker/mod.rs index 0dcbe76b9..f3ef8eb05 100644 --- a/plus/bencher_runner/src/firecracker/mod.rs +++ b/plus/bencher_runner/src/firecracker/mod.rs @@ -32,7 +32,7 @@ use std::time::{Duration, Instant}; use camino::Utf8PathBuf; use crate::cpu::CpuLayout; -use crate::jail::{CgroupManager, JailPaths, JailUser}; +use crate::jail::{CgroupManager, Cpuset, JailPaths, JailUser, VmId}; use crate::metrics::{self, RunMetrics}; pub use error::FirecrackerError; @@ -64,7 +64,7 @@ pub struct FirecrackerJobConfig { /// Identity of this microVM: the jailer id, the chroot name, and the /// cgroup name. Minted before the job's artifacts, because the jail root /// they are built in is a function of it. - pub vm_id: String, + pub vm_id: VmId, /// Both views of every file inside the jail chroot. pub jail: JailPaths, /// The unprivileged uid and gid the VMM drops to. @@ -123,7 +123,7 @@ pub fn run_firecracker( config: &FirecrackerJobConfig, cancel_flag: Option<&Arc>, ) -> Result { - let vm_id = config.vm_id.as_str(); + let vm_id = &config.vm_id; let jail = &config.jail; let start_time = Instant::now(); @@ -133,23 +133,36 @@ pub fn run_firecracker( if layout.has_isolation() { match CgroupManager::new(vm_id) { Ok(cg) => { - // A cgroup that exists but has no cpuset does not confine - // the VMM to the benchmark cores, so the run would report - // a number measured somewhere other than where it claims. - // Fatal, unlike failing to create the cgroup at all, which - // is a declared absence of isolation rather than a lie - // about it. - cg.apply_cpuset(layout) - .map_err(|e| FirecrackerError::CpusetFailed(Box::new(e)))?; - println!( - "CPU isolation: Firecracker pinned to cores {}", - layout.benchmark_cpuset() - ); - // Keep VM memory resident: swap adds run-to-run variance - if let Err(e) = cg.disable_swap() { - eprintln!("Warning: failed to disable swap for VM cgroup: {e}"); + // A cgroup that exists but does not confine the VMM to the + // benchmark cores would report a number measured somewhere + // other than where it claims, so a rejected cpuset is + // fatal. A controller the host does not delegate is a + // different thing: there is no isolation to be had, which + // is a declared limitation, so the cgroup is dropped and + // the job runs without one exactly as on a host that could + // not create it at all. + match cg + .apply_cpuset(layout) + .map_err(|e| FirecrackerError::CpusetFailed(Box::new(e)))? + { + Cpuset::Applied => { + println!( + "CPU isolation: Firecracker pinned to cores {}", + layout.benchmark_cpuset() + ); + // Keep VM memory resident: swap adds run-to-run variance + if let Err(e) = cg.disable_swap() { + eprintln!("Warning: failed to disable swap for VM cgroup: {e}"); + } + Some(cg) + }, + Cpuset::ControllerUnavailable => { + eprintln!( + "Warning: the cpuset controller is not delegated to this cgroup, so this run has no CPU isolation and its numbers carry more variance" + ); + None + }, } - Some(cg) }, Err(e) => { eprintln!("Warning: failed to create cgroup for CPU isolation: {e}"); diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index 0da4df166..106d28002 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -9,7 +9,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use crate::RunnerError; use crate::cpu::CpuLayout; use crate::error::JailError; -use crate::jail::ResourceLimits; +use crate::jail::{ResourceLimits, VmId}; /// Default cgroup v2 mount point. const CGROUP_ROOT: &str = "/sys/fs/cgroup"; @@ -24,11 +24,11 @@ pub struct CgroupManager { } impl CgroupManager { - /// Create a new cgroup for the given run ID. - pub fn new(run_id: &str) -> Result { + /// Create a new cgroup for the given microVM. + pub fn new(vm_id: &VmId) -> Result { let cgroup_path = Utf8PathBuf::from(CGROUP_ROOT) .join(BENCHER_CGROUP_BASE) - .join(run_id); + .join(vm_id.as_str()); // Ensure parent bencher cgroup exists let parent = Utf8PathBuf::from(CGROUP_ROOT).join(BENCHER_CGROUP_BASE); @@ -152,28 +152,36 @@ impl CgroupManager { /// /// # Errors /// - /// Returns an error when the cgroup exists but the cpuset cannot be - /// applied to it. A cgroup with no cpuset does not confine the VMM to the - /// benchmark cores, so the run would report a number measured somewhere - /// other than where it claims. A half-applied fidelity mechanism is a - /// confinement-grade failure; a host that cannot isolate at all is handled - /// earlier, by not creating a cgroup in the first place. - pub fn apply_cpuset(&self, layout: &CpuLayout) -> Result<(), RunnerError> { + /// Returns an error when the cpuset controller is present but rejects the + /// write. That is a half-applied fidelity mechanism: the cgroup would + /// exist without confining the VMM to the benchmark cores, so the run + /// would report a number measured somewhere other than where it claims. + /// + /// A controller that is not there at all is a different thing and is not + /// an error. `enable_controllers` falls back as far as `+cpu +memory + /// +pids`, and only those three are required, so a host that does not + /// delegate `cpuset` (a containerized runner, or a cgroup namespace + /// without it in `subtree_control`) creates its cgroup successfully and + /// then has no `cpuset.cpus` to write. That is a declared absence of + /// isolation, which the caller degrades on rather than failing. + pub fn apply_cpuset(&self, layout: &CpuLayout) -> Result { if !layout.has_isolation() { // No meaningful isolation possible (single core or overlapping sets) - return Ok(()); + return Ok(Cpuset::Applied); } let cpuset = layout.benchmark_cpuset(); if cpuset.is_empty() { - return Ok(()); + return Ok(Cpuset::Applied); } let path = self.cgroup_path.join("cpuset.cpus"); - fs::write(&path, &cpuset).map_err(|e| JailError::WriteCgroup { - path: path.clone(), - source: e, - })?; + if !path.exists() { + return Ok(Cpuset::ControllerUnavailable); + } + if let Err(e) = fs::write(&path, &cpuset) { + return classify_cpuset_error(path, e); + } // Also need to set cpuset.mems for cpuset to work. Use the parent's // effective memory nodes so multi-node NUMA hosts are not forced onto @@ -183,12 +191,11 @@ impl CgroupManager { .parent() .map_or_else(|| "0".to_owned(), effective_mems); let mems_path = self.cgroup_path.join("cpuset.mems"); - fs::write(&mems_path, &mems).map_err(|e| JailError::WriteCgroup { - path: mems_path, - source: e, - })?; + if let Err(e) = fs::write(&mems_path, &mems) { + return classify_cpuset_error(mems_path, e); + } - Ok(()) + Ok(Cpuset::Applied) } /// Apply I/O bandwidth limits. @@ -354,6 +361,43 @@ pub(crate) fn effective_mems(cgroup: &Utf8Path) -> String { } } +/// Whether the cpuset actually confined the VMM to the benchmark cores. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Cpuset { + /// The cgroup confines the VMM to the benchmark cores. + Applied, + /// The host does not delegate the cpuset controller, so there is nothing + /// to write and no CPU isolation to be had. + ControllerUnavailable, +} + +/// Decide whether a failed cpuset write is an absent controller or a refusal. +/// +/// A file that is not there is the controller not being delegated, which is a +/// limitation. Anything else is the kernel refusing a cpuset it does +/// understand, which would leave the cgroup claiming an isolation it does not +/// have. +fn classify_cpuset_error(path: Utf8PathBuf, error: std::io::Error) -> Result { + if error.kind() == std::io::ErrorKind::NotFound { + Ok(Cpuset::ControllerUnavailable) + } else { + Err(JailError::WriteCgroup { + path, + source: error, + } + .into()) + } +} + +/// How long to keep trying to remove a stale cgroup. +/// +/// `rmdir` fails while the cgroup still holds a process, and the reap that +/// precedes it may need a moment to land. +const REMOVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// How often to retry. +const REMOVE_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50); + /// Remove the cgroup a swept jail left behind. /// /// The cgroup and the chroot are named by the same VM id by construction, so @@ -363,18 +407,32 @@ pub(crate) fn effective_mems(cgroup: &Utf8Path) -> String { /// still holds a process fails, which is what forces that ordering. A cgroup /// that survives is worth shouting about, because it holds the exclusive /// benchmark CPUs and the next job's cpuset will be rejected because of it. -pub(crate) fn remove_stale_cgroup(vm_id: &str) { +pub(crate) fn remove_stale_cgroup(vm_id: &VmId) -> Result<(), JailError> { let path = Utf8PathBuf::from(CGROUP_ROOT) .join(BENCHER_CGROUP_BASE) - .join(vm_id); + .join(vm_id.as_str()); if !path.exists() { - return; + return Ok(()); } - match fs::remove_dir(&path) { - Ok(()) => eprintln!("Warning: removed stale cgroup {path} left by a previous runner"), - Err(e) => eprintln!( - "Warning: failed to remove stale cgroup {path}: {e}. It still holds the benchmark CPUs, so the next run's CPU isolation will be rejected." - ), + + let deadline = std::time::Instant::now() + REMOVE_TIMEOUT; + loop { + match fs::remove_dir(&path) { + Ok(()) => { + eprintln!("Warning: removed stale cgroup {path} left by a previous runner"); + return Ok(()); + }, + // Someone else got there first, which is the outcome either way. + Err(_) if !path.exists() => return Ok(()), + Err(e) if std::time::Instant::now() >= deadline => { + // Reported rather than warned. The leftover still owns the + // exclusive benchmark CPUs, so every later job's cpuset would + // be rejected; failing here means the next job sweeps again + // instead of inheriting a host that can never isolate. + return Err(JailError::StaleCgroup { path, source: e }); + }, + Err(_) => std::thread::sleep(REMOVE_INTERVAL), + } } } diff --git a/plus/bencher_runner/src/local_isolation.rs b/plus/bencher_runner/src/local_isolation.rs index ce25beae7..e909cb525 100644 --- a/plus/bencher_runner/src/local_isolation.rs +++ b/plus/bencher_runner/src/local_isolation.rs @@ -57,12 +57,21 @@ impl LocalIsolation { }; let benchmark = layout.benchmark.clone(); - let run_id = format!("local-{}", uuid::Uuid::new_v4()); + let run_id = crate::jail::VmId::from_chroot_name(format!("local-{}", uuid::Uuid::new_v4())); let cgroup = match crate::jail::CgroupManager::new(&run_id) { Ok(cgroup) => { - if let Err(e) = cgroup.apply_cpuset(layout) { - eprintln!("Warning: failed to apply cpuset for local run: {e}"); + // Best effort here, unlike the sandboxed path: a local run + // makes no confinement claim to begin with, so losing the + // cpuset degrades its numbers rather than falsifying them. + match cgroup.apply_cpuset(layout) { + Ok(crate::jail::Cpuset::Applied) => {}, + Ok(crate::jail::Cpuset::ControllerUnavailable) => { + eprintln!( + "Warning: the cpuset controller is not delegated, so this local run has no CPU isolation" + ); + }, + Err(e) => eprintln!("Warning: failed to apply cpuset for local run: {e}"), } // Keep benchmark memory resident, mirroring the Firecracker path if let Err(e) = cgroup.disable_swap() { From cc2af916379105e7e8fb0bcb1bc457a570e3518e Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 20:17:29 +0000 Subject: [PATCH 22/91] Stop a failed sweep from wedging a runner permanently A stale cgroup that could not be removed was only warned about, while the message itself said the leftover still holds the benchmark CPUs and the next run's isolation will be rejected. With that rejection now fatal, and with preparation latching on success, one unremovable cgroup meant every job in that daemon failed forever with no retry: recovery needed a restart and a manual rmdir. The module doc promised a failure is not remembered, and that promise was false. Removal now retries on a deadline, because rmdir fails while the cgroup still holds a process and the reap that precedes it may need a moment to land. If it still fails the sweep reports it, so preparation does not latch and the next job sweeps again rather than inheriting a host that can never isolate. A chroot that will not go away costs disk and stays a warning. The reap no longer fails silently. pidfd_open collapsed ENOSYS on kernels before 5.3, EPERM, and already-exited into one empty answer, so an orphan could be left holding the benchmark CPUs with nothing said. Already-exited is now distinguished from everything else, and everything else is reported. wait_for_exit also treats a zombie as exited. The orphan reparents to PID 1, and where the runner is itself PID 1 with no init to reap it, /proc/ persists forever and every sweep would stall its full timeout and warn about a process that was already dead. --- plus/bencher_runner/src/error.rs | 9 ++ plus/bencher_runner/src/jail/reap.rs | 113 ++++++++++++++++++++++---- plus/bencher_runner/src/jail/state.rs | 76 +++++++++++------ 3 files changed, 155 insertions(+), 43 deletions(-) diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index 57e628922..0a7a6ebbf 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -138,6 +138,15 @@ pub enum JailError { )] NetnsNotDistinct { path: Utf8PathBuf }, + #[cfg(target_os = "linux")] + #[error( + "Failed to remove the stale cgroup {path}: {source}. It still owns the exclusive benchmark CPUs, so no run can be isolated until it is gone." + )] + StaleCgroup { + path: Utf8PathBuf, + source: std::io::Error, + }, + #[cfg(target_os = "linux")] #[error("Failed to open the jail chroot {path}: {source}")] OpenJailRoot { diff --git a/plus/bencher_runner/src/jail/reap.rs b/plus/bencher_runner/src/jail/reap.rs index 41c597f9b..ec75b3a69 100644 --- a/plus/bencher_runner/src/jail/reap.rs +++ b/plus/bencher_runner/src/jail/reap.rs @@ -45,7 +45,20 @@ pub fn reap_jailed_vmm(jail_root: &Utf8Path) -> Option { // pidfd refers to one process for as long as it is open and keeps the // number from being reused, which turns the check below into a guarantee // rather than a narrow window. - let pidfd = pidfd_open(pid)?; + let pidfd = match pidfd_open(pid) { + Ok(Some(pidfd)) => pidfd, + // Already gone, which is the common case and not a failure. + Ok(None) => return None, + Err(e) => { + // Silence here is what wedges a runner: the orphan keeps the + // benchmark CPUs, the cgroup cannot be removed, and nothing says + // why. Kernels before 5.3 have no pidfd_open at all. + eprintln!( + "Warning: cannot pin orphaned VMM (pid {pid}) in {jail_root} to reap it: {e}. It is still running and still holds the benchmark CPUs." + ); + return None; + }, + }; // Re-check now that the pid cannot change underneath us. if !is_jailed_vmm(pid, jail_root) { @@ -103,9 +116,17 @@ fn matches_jail(pid: u32, jail: &fs::Metadata) -> bool { /// Open a descriptor pinned to a process. /// -/// `None` when the process is already gone, which is the common case and not -/// an error: something else reaped it first. -fn pidfd_open(pid: u32) -> Option { +/// `Ok(None)` means the process is already gone, which is the common case and +/// not a failure. Everything else is distinguished and reported by the caller, +/// because a reap that quietly does nothing leaves an orphan holding the +/// benchmark CPUs: `ENOSYS` on a kernel before 5.3, `EPERM` under a +/// restrictive policy, and a pid too large to convert all look identical from +/// the outside otherwise. +fn pidfd_open(pid: u32) -> std::io::Result> { + let pid = libc::pid_t::try_from(pid).map_err(|_err| { + std::io::Error::new(std::io::ErrorKind::InvalidInput, "pid out of range") + })?; + #[expect( unsafe_code, reason = "pidfd_open has no std wrapper; it takes plain integers" @@ -113,11 +134,19 @@ fn pidfd_open(pid: u32) -> Option { // SAFETY: `pidfd_open` takes a pid and a flag word and touches no memory. // It returns a new descriptor or -1, and the descriptor is handed straight // to `OwnedFd` so it is closed exactly once. - let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, libc::pid_t::try_from(pid).ok()?, 0) }; - let raw = libc::c_int::try_from(raw).ok()?; + let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) }; + if raw < 0 { - return None; + let error = std::io::Error::last_os_error(); + return if error.raw_os_error() == Some(libc::ESRCH) { + Ok(None) + } else { + Err(error) + }; } + + let raw = libc::c_int::try_from(raw) + .map_err(|_err| std::io::Error::other("pidfd out of descriptor range"))?; #[expect( unsafe_code, reason = "taking ownership of a descriptor this call just created" @@ -125,7 +154,7 @@ fn pidfd_open(pid: u32) -> Option { // SAFETY: `raw` is a fresh descriptor returned by the syscall above and is // not owned by anything else. let fd = unsafe { OwnedFd::from_raw_fd(raw) }; - Some(fd) + Ok(Some(fd)) } /// SIGKILL the process a descriptor is pinned to. @@ -156,11 +185,17 @@ fn pidfd_kill(pidfd: &OwnedFd) -> std::io::Result<()> { } } -/// Wait for a killed process to disappear. +/// Wait for a killed process to stop running. +/// +/// A zombie counts as exited. The orphan reparents to whatever is PID 1, and +/// if the runner is itself PID 1 (a container with no init) nothing ever reaps +/// it, so `/proc/` persists forever. Waiting on the directory alone would +/// then stall the full timeout on every sweep and warn about a process that is +/// already dead. fn wait_for_exit(pid: u32) -> bool { let deadline = Instant::now() + REAP_TIMEOUT; while Instant::now() < deadline { - if !Utf8Path::new(&format!("/proc/{pid}")).exists() { + if !is_running(pid) { return true; } std::thread::sleep(REAP_INTERVAL); @@ -168,6 +203,22 @@ fn wait_for_exit(pid: u32) -> bool { false } +/// Whether a process still exists and is not a zombie. +fn is_running(pid: u32) -> bool { + let Ok(status) = fs::read_to_string(format!("/proc/{pid}/status")) else { + return false; + }; + !is_zombie(&status) +} + +/// Whether a `/proc//status` listing describes a zombie. +fn is_zombie(status: &str) -> bool { + status + .lines() + .find_map(|line| line.strip_prefix("State:")) + .is_some_and(|state| state.trim_start().starts_with('Z')) +} + #[cfg(test)] mod tests { use camino::Utf8PathBuf; @@ -214,13 +265,43 @@ mod tests { } #[test] - fn a_pidfd_pins_a_live_process_and_refuses_a_dead_one() { - pidfd_open(std::process::id()).expect("this process is alive"); + fn a_pidfd_pins_a_live_process() { + let pidfd = pidfd_open(std::process::id()).expect("pidfd_open is available"); + assert!(pidfd.is_some(), "this process is alive"); + } + + #[test] + fn a_pidfd_distinguishes_gone_from_broken() { + // Pid 0 is never a process, and the syscall rejects it rather than + // signalling the caller's process group the way plain `kill` would. + // The distinction matters: an absent process is nothing to report, + // while a refusal leaves an orphan running and has to be said out loud. + match pidfd_open(0) { + Ok(None) => {}, + Ok(Some(_)) => panic!("pid 0 must never yield a pidfd"), + Err(e) => assert_ne!( + e.raw_os_error(), + Some(libc::ESRCH), + "ESRCH must be reported as gone, not as an error" + ), + } + } - // Pid 0 is never a process: the syscall rejects it rather than - // signalling the caller's process group, which is what the plain - // `kill` interface would have done. - assert!(pidfd_open(0).is_none()); + #[test] + fn a_zombie_counts_as_exited() { + // Without this the reap stalls its full timeout whenever the runner is + // PID 1 and never reaps what reparents to it. + let zombie = "Name:\tfirecracker\nUmask:\t0022\nState:\tZ (zombie)\nTgid:\t42\n"; + let running = "Name:\tfirecracker\nUmask:\t0022\nState:\tS (sleeping)\nTgid:\t42\n"; + + assert!(is_zombie(zombie)); + assert!(!is_zombie(running)); + assert!(!is_zombie("")); + } + + #[test] + fn this_process_is_running() { + assert!(is_running(std::process::id())); } #[test] diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index 67d946cb1..85decb8f1 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -12,6 +12,7 @@ use std::os::unix::fs::PermissionsExt as _; use camino::{Utf8Path, Utf8PathBuf}; use crate::error::JailError; +use crate::jail::VmId; /// Subdirectory of the state directory used as the jailer's chroot base. const CHROOT_BASE: &str = "jail"; @@ -61,13 +62,13 @@ impl StateDir { /// put there. Without this, `--state-dir /var/lib` would be chmodded to /// 0700 and take the host down with it. fn check_root_is_ours(&self) -> Result<(), JailError> { - let Ok(mut entries) = fs::read_dir(&self.root) else { + let Ok(entries) = fs::read_dir(&self.root) else { // Missing, or unreadable: creating it is the next step and will // report the real error. return Ok(()); }; let mut populated = false; - for entry in entries.by_ref().flatten() { + for entry in entries.flatten() { populated = true; let name = entry.file_name(); if RUNNER_ENTRIES.iter().any(|ours| name == *ours) { @@ -98,13 +99,13 @@ impl StateDir { /// The jail directory for a VM, the tree teardown removes. #[must_use] - pub fn jail_dir(&self, vm_id: &str) -> Utf8PathBuf { - self.jail_parent().join(vm_id) + pub fn jail_dir(&self, vm_id: &VmId) -> Utf8PathBuf { + self.jail_parent().join(vm_id.as_str()) } /// The chroot root for a VM, which becomes `/` inside the jail. #[must_use] - pub fn jail_root(&self, vm_id: &str) -> Utf8PathBuf { + pub fn jail_root(&self, vm_id: &VmId) -> Utf8PathBuf { self.jail_dir(vm_id).join(JAIL_ROOT) } @@ -150,9 +151,9 @@ const RUNNER_ENTRIES: [&str; 2] = [CHROOT_BASE, LOCK_FILE]; /// /// Non-directory entries are left alone: the jailer only ever creates /// directories here, so anything else was put there by someone else. -pub fn sweep_jails(jail_parent: &Utf8Path) -> usize { +pub fn sweep_jails(jail_parent: &Utf8Path) -> Result { let Ok(entries) = fs::read_dir(jail_parent) else { - return 0; + return Ok(0); }; let mut swept = 0; @@ -160,25 +161,28 @@ pub fn sweep_jails(jail_parent: &Utf8Path) -> usize { if !entry.file_type().is_ok_and(|file_type| file_type.is_dir()) { continue; } - let vm_id = entry.file_name().to_string_lossy().into_owned(); - let jail_dir = jail_parent.join(&vm_id); + let vm_id = VmId::from_chroot_name(entry.file_name().to_string_lossy().into_owned()); + let jail_dir = jail_parent.join(vm_id.as_str()); // Reap before removing. Pulling the rootfs out from under a process // that is still running leaves it running anyway, so the process goes // first and the directory second. super::reap::reap_jailed_vmm(&jail_dir.join(JAIL_ROOT)); + // A chroot that will not go away costs disk. Worth a warning, not + // worth refusing to run. match fs::remove_dir_all(&jail_dir) { Ok(()) => swept += 1, Err(e) => eprintln!("Warning: failed to sweep stale jail {jail_dir}: {e}"), } - // The cgroup is the half that actually corrupts later runs: it holds - // the exclusive benchmark CPUs, so leaving it makes the next job's - // cpuset write fail. It shares the chroot's name by construction. - super::cgroup::remove_stale_cgroup(&vm_id); + // The cgroup is the half that corrupts later runs: it holds the + // exclusive benchmark CPUs, so leaving it makes every later job's + // cpuset write fail. It shares the chroot's name by construction, and + // failing to remove it is reported rather than swallowed. + super::cgroup::remove_stale_cgroup(&vm_id)?; } - swept + Ok(swept) } #[cfg(test)] @@ -200,12 +204,12 @@ mod tests { "/var/lib/bencher-runner/jail/firecracker" ); assert_eq!( - state.jail_dir("abc"), + state.jail_dir(&VmId::from_chroot_name("abc".to_owned())), "/var/lib/bencher-runner/jail/firecracker/abc" ); // ///root assert_eq!( - state.jail_root("abc"), + state.jail_root(&VmId::from_chroot_name("abc".to_owned())), state .chroot_base() .join(EXEC_FILE_NAME) @@ -293,13 +297,27 @@ mod tests { state.create().unwrap(); // Two stale jails, one with a nested chroot tree. - fs::create_dir_all(state.jail_root("one")).unwrap(); - fs::write(state.jail_root("one").join("rootfs.ext4"), b"stale").unwrap(); - fs::create_dir_all(state.jail_dir("two")).unwrap(); - - assert_eq!(sweep_jails(&state.jail_parent()), 2); - assert!(!state.jail_dir("one").exists()); - assert!(!state.jail_dir("two").exists()); + fs::create_dir_all(state.jail_root(&VmId::from_chroot_name("one".to_owned()))).unwrap(); + fs::write( + state + .jail_root(&VmId::from_chroot_name("one".to_owned())) + .join("rootfs.ext4"), + b"stale", + ) + .unwrap(); + fs::create_dir_all(state.jail_dir(&VmId::from_chroot_name("two".to_owned()))).unwrap(); + + assert_eq!(sweep_jails(&state.jail_parent()).unwrap(), 2); + assert!( + !state + .jail_dir(&VmId::from_chroot_name("one".to_owned())) + .exists() + ); + assert!( + !state + .jail_dir(&VmId::from_chroot_name("two".to_owned())) + .exists() + ); assert!(state.jail_parent().exists()); } @@ -311,16 +329,20 @@ mod tests { let note = state.jail_parent().join("NOTES.txt"); fs::write(¬e, b"not a jail").unwrap(); - fs::create_dir_all(state.jail_dir("stale")).unwrap(); + fs::create_dir_all(state.jail_dir(&VmId::from_chroot_name("stale".to_owned()))).unwrap(); - assert_eq!(sweep_jails(&state.jail_parent()), 1); - assert!(!state.jail_dir("stale").exists()); + assert_eq!(sweep_jails(&state.jail_parent()).unwrap(), 1); + assert!( + !state + .jail_dir(&VmId::from_chroot_name("stale".to_owned())) + .exists() + ); assert!(note.exists(), "non-directory entries are not the sweep's"); } #[test] fn sweep_missing_parent_is_zero() { let (_dir, root) = temp_root(); - assert_eq!(sweep_jails(&root.join("nope")), 0); + assert_eq!(sweep_jails(&root.join("nope")).unwrap(), 0); } } From 7f27678c1872cdc8954df7f4f8d4f8ed80f3ac93 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 20:17:44 +0000 Subject: [PATCH 23/91] Own the host preparation latch, and name the VM identity The latch was a static AtomicBool, which the repo rules prohibit outright. It also read and wrote as two separate operations, harmless only because JailLock happened to serialize every caller, and it made the laziness test depend on what else had run in the process: it passed under nextest, which gives each test its own process, and was flaky under plain cargo test. It is now an owned token created by the daemon loop and by the one-shot CLI, threaded to the executor. The latch belongs to one runner process, nothing else can observe or reset it, and each test gets its own. Verified in-process and single-threaded on Linux, which is the case a global would have broken. The VM identity gets a newtype. The same string is the jailer's --id, the chroot directory name, and the cgroup name, and remove_stale_cgroup took a bare &str read straight off a directory entry, which is exactly the confusion worth making impossible. Recovering an identity from a chroot name is now a named operation rather than an implicit conversion. Also renames copy_into_jail, which was called to stage the Firecracker binary outside the chroot and printed 'Copied ... into the jail at /tmp/...', which was simply false. The jail wording moves to the kernel call sites, which are the ones that actually copy into the chroot. --- .../bencher_runner/src/firecracker/process.rs | 30 ++-- plus/bencher_runner/src/jail/chroot.rs | 31 ++-- plus/bencher_runner/src/jail/mod.rs | 154 ++++++++++++++---- plus/bencher_runner/src/lib.rs | 5 +- plus/bencher_runner/src/run.rs | 16 +- plus/bencher_runner/src/up/job.rs | 3 +- plus/bencher_runner/src/up/mod.rs | 8 +- plus/bencher_runner/src/vm.rs | 32 ++-- 8 files changed, 200 insertions(+), 79 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/process.rs b/plus/bencher_runner/src/firecracker/process.rs index f0da6aa6a..5c0e466c1 100644 --- a/plus/bencher_runner/src/firecracker/process.rs +++ b/plus/bencher_runner/src/firecracker/process.rs @@ -11,7 +11,7 @@ use camino::Utf8Path; use crate::firecracker::client::FirecrackerClient; use crate::firecracker::config::{Action, ActionType}; use crate::firecracker::error::FirecrackerError; -use crate::jail::{JailFile, JailUser}; +use crate::jail::{JailFile, JailUser, VmId}; /// Everything needed to spawn the VMM under the jailer. #[derive(Debug)] @@ -26,7 +26,7 @@ pub struct JailedSpawn<'a> { /// chroot layout, so it is fixed rather than incidental. pub exec_file: &'a Utf8Path, /// The jailer `--id`, which is also the chroot name and the cgroup name. - pub vm_id: &'a str, + pub vm_id: &'a VmId, /// The unprivileged uid and gid the VMM drops to. pub jail_user: JailUser, /// The jailer `--chroot-base-dir`. @@ -115,14 +115,15 @@ impl FirecrackerProcess { } } - let mut child = command.spawn().map_err(|e| { - FirecrackerError::ProcessStart(format!("failed to spawn {jailer_bin}: {e}")) + let mut child = command.spawn().map_err(|e| FirecrackerError::Spawn { + path: jailer_bin.to_owned(), + source: e, })?; // Spawn a thread to read stderr line-by-line - let stderr = child.stderr.take().ok_or_else(|| { - FirecrackerError::ProcessStart("stderr was piped but not available".into()) - })?; + let stderr = child.stderr.take().ok_or(FirecrackerError::Stdio( + "stderr was piped but not available", + ))?; let stderr_thread = std::thread::spawn(move || { use std::io::BufRead as _; @@ -230,7 +231,7 @@ impl Drop for FirecrackerProcess { fn jailer_args(spawn: &JailedSpawn<'_>) -> Vec { vec![ "--id".to_owned(), - spawn.vm_id.to_owned(), + spawn.vm_id.to_string(), "--exec-file".to_owned(), spawn.exec_file.to_string(), "--uid".to_owned(), @@ -273,11 +274,11 @@ mod tests { use super::*; use crate::jail::JailPaths; - fn spawn_for(jail: &JailPaths) -> JailedSpawn<'_> { + fn spawn_for<'a>(jail: &'a JailPaths, vm_id: &'a VmId) -> JailedSpawn<'a> { JailedSpawn { jailer_bin: Utf8Path::new("/tmp/work/jailer"), exec_file: Utf8Path::new("/tmp/work/firecracker"), - vm_id: "vm-1", + vm_id, jail_user: JailUser::default(), chroot_base_dir: Utf8Path::new("/var/lib/bencher-runner/jail"), netns: Utf8Path::new("/run/netns/bencher-jail"), @@ -290,7 +291,12 @@ mod tests { fn args() -> Vec { let (_dir, jail) = jail_in_tmpdir(); - jailer_args(&spawn_for(&jail)) + jailer_args(&spawn_for(&jail, &vm_id())) + } + + /// A stand-in identity for tests. + fn vm_id() -> VmId { + VmId::from_chroot_name("vm-1".to_owned()) } /// The jail root has to exist: the paths hold a descriptor on it. @@ -367,7 +373,7 @@ mod tests { // receive the path as it will exist inside the chroot. The host view // names a directory the jailed process cannot reach. let (_dir, jail) = jail_in_tmpdir(); - let args = jailer_args(&spawn_for(&jail)); + let args = jailer_args(&spawn_for(&jail, &vm_id())); assert_eq!(value_of(&args, "--api-sock"), Some("/api.sock")); let jail_root = jail.root().as_str(); diff --git a/plus/bencher_runner/src/jail/chroot.rs b/plus/bencher_runner/src/jail/chroot.rs index d61244f2a..b6fa33e8c 100644 --- a/plus/bencher_runner/src/jail/chroot.rs +++ b/plus/bencher_runner/src/jail/chroot.rs @@ -14,7 +14,7 @@ use std::os::unix::fs::{PermissionsExt as _, chown}; use camino::{Utf8Path, Utf8PathBuf}; use crate::error::JailError; -use crate::jail::{JailUser, StateDir}; +use crate::jail::{JailUser, StateDir, VmId}; /// A job's chroot tree, removed when this value is dropped. /// @@ -29,7 +29,7 @@ pub struct JailDir { impl JailDir { /// Create the chroot tree for `vm_id` at mode 0700. - pub fn create(state: &StateDir, vm_id: &str) -> Result { + pub fn create(state: &StateDir, vm_id: &VmId) -> Result { let dir = state.jail_dir(vm_id); let root = state.jail_root(vm_id); @@ -87,6 +87,11 @@ pub fn chown_to_jail(path: &Utf8Path, jail_user: JailUser) -> Result<(), JailErr mod tests { use super::*; + /// A stand-in identity for tests. + fn vm_id() -> VmId { + VmId::from_chroot_name("vm-1".to_owned()) + } + fn state_in_tmpdir() -> (tempfile::TempDir, StateDir) { let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); @@ -99,11 +104,11 @@ mod tests { fn create_builds_a_private_chroot_tree() { let (_dir, state) = state_in_tmpdir(); - let jail = JailDir::create(&state, "vm-1").unwrap(); + let jail = JailDir::create(&state, &vm_id()).unwrap(); - assert_eq!(jail.root(), state.jail_root("vm-1")); + assert_eq!(jail.root(), state.jail_root(&vm_id())); assert!(jail.root().is_dir()); - for path in [state.jail_dir("vm-1"), state.jail_root("vm-1")] { + for path in [state.jail_dir(&vm_id()), state.jail_root(&vm_id())] { let mode = fs::metadata(&path).unwrap().permissions().mode(); assert_eq!(mode & 0o777, 0o700, "{path} should be private"); } @@ -112,9 +117,9 @@ mod tests { #[test] fn create_tolerates_an_existing_directory() { let (_dir, state) = state_in_tmpdir(); - fs::create_dir_all(state.jail_root("vm-1")).unwrap(); + fs::create_dir_all(state.jail_root(&vm_id())).unwrap(); - JailDir::create(&state, "vm-1").unwrap(); + JailDir::create(&state, &vm_id()).unwrap(); } #[test] @@ -122,13 +127,13 @@ mod tests { let (_dir, state) = state_in_tmpdir(); { - let jail = JailDir::create(&state, "vm-1").unwrap(); + let jail = JailDir::create(&state, &vm_id()).unwrap(); fs::write(jail.root().join("rootfs.ext4"), b"guest").unwrap(); fs::create_dir_all(jail.root().join("dev")).unwrap(); } assert!( - !state.jail_dir("vm-1").exists(), + !state.jail_dir(&vm_id()).exists(), "the chroot is the runner's to reclaim, not the jailer's" ); assert!(state.jail_parent().exists()); @@ -141,16 +146,16 @@ mod tests { let (_dir, state) = state_in_tmpdir(); // A file where the jail directory has to go makes the tree // impossible to create. - fs::write(state.jail_dir("vm-1"), b"in the way").unwrap(); + fs::write(state.jail_dir(&vm_id()), b"in the way").unwrap(); - JailDir::create(&state, "vm-1").unwrap_err(); + JailDir::create(&state, &vm_id()).unwrap_err(); } #[test] fn drop_tolerates_an_already_removed_tree() { let (_dir, state) = state_in_tmpdir(); - let jail = JailDir::create(&state, "vm-1").unwrap(); - fs::remove_dir_all(state.jail_dir("vm-1")).unwrap(); + let jail = JailDir::create(&state, &vm_id()).unwrap(); + fs::remove_dir_all(state.jail_dir(&vm_id())).unwrap(); drop(jail); } } diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 68bdc59ca..e008d515d 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -21,11 +21,11 @@ pub mod reap; #[cfg(target_os = "linux")] pub mod state; -#[cfg(target_os = "linux")] -pub use cgroup::CgroupManager; #[cfg(target_os = "linux")] pub(crate) use cgroup::{BENCHER_CGROUP_BASE, effective_mems}; #[cfg(target_os = "linux")] +pub use cgroup::{CgroupManager, Cpuset}; +#[cfg(target_os = "linux")] pub use chroot::JailDir; #[cfg(target_os = "linux")] pub use lock::JailLock; @@ -34,9 +34,6 @@ pub use paths::{ChrootPath, HostPath, JailFile, JailPaths, SocketPath}; #[cfg(target_os = "linux")] pub use state::StateDir; -#[cfg(target_os = "linux")] -use std::sync::atomic::{AtomicBool, Ordering}; - use serde::{Deserialize, Serialize}; /// Default location of the runner's persistent state directory. @@ -117,35 +114,112 @@ impl Default for JailUser { } } -/// Set once the host has been prepared, so it happens at most once per -/// runner process. -#[cfg(target_os = "linux")] -static HOST_PREPARED: AtomicBool = AtomicBool::new(false); - -/// Prepare the host for jailed execution, once per runner process. +/// The identity of one microVM. /// -/// Called on demand, immediately before the first job builds a jail, never at -/// startup. The daemon learns which Specs it serves from the server, so at -/// startup it cannot know whether it will ever need a jail, and a Runner that -/// serves only non-sandboxed Specs is a supported configuration that must come -/// up on a host where the runner is not root. Preparing eagerly would make -/// `runner up` require root just to start. +/// The same string is the jailer's `--id`, the name of the chroot directory, +/// and the name of the cgroup, by construction. Naming it once keeps the three +/// from drifting, and keeps a bare directory name read off the filesystem from +/// being mistaken for an identity that was minted. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VmId(String); + +impl VmId { + /// Mint a fresh identity for a job. + #[must_use] + pub fn new() -> Self { + Self(uuid::Uuid::new_v4().to_string()) + } + + /// Recover the identity of a jail from its chroot directory name. + /// + /// The sweep works backwards from the filesystem, and the directory name + /// is the identity that created it. + #[must_use] + pub fn from_chroot_name(name: String) -> Self { + Self(name) + } + + /// The identity as a string. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Default for VmId { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Display for VmId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +/// Tracks whether this runner process has prepared the host. /// -/// Failure is fatal. Untrusted code never runs with silently degraded -/// confinement, so a host that cannot be prepared does not execute a job. A -/// failure is not remembered, so a transient permission problem is retried by -/// the next job rather than requiring a restart. -#[cfg(target_os = "linux")] -pub fn prepare_host_once( - state_dir: &camino::Utf8Path, - jail_user: JailUser, -) -> Result<(), crate::error::JailError> { - if HOST_PREPARED.load(Ordering::SeqCst) { - return Ok(()); +/// Owned and threaded through the callers rather than kept in a global, +/// so that the latch belongs to one runner and cannot be observed, or reset, +/// by anything else. Tests get their own. +#[derive(Debug, Default)] +pub struct HostPreparation { + /// Only the jail reads this, and the jail is Linux-only. + #[cfg_attr( + not(target_os = "linux"), + expect(dead_code, reason = "host preparation is Linux-only") + )] + prepared: bool, +} + +impl HostPreparation { + /// A runner process that has not prepared the host yet. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Prepare the host for jailed execution, at most once. + /// + /// Called on demand, immediately before the first job builds a jail, never + /// at startup. The daemon learns which Specs it serves from the server, so + /// at startup it cannot know whether it will ever need a jail, and a + /// Runner that serves only non-sandboxed Specs is a supported + /// configuration that must come up on a host where the runner is not root. + /// Preparing eagerly would make `runner up` require root just to start. + /// + /// Failure is fatal. Untrusted code never runs with silently degraded + /// confinement, so a host that cannot be prepared does not execute a job. + /// A failure is not remembered, so the next job retries rather than + /// needing a restart. That matters most for a sweep that could not reclaim + /// a stale cgroup: the orphan may simply not have exited yet, and a + /// latched failure would leave every later job failing with no retry. + #[cfg(target_os = "linux")] + pub fn ensure( + &mut self, + state_dir: &camino::Utf8Path, + jail_user: JailUser, + ) -> Result<(), crate::error::JailError> { + if self.prepared { + return Ok(()); + } + prepare_host(state_dir, jail_user)?; + self.prepared = true; + Ok(()) + } + + /// Prepare the host for jailed execution, at most once. + /// + /// The jail is Linux-only, as is the VM executor it protects. + #[cfg(not(target_os = "linux"))] + pub fn ensure( + &mut self, + _state_dir: &camino::Utf8Path, + _jail_user: JailUser, + ) -> Result<(), crate::error::JailError> { + Ok(()) } - prepare_host(state_dir, jail_user)?; - HOST_PREPARED.store(true, Ordering::SeqCst); - Ok(()) } /// Create the state directory and reclaim what a previous runner left behind. @@ -170,7 +244,7 @@ fn prepare_host( warn_on_named_account(jail_user); let _lock = JailLock::acquire(state.path())?; - let swept = state::sweep_jails(&state.jail_parent()); + let swept = state::sweep_jails(&state.jail_parent())?; if swept > 0 { // Each one held a copy of the VMM binary and a full guest rootfs // image, so an operator should hear about it. @@ -373,14 +447,24 @@ mod tests { let state_dir = root.join("state"); assert!(!state_dir.exists(), "startup has not prepared anything"); - prepare_host_once(&state_dir, JailUser::default()).unwrap(); + let mut host = HostPreparation::new(); + host.ensure(&state_dir, JailUser::default()).unwrap(); assert!(state_dir.join("jail").is_dir(), "the first job prepares"); // A second job must not redo it: proven by removing the tree and - // seeing that it is not rebuilt. + // seeing that it is not rebuilt. Owning the token is what makes this + // independent of every other test in the process. std::fs::remove_dir_all(&state_dir).unwrap(); - prepare_host_once(&state_dir, JailUser::default()).unwrap(); + host.ensure(&state_dir, JailUser::default()).unwrap(); assert!(!state_dir.exists(), "preparation happens at most once"); + + // A different runner prepares its own host. + let mut other = HostPreparation::new(); + other.ensure(&state_dir, JailUser::default()).unwrap(); + assert!( + state_dir.join("jail").is_dir(), + "a fresh token prepares again" + ); } #[cfg(target_os = "linux")] diff --git a/plus/bencher_runner/src/lib.rs b/plus/bencher_runner/src/lib.rs index 4feb878ca..0fae74702 100644 --- a/plus/bencher_runner/src/lib.rs +++ b/plus/bencher_runner/src/lib.rs @@ -62,7 +62,10 @@ pub use config::Config; #[cfg(feature = "plus")] pub use error::{ConfigError, ExecutionError, JailError, RunnerError}; #[cfg(feature = "plus")] -pub use jail::{DEFAULT_JAIL_GID, DEFAULT_JAIL_UID, DEFAULT_STATE_DIR, JailUser, ResourceLimits}; +pub use jail::{ + DEFAULT_JAIL_GID, DEFAULT_JAIL_UID, DEFAULT_STATE_DIR, HostPreparation, JailUser, + ResourceLimits, +}; #[cfg(feature = "plus")] pub use log_level::SandboxLogLevel; #[cfg(feature = "plus")] diff --git a/plus/bencher_runner/src/run.rs b/plus/bencher_runner/src/run.rs index f9a33cb19..4f2abeab8 100644 --- a/plus/bencher_runner/src/run.rs +++ b/plus/bencher_runner/src/run.rs @@ -177,9 +177,13 @@ pub fn run_with_args(args: &RunArgs) -> Result<(), RunnerError> { } } + // Owned by this invocation rather than shared: the latch belongs to one + // runner process and nothing else can observe or reset it. + let mut host = crate::jail::HostPreparation::new(); + let iter_count = args.iter.as_usize(); for iteration in 0..iter_count { - match execute(&config, None) { + match execute(&config, &mut host, None) { Ok(output) => { println!("{}", output.stdout); if !output.stderr.is_empty() { @@ -402,15 +406,23 @@ pub fn resolve_oci_config( /// # Returns /// /// The benchmark output including exit code and stdout. +#[cfg_attr( + not(target_os = "linux"), + expect( + unused_variables, + reason = "host preparation is Linux-only, as is the VM executor it prepares for" + ) +)] pub fn execute( config: &crate::Config, + host: &mut crate::jail::HostPreparation, cancel_flag: Option<&Arc>, ) -> Result { match config.sandbox { Some(bencher_json::Sandbox::Firecracker) => { #[cfg(target_os = "linux")] { - crate::vm::vm_execute(config, cancel_flag) + crate::vm::vm_execute(config, host, cancel_flag) } #[cfg(not(target_os = "linux"))] { diff --git a/plus/bencher_runner/src/up/job.rs b/plus/bencher_runner/src/up/job.rs index d26f9c463..42fb8ae77 100644 --- a/plus/bencher_runner/src/up/job.rs +++ b/plus/bencher_runner/src/up/job.rs @@ -44,6 +44,7 @@ pub fn execute_job( config: &UpConfig, job: &JsonClaimedJob, ws: &Arc>, + host: &mut crate::jail::HostPreparation, ) -> JobFinishResult { // Only allow jobs with a known sandbox type or explicit opt-in for non-sandboxed. if let Err(reason) = check_sandbox_allowed(job.spec.sandbox, config.allow_no_sandbox) { @@ -101,7 +102,7 @@ pub fn execute_job( job.uuid ); let start = build_time.then(std::time::Instant::now); - let result = crate::execute(&job_config, Some(&cancel_flag)); + let result = crate::execute(&job_config, host, Some(&cancel_flag)); let elapsed = start.map(|s| s.elapsed()); match result { Ok(output) => { diff --git a/plus/bencher_runner/src/up/mod.rs b/plus/bencher_runner/src/up/mod.rs index a84e65649..c4bca4aea 100644 --- a/plus/bencher_runner/src/up/mod.rs +++ b/plus/bencher_runner/src/up/mod.rs @@ -184,6 +184,9 @@ fn run_driver(config: &UpConfig, channel_url: &Url, key: &str) -> Result<(), UpE { println!(" Update channel: {channel}"); } + // Owned by the daemon loop. The latch belongs to this runner process, and + // a failure is deliberately not remembered so the next job retries. + let mut host = crate::jail::HostPreparation::new(); let mut sm = ChannelStateMachine::new(config.poll_timeout_secs, runner_metadata); let mut effects: VecDeque = ChannelStateMachine::initial_effects().into_iter().collect(); @@ -197,7 +200,7 @@ fn run_driver(config: &UpConfig, channel_url: &Url, key: &str) -> Result<(), UpE continue; } - match execute_effect(effect, config, channel_url, key, &mut ws) { + match execute_effect(effect, config, channel_url, key, &mut ws, &mut host) { EffectResult::Continue => {}, EffectResult::Input(input) => { effects.clear(); @@ -236,6 +239,7 @@ fn execute_effect( channel_url: &Url, key: &str, ws: &mut Option>>, + host: &mut crate::jail::HostPreparation, ) -> EffectResult { match effect { Effect::Connect => match JobChannel::connect(channel_url, key) { @@ -264,7 +268,7 @@ fn execute_effect( eprintln!("Error: WS not connected during job execution"); return EffectResult::Input(Input::ConnectionFailed); }; - let result = execute_job(config, &job, ws_ref); + let result = execute_job(config, &job, ws_ref, host); EffectResult::Input(Input::JobFinished(result)) }, Effect::SleepBeforeReconnect(reason) => { diff --git a/plus/bencher_runner/src/vm.rs b/plus/bencher_runner/src/vm.rs index 593365043..165d8717a 100644 --- a/plus/bencher_runner/src/vm.rs +++ b/plus/bencher_runner/src/vm.rs @@ -8,12 +8,15 @@ use std::sync::atomic::AtomicBool; use camino::{Utf8Path, Utf8PathBuf}; use crate::error::RunnerError; -use crate::jail::{JailDir, JailLock, JailPaths, StateDir, chroot, netns, state}; +use crate::jail::{ + HostPreparation, JailDir, JailLock, JailPaths, StateDir, VmId, chroot, netns, state, +}; use crate::run::{RunOutput, prepare_oci_workspace}; /// Execute a single benchmark run in a jailed Firecracker microVM. pub fn vm_execute( config: &crate::Config, + host: &mut HostPreparation, cancel_flag: Option<&Arc>, ) -> Result { use crate::firecracker::run_firecracker; @@ -33,7 +36,7 @@ pub fn vm_execute( // Prepare the host on demand, before the first jail this process builds. // Must come before the lock is taken: preparation takes the same lock, and // `flock` is per open file description, so nesting would block on itself. - crate::jail::prepare_host_once(state_dir.path(), config.jail_user)?; + host.ensure(state_dir.path(), config.jail_user)?; // Held for the whole job. Another runner's sweep removes every chroot it // finds, so it must not run while this one is live. Declared before the @@ -48,7 +51,7 @@ pub fn vm_execute( // built inside it rather than copied in afterwards, so the id is minted // before any of them exist. Dropping this guard removes the chroot tree, // which is what the workspace temp directory used to cover. - let vm_id = uuid::Uuid::new_v4().to_string(); + let vm_id = VmId::new(); let jail_dir = JailDir::create(&state_dir, &vm_id)?; let jail = JailPaths::new(jail_dir.root())?; println!(" Jail: {}", jail.root()); @@ -63,12 +66,14 @@ pub fn vm_execute( // job, or found on the host. let kernel_dest = jail.kernel().host().as_path(); if let Some(kernel) = &config.kernel { - copy_into_jail(kernel, kernel_dest)?; + println!(" Copying the job's kernel into the jail..."); + copy_file(kernel, kernel_dest)?; } else if crate::kernel::KERNEL_BUNDLED { crate::kernel::write_kernel_to_file(kernel_dest)?; - println!(" Extracted bundled kernel to {kernel_dest}"); + println!(" Extracted bundled kernel into the jail at {kernel_dest}"); } else { - copy_into_jail(&find_kernel()?, kernel_dest)?; + println!(" Copying the host's kernel into the jail..."); + copy_file(&find_kernel()?, kernel_dest)?; } let command = oci_config.command; @@ -116,7 +121,7 @@ pub fn vm_execute( fn build_firecracker_config( config: &crate::Config, work_dir: &Utf8Path, - vm_id: String, + vm_id: VmId, state_dir: &StateDir, jail: JailPaths, netns: Utf8PathBuf, @@ -179,17 +184,18 @@ fn build_firecracker_config( }) } -/// Copy a file the jailed VMM has to read into the jail root. +/// Copy a file the job needs to a path the runner controls. /// -/// The rule is uniform: a path that resolves outside the chroot is -/// unreachable once Firecracker is confined, whatever produced it. -fn copy_into_jail(src: &Utf8Path, dest: &Utf8Path) -> Result<(), RunnerError> { +/// Used both for artifacts placed inside the chroot and for binaries staged +/// outside it, so the message says where the copy landed rather than claiming +/// a destination it does not know about. +fn copy_file(src: &Utf8Path, dest: &Utf8Path) -> Result<(), RunnerError> { std::fs::copy(src, dest).map_err(|e| crate::error::ConfigError::CopyFile { src: src.to_owned(), dest: dest.to_owned(), source: e, })?; - println!(" Copied {src} into the jail at {dest}"); + println!(" Copied {src} to {dest}"); Ok(()) } @@ -197,7 +203,7 @@ fn copy_into_jail(src: &Utf8Path, dest: &Utf8Path) -> Result<(), RunnerError> { fn copy_binary(src: &Utf8Path, dest: &Utf8Path) -> Result<(), RunnerError> { use std::os::unix::fs::PermissionsExt as _; - copy_into_jail(src, dest)?; + copy_file(src, dest)?; let mut perms = std::fs::metadata(dest)?.permissions(); perms.set_mode(0o755); std::fs::set_permissions(dest, perms)?; From 6c8d5c4ca53be15970578834a05d462a6ee57d14 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 20:17:57 +0000 Subject: [PATCH 24/91] Wrap the original error instead of formatting it into a string ProcessStart(String) violated the rule that an error variant wraps the error it came from, and it had become a catch-all: a failed spawn, missing stdio, five different serialization failures, and a malformed HTTP response all arrived as the same variant with the cause flattened into text. They are now four variants carrying io::Error, serde_json::Error, and the static context, so a caller can match on what actually happened. The network namespace comment also stops claiming more than the lock delivers. The lock covers the rebuild, not the use: the jailer opens the handle after the lock is released, so a second runner rebuilding it in that window can make the first runner's jailer see ENOENT. Narrow, requires two runners on one host, and it fails the job loudly rather than leaving the VMM on the host network, which is the failure that would matter. Closing it would mean holding a process-global lock across every job, and the comment now says so rather than implying the race is already covered. --- plus/bencher_runner/src/firecracker/client.rs | 31 ++++++++++++------- plus/bencher_runner/src/firecracker/error.rs | 28 +++++++++++++++-- plus/bencher_runner/src/jail/netns.rs | 9 ++++++ 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/client.rs b/plus/bencher_runner/src/firecracker/client.rs index 1296b15b6..c83986faf 100644 --- a/plus/bencher_runner/src/firecracker/client.rs +++ b/plus/bencher_runner/src/firecracker/client.rs @@ -74,8 +74,9 @@ impl FirecrackerClient { /// Configure the machine (vCPUs, memory). pub fn put_machine_config(&self, config: &MachineConfig) -> Result<(), FirecrackerError> { - let body = serde_json::to_string(config).map_err(|e| { - FirecrackerError::ProcessStart(format!("serialize machine config: {e}")) + let body = serde_json::to_string(config).map_err(|e| FirecrackerError::ApiEncoding { + context: "serialize machine config", + source: e, })?; let (status, response_body) = self.http_put("/machine-config", &body)?; if status >= 300 { @@ -89,8 +90,10 @@ impl FirecrackerClient { /// Configure the boot source (kernel and boot args). pub fn put_boot_source(&self, config: &BootSource) -> Result<(), FirecrackerError> { - let body = serde_json::to_string(config) - .map_err(|e| FirecrackerError::ProcessStart(format!("serialize boot source: {e}")))?; + let body = serde_json::to_string(config).map_err(|e| FirecrackerError::ApiEncoding { + context: "serialize boot source", + source: e, + })?; let (status, response_body) = self.http_put("/boot-source", &body)?; if status >= 300 { return Err(FirecrackerError::Api { @@ -103,8 +106,10 @@ impl FirecrackerClient { /// Configure a block device (drive). pub fn put_drive(&self, config: &Drive) -> Result<(), FirecrackerError> { - let body = serde_json::to_string(config) - .map_err(|e| FirecrackerError::ProcessStart(format!("serialize drive: {e}")))?; + let body = serde_json::to_string(config).map_err(|e| FirecrackerError::ApiEncoding { + context: "serialize drive", + source: e, + })?; let path = format!("/drives/{}", config.drive_id); let (status, response_body) = self.http_put(&path, &body)?; if status >= 300 { @@ -118,8 +123,10 @@ impl FirecrackerClient { /// Configure the vsock device. pub fn put_vsock(&self, config: &VsockConfig) -> Result<(), FirecrackerError> { - let body = serde_json::to_string(config) - .map_err(|e| FirecrackerError::ProcessStart(format!("serialize vsock: {e}")))?; + let body = serde_json::to_string(config).map_err(|e| FirecrackerError::ApiEncoding { + context: "serialize vsock", + source: e, + })?; let (status, response_body) = self.http_put("/vsock", &body)?; if status >= 300 { return Err(FirecrackerError::Api { @@ -132,8 +139,10 @@ impl FirecrackerClient { /// Perform a VM action (start, shutdown, etc.). pub fn put_action(&self, action: &Action) -> Result<(), FirecrackerError> { - let body = serde_json::to_string(action) - .map_err(|e| FirecrackerError::ProcessStart(format!("serialize action: {e}")))?; + let body = serde_json::to_string(action).map_err(|e| FirecrackerError::ApiEncoding { + context: "serialize action", + source: e, + })?; let (status, response_body) = self.http_put("/actions", &body)?; if status >= 300 { return Err(FirecrackerError::Api { @@ -269,7 +278,7 @@ fn parse_http_response(data: &[u8]) -> Result<(u16, String), FirecrackerError> { let status_line = response .lines() .next() - .ok_or_else(|| FirecrackerError::ProcessStart("empty HTTP response".to_owned()))?; + .ok_or(FirecrackerError::MalformedResponse("empty HTTP response"))?; let status_code: u16 = status_line .split_whitespace() diff --git a/plus/bencher_runner/src/firecracker/error.rs b/plus/bencher_runner/src/firecracker/error.rs index 721725144..a21632ee7 100644 --- a/plus/bencher_runner/src/firecracker/error.rs +++ b/plus/bencher_runner/src/firecracker/error.rs @@ -5,9 +5,31 @@ use thiserror::Error; /// Errors from the Firecracker integration. #[derive(Debug, Error)] pub enum FirecrackerError { - /// Failed to start the Firecracker process. - #[error("Failed to start Firecracker process: {0}")] - ProcessStart(String), + /// Failed to spawn the jailer that starts the Firecracker process. + #[error("Failed to spawn {path}: {source}")] + Spawn { + /// The binary that could not be spawned. + path: camino::Utf8PathBuf, + /// Why it could not be spawned. + source: std::io::Error, + }, + + /// The spawned process did not provide the stdio the runner asked for. + #[error("Firecracker process stdio unavailable: {0}")] + Stdio(&'static str), + + /// A Firecracker API request or response could not be handled. + #[error("Firecracker API {context}: {source}")] + ApiEncoding { + /// What was being encoded or decoded. + context: &'static str, + /// The underlying serialization failure. + source: serde_json::Error, + }, + + /// The Firecracker API returned a response that could not be parsed. + #[error("Firecracker API response malformed: {0}")] + MalformedResponse(&'static str), /// Firecracker API returned an error. #[error("Firecracker API error: {status} {body}")] diff --git a/plus/bencher_runner/src/jail/netns.rs b/plus/bencher_runner/src/jail/netns.rs index 085b01066..838cab0ba 100644 --- a/plus/bencher_runner/src/jail/netns.rs +++ b/plus/bencher_runner/src/jail/netns.rs @@ -72,6 +72,15 @@ pub fn handle_path() -> Utf8PathBuf { /// so this takes its own lock: two runners started with different /// `--state-dir` values hold different jail locks and would otherwise clear /// and rebind the same handle concurrently. +/// +/// The lock covers the rebuild, not the use. The jailer opens the handle +/// itself, after this returns and after the lock is released, so a second +/// runner rebuilding the handle in that window can make the first runner's +/// jailer see `ENOENT` between the unlink and the bind. That is narrow, it +/// requires two runners on one host, and it fails the job loudly rather than +/// silently leaving the VMM on the host network, which is the failure that +/// would matter. Holding the lock until the VMM has started would close it, +/// at the cost of serializing every job on a process-global lock. pub fn ensure() -> Result { let handle = handle_path(); From 2de2ccc673f383e760a6004c3aa28e9d6b4c4d14 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 20:18:07 +0000 Subject: [PATCH 25/91] Give Scenario a Default, and split the changelog entry Every scenario spelled out four fields it did not vary, so adding one field meant touching all seventy-eight. They now build with ..Scenario::default() and name only what they change, which removes 204 lines and makes the next field free. The collapse shrank nosandbox_scenarios enough that its too_many_lines exemption is no longer needed. The changelog entry is three bullets rather than one. The root requirement is the part that breaks existing deployments, and it was buried mid-paragraph behind the jail description; it now leads on its own, with the confinement and the state directory following as the non-breaking changes they are. --- .../docs-reference/changelog/en/changelog.mdx | 4 +- tasks/test_runner/src/task/scenarios.rs | 406 +++++------------- 2 files changed, 104 insertions(+), 306 deletions(-) diff --git a/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx b/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx index 1d6fed892..ef92101f6 100644 --- a/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx +++ b/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx @@ -1,5 +1,7 @@ ## `v0.6.11` -- **BREAKING CHANGE** Confine the Firecracker VMM on Bare Metal Runners with the [Firecracker jailer](https://github.com/firecracker-microvm/firecracker/blob/main/docs/jailer.md): each sandboxed Job now runs in a chroot as a dedicated unprivileged user (`61016` by default, settable with `--jail-uid`/`--jail-gid`), in an empty network namespace, and is placed in its cgroup before it starts. A Runner that executes sandboxed Jobs must now run as **root**, where before it did not: building the jail requires `mknod` for the chroot's `/dev/kvm`, `chown` to hand the guest images to the jail user, `pivot_root`, and `setns` to join the network namespace. A world-readable `/dev/kvm` is enough to *use* KVM unprivileged but not to build the jail around it. Run `runner up` as root, or, if you cannot, start it with `--danger-allow-no-sandbox` and assign it only Specs with no Sandbox; be aware that this trades away the microVM itself and not just the jail, so those Jobs execute directly on the host. Runners also gain a persistent state directory (`--state-dir`, default `/var/lib/bencher-runner`) holding each Job's chroot, which is swept of anything a previous Runner left behind +- **BREAKING CHANGE** A Bare Metal Runner that executes sandboxed Jobs must now run as `root`, where before it did not. Confining the Firecracker VMM requires privileges that enabling KVM does not grant: `mknod` to create the sandbox's `/dev/kvm`, `chown` to hand the guest images to the sandbox user, `pivot_root`, and `setns` to join an empty network namespace. A world-readable `/dev/kvm` is enough to *use* KVM unprivileged but not to build the sandbox around it. Start `runner up` as `root`, or as a service that starts it as `root`. If you cannot run as `root`, start the Runner with `--danger-allow-no-sandbox` and assign it only Specs with no Sandbox; be aware that this gives up the microVM itself and not just its confinement, so those Jobs execute directly on the host +- Confine the Firecracker VMM on Bare Metal Runners with the [Firecracker jailer](https://github.com/firecracker-microvm/firecracker/blob/main/docs/jailer.md): each sandboxed Job now runs in a chroot as a dedicated unprivileged user, in an empty network namespace, and is placed in its cgroup before it starts, so a VMM escape no longer lands as `root` on the Runner host holding the Runner key and every prior Job's work. The user defaults to `61016` and is settable with `--jail-uid`/`--jail-gid` +- Add `--state-dir` to the Runner (default `/var/lib/bencher-runner`), a persistent directory holding each sandboxed Job's chroot. It is swept of anything a previous Runner left behind, since a Runner killed mid-Job cannot clean up after itself - Fix the percent difference shown in the report table when a metric drops to zero; a value of `0.00` against a positive baseline now shows `-100.00%` instead of `0.00%` in both the Console and the CI comment (Thank you [@OmarTawfik](https://github.com/OmarTawfik)) - **BREAKING CHANGE** Name the GitHub Check after the Project (ie `Bencher Report ()`) so runs for different Projects on the same commit no longer overwrite each other; `--ci-id` takes the Project name's place when set, the in-progress Check falls back to `Bencher Report` until the results are posted when `--project` is not set, and the name is truncated to 255 bytes. Update any required status check named `Bencher Report` in branch protection to the new name, and note that renaming a Project changes the Check name too (Thank you [@OmarTawfik](https://github.com/OmarTawfik)) diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index f8abcf2f9..56e1e3bd6 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -47,6 +47,10 @@ fn extract_json_substr(line: &str) -> &str { type Probe = fn(&Utf8Path) -> Result; /// Test scenario definition. +/// +/// Build one with `..Scenario::default()` so a scenario names only what it +/// actually varies, and so adding a field does not have to be written out +/// across every scenario in this file. struct Scenario { name: &'static str, description: &'static str, @@ -67,6 +71,25 @@ struct Scenario { validate: fn(&ScenarioOutput) -> Result<()>, } +impl Default for Scenario { + fn default() -> Self { + Self { + name: "", + description: "", + dockerfile: "", + extra_args: &[], + cancel_after_secs: None, + probe: None, + orphan_then_rerun: false, + // Sandboxed is the interesting case and the overwhelming majority, + // so the handful of non-sandboxed scenarios opt out rather than + // every other scenario opting in. + sandboxed: true, + validate: |_output| Ok(()), + } + } +} + /// Output from running a scenario. #[derive(Debug)] struct ScenarioOutput { @@ -275,10 +298,6 @@ fn all_scenarios() -> Vec { description: "Simple echo command", dockerfile: r#"FROM busybox CMD ["echo", "hello from vm"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("hello from vm") { @@ -287,6 +306,7 @@ CMD ["echo", "hello from vm"]"#, bail!("Expected 'hello from vm' in output, got: {}", output.stdout) } }, + ..Scenario::default() }, Scenario { name: "environment_variables", @@ -294,10 +314,6 @@ CMD ["echo", "hello from vm"]"#, dockerfile: r#"FROM busybox ENV MY_VAR=test_value CMD ["sh", "-c", "echo $MY_VAR"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("test_value") { @@ -311,6 +327,7 @@ CMD ["sh", "-c", "echo $MY_VAR"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "working_directory", @@ -318,10 +335,6 @@ CMD ["sh", "-c", "echo $MY_VAR"]"#, dockerfile: r#"FROM busybox WORKDIR /myapp CMD ["pwd"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("/myapp") { @@ -330,16 +343,13 @@ CMD ["pwd"]"#, bail!("Expected '/myapp' in output, got: {}", output.stdout) } }, + ..Scenario::default() }, Scenario { name: "file_output", description: "Output file collection via vsock", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo '{\"result\": 42}' > /tmp/output.json && cat /tmp/output.json"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--output", "/tmp/output.json"], validate: |output| { if output.stdout.contains("\"result\"") || output.stdout.contains("42") { @@ -348,16 +358,13 @@ CMD ["sh", "-c", "echo '{\"result\": 42}' > /tmp/output.json && cat /tmp/output. bail!("Expected JSON output, got: {}", output.stdout) } }, + ..Scenario::default() }, Scenario { name: "exit_code", description: "Non-zero exit codes captured", dockerfile: r#"FROM busybox CMD ["sh", "-c", "exit 42"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -367,16 +374,13 @@ CMD ["sh", "-c", "exit 42"]"#, bail!("Expected exit code 42 in output") } }, + ..Scenario::default() }, Scenario { name: "timeout_handling", description: "VM killed after timeout", dockerfile: r#"FROM busybox CMD ["sleep", "3600"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr).to_lowercase(); @@ -386,16 +390,13 @@ CMD ["sleep", "3600"]"#, bail!("Expected timeout error") } }, + ..Scenario::default() }, Scenario { name: "writable_filesystem", description: "Guest can write to ext4 rootfs", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo test > /data.txt && cat /data.txt"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("test") { @@ -407,16 +408,13 @@ CMD ["sh", "-c", "echo test > /data.txt && cat /data.txt"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "stderr_capture", description: "Stderr captured separately", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo stdout && echo stderr >&2"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -426,16 +424,13 @@ CMD ["sh", "-c", "echo stdout && echo stderr >&2"]"#, bail!("Expected 'stdout' in output") } }, + ..Scenario::default() }, Scenario { name: "multi_cpu", description: "Multiple vCPUs work (expected: timeout, SMP boot unsupported)", dockerfile: r#"FROM busybox CMD ["sh", "-c", "cat /proc/cpuinfo | grep processor | wc -l"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "10", "--vcpus", "4"], validate: |output| { // SMP boot is not yet supported (requires LAPIC/APIC emulation). @@ -454,6 +449,7 @@ CMD ["sh", "-c", "cat /proc/cpuinfo | grep processor | wc -l"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "entrypoint_with_args", @@ -461,10 +457,6 @@ CMD ["sh", "-c", "cat /proc/cpuinfo | grep processor | wc -l"]"#, dockerfile: r#"FROM busybox ENTRYPOINT ["echo"] CMD ["hello", "world"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("hello world") { @@ -473,16 +465,13 @@ CMD ["hello", "world"]"#, bail!("Expected 'hello world' in output, got: {}", output.stdout) } }, + ..Scenario::default() }, Scenario { name: "no_network_access", description: "Guest has no network", dockerfile: r#"FROM busybox CMD ["sh", "-c", "ping -c 1 -W 1 8.8.8.8 2>&1 || echo no_network"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -495,6 +484,7 @@ CMD ["sh", "-c", "ping -c 1 -W 1 8.8.8.8 2>&1 || echo no_network"]"#, bail!("Expected network failure, got: {combined}") } }, + ..Scenario::default() }, // ======================================================================= // Security hardening scenarios @@ -505,10 +495,6 @@ CMD ["sh", "-c", "ping -c 1 -W 1 8.8.8.8 2>&1 || echo no_network"]"#, // Generate ~20MB of output - should be truncated to the 10MB limit dockerfile: r#"FROM busybox CMD ["sh", "-c", "dd if=/dev/zero bs=1M count=20 2>/dev/null | tr '\\0' 'A' && echo DONE"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "120", "--max-output-size", "10485760"], validate: |output| { // The key test: the runner completes without OOM and output is bounded. @@ -522,6 +508,7 @@ CMD ["sh", "-c", "dd if=/dev/zero bs=1M count=20 2>/dev/null | tr '\\0' 'A' && e // Runner completed (didn't hang or OOM) - that's a pass Ok(()) }, + ..Scenario::default() }, Scenario { name: "timeout_enforced", @@ -529,10 +516,6 @@ CMD ["sh", "-c", "dd if=/dev/zero bs=1M count=20 2>/dev/null | tr '\\0' 'A' && e // This process ignores signals and runs forever dockerfile: r#"FROM busybox CMD ["sh", "-c", "trap '' TERM INT; echo started; while true; do sleep 1; done"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { // The VM should be killed after 5 seconds due to timeout @@ -548,6 +531,7 @@ CMD ["sh", "-c", "trap '' TERM INT; echo started; while true; do sleep 1; done"] ) } }, + ..Scenario::default() }, // ======================================================================= // Error regression scenarios @@ -562,10 +546,6 @@ CMD ["sh", "-c", "trap '' TERM INT; echo started; while true; do sleep 1; done"] // causing uid_map writes to fail with EPERM. dockerfile: r#"FROM busybox CMD ["id"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // The runner should not fail with uid_map errors. @@ -581,6 +561,7 @@ CMD ["id"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "dev_kvm_available", @@ -590,10 +571,6 @@ CMD ["id"]"#, // the bind-mounted /dev/kvm. dockerfile: r#"FROM busybox CMD ["echo", "kvm_test_ok"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -605,6 +582,7 @@ CMD ["echo", "kvm_test_ok"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "proc_mount_works", @@ -614,10 +592,6 @@ CMD ["echo", "kvm_test_ok"]"#, // which we fixed by bind-mounting the host's /proc instead. dockerfile: r#"FROM busybox CMD ["cat", "/proc/version"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -631,6 +605,7 @@ CMD ["cat", "/proc/version"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "rootfs_writable", @@ -640,10 +615,6 @@ CMD ["cat", "/proc/version"]"#, // when trying to write to the filesystem. dockerfile: r#"FROM busybox CMD ["sh", "-c", "touch /tmp/write_test && echo write_ok"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("write_ok") { @@ -658,6 +629,7 @@ CMD ["sh", "-c", "touch /tmp/write_test && echo write_ok"]"#, bail!("Expected 'write_ok' in output, got: {combined}") } }, + ..Scenario::default() }, Scenario { name: "timeout_includes_partial_output", @@ -667,10 +639,6 @@ CMD ["sh", "-c", "touch /tmp/write_test && echo write_ok"]"#, // short-circuited before serial output extraction. dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo partial_output_marker && sleep 3600"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "10"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -685,6 +653,7 @@ CMD ["sh", "-c", "echo partial_output_marker && sleep 3600"]"#, bail!("Expected timeout error in output, got: {combined}") } }, + ..Scenario::default() }, Scenario { name: "no_seccomp_sigsys", @@ -695,10 +664,6 @@ CMD ["sh", "-c", "echo partial_output_marker && sleep 3600"]"#, // This scenario exercises the timeout path which requires kill(). dockerfile: r#"FROM busybox CMD ["sleep", "3600"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { // SIGSYS from seccomp violation produces exit code 159 (128 + 31) @@ -716,6 +681,7 @@ CMD ["sleep", "3600"]"#, bail!("Expected timeout exit, got exit_code={}", output.exit_code) } }, + ..Scenario::default() }, Scenario { name: "iopl_dropped_before_exec", @@ -730,10 +696,6 @@ RUN printf '#include \n#include \n#include \nstatic FROM busybox COPY --from=build /test_iopl /test_iopl CMD ["/test_iopl"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("IOPL_DROPPED") { @@ -752,6 +714,7 @@ CMD ["/test_iopl"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "unique_output_validation", @@ -761,10 +724,6 @@ CMD ["/test_iopl"]"#, // never appear in runner logs. dockerfile: r#"FROM busybox CMD ["echo", "UNIQUE_VM_OUTPUT_a7f3b2c9"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // This unique string should only appear if the VM actually ran @@ -780,6 +739,7 @@ CMD ["echo", "UNIQUE_VM_OUTPUT_a7f3b2c9"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // PID namespace isolation scenarios (Item 9) @@ -791,10 +751,6 @@ CMD ["echo", "UNIQUE_VM_OUTPUT_a7f3b2c9"]"#, // The init process should be PID 1, and there should be very few processes. dockerfile: r#"FROM busybox CMD ["sh", "-c", "ls /proc | grep -E '^[0-9]+$' | wc -l"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // The guest should see a small number of PIDs (1-5), not hundreds @@ -816,6 +772,7 @@ CMD ["sh", "-c", "ls /proc | grep -E '^[0-9]+$' | wc -l"]"#, Ok(()) } }, + ..Scenario::default() }, Scenario { name: "pid_namespace_procfs", @@ -825,10 +782,6 @@ CMD ["sh", "-c", "ls /proc | grep -E '^[0-9]+$' | wc -l"]"#, // should be accessible and /proc/1/cmdline should show the init process. dockerfile: r#"FROM busybox CMD ["sh", "-c", "cat /proc/version && echo PID1=$(cat /proc/1/cmdline | tr '\\0' ' ')"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -844,6 +797,7 @@ CMD ["sh", "-c", "cat /proc/version && echo PID1=$(cat /proc/1/cmdline | tr '\\0 ) } }, + ..Scenario::default() }, // ======================================================================= // Telemetry/Metrics scenarios (Item 10) @@ -854,10 +808,6 @@ CMD ["sh", "-c", "cat /proc/version && echo PID1=$(cat /proc/1/cmdline | tr '\\0 // Verifies the runner outputs ---BENCHER_METRICS:{json}--- on stderr. dockerfile: r#"FROM busybox CMD ["echo", "metrics_test"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stderr.contains("---BENCHER_METRICS:") && output.stderr.contains("---") { @@ -870,6 +820,7 @@ CMD ["echo", "metrics_test"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "metrics_wall_clock_reasonable", @@ -878,10 +829,6 @@ CMD ["echo", "metrics_test"]"#, // This catches cases where timing is broken (e.g., always 0 or absurdly large). dockerfile: r#"FROM busybox CMD ["echo", "fast_benchmark"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // Parse metrics from stderr @@ -910,6 +857,7 @@ CMD ["echo", "fast_benchmark"]"#, } bail!("Could not parse wall_clock_ms from metrics: {json_str}") }, + ..Scenario::default() }, Scenario { name: "metrics_timeout_flag", @@ -917,10 +865,6 @@ CMD ["echo", "fast_benchmark"]"#, // When a VM times out, the metrics should include timed_out: true. dockerfile: r#"FROM busybox CMD ["sleep", "3600"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { // The stderr should contain metrics with timed_out: true @@ -946,6 +890,7 @@ CMD ["sleep", "3600"]"#, } bail!("Expected timed_out: true in metrics: {json_str}") }, + ..Scenario::default() }, // ======================================================================= // HMAC Result Integrity scenarios (Item 11) @@ -957,10 +902,6 @@ CMD ["sleep", "3600"]"#, // The vmm child process should log [HMAC] status on stderr. dockerfile: r#"FROM busybox CMD ["echo", "hmac_test_output"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -984,6 +925,7 @@ CMD ["echo", "hmac_test_output"]"#, } } }, + ..Scenario::default() }, Scenario { name: "metrics_transport_type", @@ -991,10 +933,6 @@ CMD ["echo", "hmac_test_output"]"#, // Verifies the metrics include the transport type (vsock or serial). dockerfile: r#"FROM busybox CMD ["echo", "transport_test"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { let metrics_line = output @@ -1016,6 +954,7 @@ CMD ["echo", "transport_test"]"#, } bail!("Could not find transport in metrics: {json_str}") }, + ..Scenario::default() }, // ======================================================================= // Cancellation scenarios @@ -1028,9 +967,6 @@ CMD ["echo", "transport_test"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo started && sleep 3600"]"#, cancel_after_secs: Some(5), - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { // The runner should exit with a non-zero code (killed by signal) @@ -1046,6 +982,7 @@ CMD ["sh", "-c", "echo started && sleep 3600"]"#, } Ok(()) }, + ..Scenario::default() }, // ======================================================================= // Output edge-case scenarios @@ -1055,10 +992,6 @@ CMD ["sh", "-c", "echo started && sleep 3600"]"#, description: "Stderr captured when stdout is empty", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo error_output >&2"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stderr.contains("error_output") { @@ -1071,16 +1004,13 @@ CMD ["sh", "-c", "echo error_output >&2"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "empty_output", description: "Process exits 0 with no output", dockerfile: r#"FROM busybox CMD ["true"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1093,6 +1023,7 @@ CMD ["true"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "binary_output", @@ -1101,10 +1032,6 @@ CMD ["true"]"#, // The runner should not panic — it should lossy-convert or pass through. dockerfile: r#"FROM busybox CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // The runner must not crash. Exit code 0 and "done" somewhere @@ -1126,6 +1053,7 @@ CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // OCI config parsing scenarios @@ -1137,10 +1065,6 @@ CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, // OCI config stores this as ["/bin/sh", "-c", "echo shell_form_works"] // which differs from exec form ["echo", "shell_form_works"]. dockerfile: "FROM busybox\nCMD echo shell_form_works", - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("shell_form_works") { @@ -1154,6 +1078,7 @@ CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "entrypoint_only", @@ -1162,10 +1087,6 @@ CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, // CMD args appended. The runner must not fail when Cmd is null/empty. dockerfile: r#"FROM busybox ENTRYPOINT ["echo", "entrypoint_only_works"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("entrypoint_only_works") { @@ -1179,6 +1100,7 @@ ENTRYPOINT ["echo", "entrypoint_only_works"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "shell_form_entrypoint", @@ -1186,10 +1108,6 @@ ENTRYPOINT ["echo", "entrypoint_only_works"]"#, // Shell form ENTRYPOINT: stored as ["/bin/sh", "-c", "echo ..."] // in OCI config. CMD is ignored when ENTRYPOINT uses shell form. dockerfile: "FROM busybox\nENTRYPOINT echo shell_entrypoint_works", - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("shell_entrypoint_works") { @@ -1203,6 +1121,7 @@ ENTRYPOINT ["echo", "entrypoint_only_works"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "entrypoint_shell_with_cmd", @@ -1217,10 +1136,6 @@ ENTRYPOINT ["echo", "entrypoint_only_works"]"#, dockerfile: r#"FROM busybox ENTRYPOINT echo ep_marker CMD ["cmd_arg"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1240,6 +1155,7 @@ CMD ["cmd_arg"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "no_cmd_no_entrypoint", @@ -1248,10 +1164,6 @@ CMD ["cmd_arg"]"#, // to fail with a clear error, not crash or hang. dockerfile: r#"FROM busybox RUN echo "no command set""#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "30"], validate: |output| { // The runner should fail (non-zero exit) since there's nothing to run. @@ -1266,6 +1178,7 @@ RUN echo "no command set""#, ) } }, + ..Scenario::default() }, Scenario { name: "bencher_cli_mock", @@ -1277,10 +1190,6 @@ RUN echo "no command set""#, // shared libraries, and ld.so.cache from multi-layer images. dockerfile: r#"FROM ghcr.io/bencherdev/bencher:latest CMD ["mock"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { if output.exit_code == 127 { @@ -1306,6 +1215,7 @@ CMD ["mock"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "distroless_glibc_image", @@ -1322,10 +1232,6 @@ RUN echo '#include \nint main(){printf("distroless_glibc_ok\\n");return FROM gcr.io/distroless/cc-debian12 COPY --from=builder /tmp/hello /usr/bin/hello CMD ["/usr/bin/hello"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { if output.exit_code == 127 { @@ -1350,6 +1256,7 @@ CMD ["/usr/bin/hello"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // Race condition scenarios @@ -1362,10 +1269,6 @@ CMD ["/usr/bin/hello"]"#, // results are collected even for very short-lived processes. dockerfile: r#"FROM busybox CMD ["echo", "rapid_exit_marker"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1386,6 +1289,7 @@ CMD ["echo", "rapid_exit_marker"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // Exit code scenarios @@ -1397,10 +1301,6 @@ CMD ["echo", "rapid_exit_marker"]"#, // The runner should capture and report this exit code. dockerfile: r#"FROM busybox CMD ["sh", "-c", "exit 137"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // The runner should report exit code 137 somewhere in its output, @@ -1417,6 +1317,7 @@ CMD ["sh", "-c", "exit 137"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // Environment scenarios @@ -1431,10 +1332,6 @@ ENV A1=val1 A2=val2 A3=val3 A4=val4 A5=val5 A6=val6 A7=val7 A8=val8 A9=val9 A10= ENV B1=val11 B2=val12 B3=val13 B4=val14 B5=val15 B6=val16 B7=val17 B8=val18 B9=val19 B10=val20 ENV LARGE_VALUE=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA CMD ["sh", "-c", "echo A1=$A1 B10=$B10 LARGE_LEN=${#LARGE_VALUE}"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1454,6 +1351,7 @@ CMD ["sh", "-c", "echo A1=$A1 B10=$B10 LARGE_LEN=${#LARGE_VALUE}"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // File output edge cases @@ -1465,10 +1363,6 @@ CMD ["sh", "-c", "echo A1=$A1 B10=$B10 LARGE_LEN=${#LARGE_VALUE}"]"#, // The runner should still succeed (exit 0) without crashing. dockerfile: r#"FROM busybox CMD ["echo", "no file written"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--output", "/nonexistent/path.json"], validate: |output| { // Runner should not crash, regardless of exit code. @@ -1479,16 +1373,13 @@ CMD ["echo", "no file written"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "large_file_output", description: "Large output file (~2 MB) transferred via vsock", dockerfile: r#"FROM busybox CMD ["sh", "-c", "dd if=/dev/urandom bs=1024 count=2048 2>/dev/null | base64 > /tmp/output.json && echo done"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--output", "/tmp/output.json"], validate: |output| { if output.exit_code != 0 { @@ -1497,16 +1388,13 @@ CMD ["sh", "-c", "dd if=/dev/urandom bs=1024 count=2048 2>/dev/null | base64 > / } Ok(()) }, + ..Scenario::default() }, Scenario { name: "completed_with_all_fields", description: "Stdout + stderr + output file simultaneously", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo stdout_marker && echo stderr_marker >&2 && echo '{\"data\":true}' > /tmp/out.json"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--output", "/tmp/out.json"], validate: |output| { if output.exit_code != 0 { @@ -1521,16 +1409,13 @@ CMD ["sh", "-c", "echo stdout_marker && echo stderr_marker >&2 && echo '{\"data\ } Ok(()) }, + ..Scenario::default() }, Scenario { name: "multi_file_output", description: "Multiple output files collected via vsock", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo '{\"result\": 1}' > /tmp/a.json && echo '{\"result\": 2}' > /tmp/b.json && echo done"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &[ "--timeout", "60", @@ -1546,6 +1431,7 @@ CMD ["sh", "-c", "echo '{\"result\": 1}' > /tmp/a.json && echo '{\"result\": 2}' } Ok(()) }, + ..Scenario::default() }, // ======================================================================= // OCI image variations @@ -1558,10 +1444,6 @@ RUN echo "a" > /tmp/file_a.txt RUN mkdir -p /opt && echo "b" > /opt/file_b.txt RUN echo "c" > /var/file_c.txt CMD ["sh", "-c", "cat /tmp/file_a.txt /opt/file_b.txt /var/file_c.txt"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1577,6 +1459,7 @@ CMD ["sh", "-c", "cat /tmp/file_a.txt /opt/file_b.txt /var/file_c.txt"]"#, bail!("Expected 'a', 'b', 'c' in output, got: {}", output.stdout) } }, + ..Scenario::default() }, Scenario { name: "image_with_symlinks", @@ -1584,10 +1467,6 @@ CMD ["sh", "-c", "cat /tmp/file_a.txt /opt/file_b.txt /var/file_c.txt"]"#, dockerfile: r#"FROM busybox RUN echo "target" > /tmp/target.txt && ln -s /tmp/target.txt /tmp/link.txt CMD ["cat", "/tmp/link.txt"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1603,6 +1482,7 @@ CMD ["cat", "/tmp/link.txt"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // Error / edge case scenarios @@ -1612,10 +1492,6 @@ CMD ["cat", "/tmp/link.txt"]"#, description: "Writes stdout+stderr then exits non-zero", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo partial_stdout && echo partial_stderr >&2 && exit 1"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // The runner may succeed (exit 0) even when the guest exits non-zero. @@ -1627,16 +1503,13 @@ CMD ["sh", "-c", "echo partial_stdout && echo partial_stderr >&2 && exit 1"]"#, bail!("Expected partial output to be captured, got: {combined}") } }, + ..Scenario::default() }, Scenario { name: "minimum_timeout", description: "1-second timeout kills long-running process", dockerfile: r#"FROM busybox CMD ["sleep", "3600"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "1"], validate: |output| { if output.exit_code == 0 { @@ -1644,6 +1517,7 @@ CMD ["sleep", "3600"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "max_output_size_truncation", @@ -1651,10 +1525,6 @@ CMD ["sleep", "3600"]"#, // Generate ~50 KB of output, but limit to 1024 bytes. dockerfile: r#"FROM busybox CMD ["sh", "-c", "dd if=/dev/zero bs=1024 count=50 2>/dev/null | tr '\\0' 'X'"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--max-output-size", "1024"], validate: |output| { // Output should be bounded — not the full ~50KB @@ -1667,6 +1537,7 @@ CMD ["sh", "-c", "dd if=/dev/zero bs=1024 count=50 2>/dev/null | tr '\\0' 'X'"]" // Runner didn't OOM or crash — that's a pass Ok(()) }, + ..Scenario::default() }, Scenario { name: "env_var_passthrough", @@ -1676,10 +1547,6 @@ ENV LD_PRELOAD=/test.so ENV LD_LIBRARY_PATH=/testlib ENV SAFE_VAR=safe_value CMD ["sh", "-c", "echo LD_PRELOAD=$LD_PRELOAD LD_LIBRARY_PATH=$LD_LIBRARY_PATH SAFE=$SAFE_VAR"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1706,6 +1573,7 @@ CMD ["sh", "-c", "echo LD_PRELOAD=$LD_PRELOAD LD_LIBRARY_PATH=$LD_LIBRARY_PATH S } Ok(()) }, + ..Scenario::default() }, // ======================================================================= // Resource constraint enforcement @@ -1717,10 +1585,6 @@ CMD ["sh", "-c", "echo LD_PRELOAD=$LD_PRELOAD LD_LIBRARY_PATH=$LD_LIBRARY_PATH S // `free -m` reports total memory; we check it's in the right ballpark. dockerfile: r#"FROM busybox CMD ["free", "-m"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--memory", "64", "--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1736,6 +1600,7 @@ CMD ["free", "-m"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "disk_size_override", @@ -1746,10 +1611,6 @@ CMD ["free", "-m"]"#, // the block device level. This test validates the config path. dockerfile: r#"FROM busybox CMD ["sh", "-c", "df -m / | tail -1 | awk '{print $2}'"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--disk", "64", "--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1766,6 +1627,7 @@ CMD ["sh", "-c", "df -m / | tail -1 | awk '{print $2}'"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "disk_limit_enforced", @@ -1775,10 +1637,6 @@ CMD ["sh", "-c", "df -m / | tail -1 | awk '{print $2}'"]"#, // approximately 64 MiB total (minus overhead), not more. dockerfile: r#"FROM busybox CMD ["sh", "-c", "df -m / | tail -1 | awk '{print \"TOTAL_MB=\" $2}'"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--disk", "64", "--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1803,16 +1661,13 @@ CMD ["sh", "-c", "df -m / | tail -1 | awk '{print \"TOTAL_MB=\" $2}'"]"#, output.stdout ) }, + ..Scenario::default() }, Scenario { name: "cpu_count_visible", description: "Guest sees 1 CPU with default vCPU count", dockerfile: r#"FROM busybox CMD ["nproc"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1825,6 +1680,7 @@ CMD ["nproc"]"#, bail!("Expected '1' CPU from nproc, got: {}", output.stdout) } }, + ..Scenario::default() }, // ======================================================================= // Network enabled @@ -1836,10 +1692,6 @@ CMD ["nproc"]"#, // Use wget to a well-known URL as a connectivity test. dockerfile: r#"FROM busybox CMD ["sh", "-c", "wget -q -O /dev/null http://detectportal.firefox.com/success.txt && echo net_ok || echo net_fail"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "30", "--network"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -1856,6 +1708,7 @@ CMD ["sh", "-c", "wget -q -O /dev/null http://detectportal.firefox.com/success.t Ok(()) } }, + ..Scenario::default() }, // ======================================================================= // File permissions @@ -1868,10 +1721,6 @@ CMD ["sh", "-c", "wget -q -O /dev/null http://detectportal.firefox.com/success.t dockerfile: r#"FROM busybox RUN mkdir -p /data && echo "content_ok" > /data/file.txt CMD ["cat", "/data/file.txt"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1884,6 +1733,7 @@ CMD ["cat", "/data/file.txt"]"#, bail!("Expected 'content_ok' in output, got: {}", output.stdout) } }, + ..Scenario::default() }, Scenario { name: "file_permissions_preserved", @@ -1893,10 +1743,6 @@ CMD ["cat", "/data/file.txt"]"#, dockerfile: r#"FROM busybox RUN mkdir -p /data && printf '#!/bin/sh\necho hello' > /data/test.sh && chmod +x /data/test.sh CMD ["sh", "-c", "test -x /data/test.sh && echo perm_ok"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1912,6 +1758,7 @@ CMD ["sh", "-c", "test -x /data/test.sh && echo perm_ok"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "directory_permissions_preserved", @@ -1921,10 +1768,6 @@ CMD ["sh", "-c", "test -x /data/test.sh && echo perm_ok"]"#, dockerfile: r#"FROM busybox RUN mkdir -p /data/restricted && chmod 750 /data/restricted CMD ["stat", "-c", "%a", "/data/restricted"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1940,6 +1783,7 @@ CMD ["stat", "-c", "%a", "/data/restricted"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // Special characters in environment variables @@ -1949,10 +1793,6 @@ CMD ["stat", "-c", "%a", "/data/restricted"]"#, description: "Env vars with spaces, equals, and quotes work", // Use Docker's multi-line ENV syntax with quotes for values with spaces. dockerfile: "FROM busybox\nENV SPACED=\"hello world\" WITH_EQ=\"key=value\"\nCMD [\"sh\", \"-c\", \"echo SPACED=$SPACED EQ=$WITH_EQ\"]", - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1970,6 +1810,7 @@ CMD ["stat", "-c", "%a", "/data/restricted"]"#, } Ok(()) }, + ..Scenario::default() }, // ======================================================================= // CLI override scenarios (--entrypoint, --cmd, --env) @@ -1980,10 +1821,6 @@ CMD ["stat", "-c", "%a", "/data/restricted"]"#, dockerfile: r#"FROM busybox ENTRYPOINT ["echo", "image_ep"] CMD ["image_cmd"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--entrypoint", "echo", "cli_ep"], validate: |output| { if output.exit_code != 0 { @@ -2012,6 +1849,7 @@ CMD ["image_cmd"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "cli_cmd_override", @@ -2019,10 +1857,6 @@ CMD ["image_cmd"]"#, dockerfile: r#"FROM busybox ENTRYPOINT ["echo"] CMD ["image_cmd"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--cmd", "cli_cmd"], validate: |output| { if output.exit_code != 0 { @@ -2044,6 +1878,7 @@ CMD ["image_cmd"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "cli_entrypoint_and_cmd_override", @@ -2051,10 +1886,6 @@ CMD ["image_cmd"]"#, dockerfile: r#"FROM busybox ENTRYPOINT ["echo", "image_ep"] CMD ["image_cmd"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &[ "--timeout", "60", @@ -2083,6 +1914,7 @@ CMD ["image_cmd"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "cli_env_override", @@ -2090,10 +1922,6 @@ CMD ["image_cmd"]"#, dockerfile: r#"FROM busybox ENV MY_VAR=image_value CMD ["sh", "-c", "echo MY_VAR=$MY_VAR"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--env", "MY_VAR=cli_value"], validate: |output| { if output.exit_code != 0 { @@ -2110,6 +1938,7 @@ CMD ["sh", "-c", "echo MY_VAR=$MY_VAR"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "cli_env_add", @@ -2117,10 +1946,6 @@ CMD ["sh", "-c", "echo MY_VAR=$MY_VAR"]"#, dockerfile: r#"FROM busybox ENV EXISTING=from_image CMD ["sh", "-c", "echo EXISTING=$EXISTING NEW=$NEW_VAR"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--env", "NEW_VAR=from_cli"], validate: |output| { if output.exit_code != 0 { @@ -2141,16 +1966,13 @@ CMD ["sh", "-c", "echo EXISTING=$EXISTING NEW=$NEW_VAR"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "cli_env_multiple", description: "Multiple --env flags", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo A=$A B=$B"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--env", "A=one", "--env", "B=two"], validate: |output| { if output.exit_code != 0 { @@ -2165,16 +1987,13 @@ CMD ["sh", "-c", "echo A=$A B=$B"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "cli_entrypoint_no_image_entrypoint", description: "Add entrypoint when image only has CMD", dockerfile: r#"FROM busybox CMD ["hello", "world"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--entrypoint", "echo"], validate: |output| { if output.exit_code != 0 { @@ -2192,16 +2011,13 @@ CMD ["hello", "world"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "multiple_iterations", description: "Multiple iterations execute sequentially", dockerfile: r#"FROM busybox CMD ["echo", "iter_output"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--iter", "3"], validate: |output| { if output.exit_code != 0 { @@ -2214,16 +2030,13 @@ CMD ["echo", "iter_output"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "zero_iterations", description: "Zero iterations executes no benchmarks", dockerfile: r#"FROM busybox CMD ["echo", "should_not_appear"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--iter", "0"], validate: |output| { if output.exit_code != 0 { @@ -2234,16 +2047,13 @@ CMD ["echo", "should_not_appear"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "allow_failure_false_aborts", description: "Non-zero exit code aborts iteration without --allow-failure", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--iter", "3"], validate: |output| { if output.exit_code == 0 { @@ -2262,16 +2072,13 @@ CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "allow_failure_true_continues", description: "Non-zero exit code continues with --allow-failure", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "60", "--iter", "3", "--allow-failure"], validate: |output| { if output.exit_code != 0 { @@ -2290,6 +2097,7 @@ CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, } Ok(()) }, + ..Scenario::default() }, ] } @@ -2298,10 +2106,6 @@ CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, /// /// These test the `local_execute` code path (no Firecracker VM). /// The OCI image is unpacked and the command runs directly on the host. -#[expect( - clippy::too_many_lines, - reason = "Each scenario needs its configuration" -)] fn nosandbox_scenarios() -> Vec { vec![ Scenario { @@ -2309,9 +2113,6 @@ fn nosandbox_scenarios() -> Vec { description: "Non-sandboxed: simple echo", dockerfile: r#"FROM busybox:musl CMD ["echo", "hello from host"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2325,6 +2126,7 @@ CMD ["echo", "hello from host"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "nosandbox_env", @@ -2332,9 +2134,6 @@ CMD ["echo", "hello from host"]"#, dockerfile: r#"FROM busybox:musl ENV MY_VAR=host_test_value CMD ["sh", "-c", "echo $MY_VAR"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2348,6 +2147,7 @@ CMD ["sh", "-c", "echo $MY_VAR"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "nosandbox_metrics", @@ -2356,9 +2156,6 @@ CMD ["sh", "-c", "echo $MY_VAR"]"#, // transport "local" (it previously emitted no metrics at all). dockerfile: r#"FROM busybox:musl CMD ["echo", "local_metrics_test"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2384,15 +2181,13 @@ CMD ["echo", "local_metrics_test"]"#, } bail!("Could not find transport in metrics: {json_str}") }, + ..Scenario::default() }, Scenario { name: "nosandbox_exit_code", description: "Non-sandboxed: non-zero exit code propagation", dockerfile: r#"FROM busybox:musl CMD ["sh", "-c", "exit 42"]"#, - cancel_after_secs: None, - probe: None, - orphan_then_rerun: false, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2409,6 +2204,7 @@ CMD ["sh", "-c", "exit 42"]"#, ) } }, + ..Scenario::default() }, ] } @@ -2666,7 +2462,6 @@ CMD ["sh", "-c", "echo JAIL_CONFINEMENT_a7f3b2c9 && sleep 5"]"#, cancel_after_secs: None, probe: Some(probe_confinement), orphan_then_rerun: false, - sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { // The job has to have actually run before anything the probe @@ -2677,6 +2472,7 @@ CMD ["sh", "-c", "echo JAIL_CONFINEMENT_a7f3b2c9 && sleep 5"]"#, assert_job_succeeded(output, "JAIL_CONFINEMENT_a7f3b2c9")?; assert_no_chroot_remains(&scenario_state_dir()) }, + ..Scenario::default() }, Scenario { name: "jail_sweep_reclaims_orphan", @@ -2688,12 +2484,12 @@ CMD ["sh", "-c", "echo JAIL_SWEEP_a7f3b2c9 && sleep 10"]"#, cancel_after_secs: None, probe: None, orphan_then_rerun: true, - sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { assert_job_succeeded(output, "JAIL_SWEEP_a7f3b2c9")?; assert_no_chroot_remains(&scenario_state_dir()) }, + ..Scenario::default() }, ] } From abf6e5843e89387d551134fc81c1529ec042ae62 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 20:37:54 +0000 Subject: [PATCH 26/91] Let the sweep reap the orphan the scenario creates The scenario hand-killed the orphaned VMM before running the second job, so by the time the sweep ran there was nothing left to find: the reap returned immediately, and the cgroup it was handed was already empty. The green run proved only that a directory can be deleted. Every line of the reap and of the cgroup retry was unexecuted, here and everywhere else. The hand-reap is gone. The scenario now asserts what the sweep actually did: the chroot is reclaimed, the orphaned pid is no longer a running Firecracker, and the cgroup that shares its name is gone. The cgroup assertion is conditional on one having existed, so a host with no CPU isolation cannot pass it for the wrong reason, and the pid check matches on the command as well so a recycled pid does not read as a VMM that was never reaped. The stray Firecracker the hand-reap guarded against is exactly what the reap exists to prevent, so if the reap fails this scenario goes red. That is the point of it. --- tasks/test_runner/src/task/scenarios.rs | 44 +++++++++++++++++++++---- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index 56e1e3bd6..09c5b30c8 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -2743,23 +2743,53 @@ fn run_runner_after_orphan( ); } - // Reap the VMM the killed runner left behind. The sweep reclaims the - // chroot but nothing reaps an orphaned VMM or its cgroup, so without this - // a stray Firecracker would burn benchmark cores for the rest of the - // suite. Compensating for a known gap, not hiding one. - kill_pid(vmm_pid, libc::SIGKILL); - - println!(" orphaned jail {vm_id} (VMM pid {vmm_pid}), running a second job..."); + // Deliberately NOT reaping the VMM here. Killing it by hand would leave + // the next job's sweep with nothing to find, so the reap, the pidfd + // handling, and the cgroup removal would all be skipped and the scenario + // would prove only that a directory can be deleted. The stray Firecracker + // that a hand-reap guards against is exactly what the sweep now exists to + // prevent, so if the sweep fails this scenario has to go red. + let cgroup = stale_cgroup(&vm_id); + let cgroup_existed = cgroup.exists(); + println!( + " orphaned jail {vm_id} (VMM pid {vmm_pid}, cgroup present: {cgroup_existed}), running a second job..." + ); let output = run_runner(image_path, args, runner_bin)?; if jail_root.exists() { bail!("The orphaned chroot {jail_root} survived the next job, so it was never swept"); } + if is_firecracker(vmm_pid) { + bail!( + "The orphaned VMM (pid {vmm_pid}) is still running after the next job, so the sweep never reaped it. It still holds the benchmark cores." + ); + } + // Only meaningful where a cgroup was created at all: a host with no CPU + // isolation never makes one, and asserting its absence would pass for the + // wrong reason. + if cgroup_existed && cgroup.exists() { + bail!( + "The orphaned cgroup {cgroup} survived the next job. It still owns the exclusive benchmark CPUs, so no later run can be isolated." + ); + } Ok(output) } +/// The cgroup a jail leaves behind, which shares the jail's name. +fn stale_cgroup(vm_id: &str) -> Utf8PathBuf { + Utf8PathBuf::from("/sys/fs/cgroup/bencher").join(vm_id) +} + +/// Whether a pid is a running Firecracker. +/// +/// Checking the command as well as the pid keeps a recycled pid from reading +/// as a VMM that was never reaped. +fn is_firecracker(pid: u32) -> bool { + fs::read_to_string(format!("/proc/{pid}/comm")).is_ok_and(|comm| comm.trim() == "firecracker") +} + /// Send a signal to a process, ignoring the result. fn kill_pid(pid: u32, signal: libc::c_int) { #[expect( From 98ddcdda3445483c89b480226392ae0b9efc3298 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 20:38:07 +0000 Subject: [PATCH 27/91] Keep the chroot when the reap leaves a VMM running, and finish the sweep The sweep discarded the reap's result and removed the tree either way. The reap is best effort and reports nothing done on real failures: no pidfd_open on kernels before 5.3, a kill that failed, or no exit within five seconds. In each case the chroot was then removed out from under a live VMM, contradicting this module's own claim that the process goes first and the directory second. The worse half was quieter. Removing the chroot destroys the only handle for re-identifying that process, so the next sweep never sees the directory entry, never calls remove_stale_cgroup for that id, and the cgroup leaks permanently and silently while still owning the exclusive benchmark CPUs. The reap now reports whether the jail is clear rather than which pid it touched, and a jail that is not clear is left alone with a warning naming the pid and the cgroup still holding the cores. The sweep also no longer abandons the rest of its work on the first unremovable cgroup, which left every later stale jail unreaped with its chroot and cgroup in place. The first failure is remembered, the loop finishes, and the failure is returned at the end, so preparation still does not latch and the next job still retries. --- plus/bencher_runner/src/jail/reap.rs | 61 +++++++++++++++++++++------ plus/bencher_runner/src/jail/state.rs | 35 ++++++++++++--- 2 files changed, 76 insertions(+), 20 deletions(-) diff --git a/plus/bencher_runner/src/jail/reap.rs b/plus/bencher_runner/src/jail/reap.rs index ec75b3a69..bdc77d422 100644 --- a/plus/bencher_runner/src/jail/reap.rs +++ b/plus/bencher_runner/src/jail/reap.rs @@ -31,13 +31,33 @@ const REAP_TIMEOUT: Duration = Duration::from_secs(5); /// How often to check whether it has. const REAP_INTERVAL: Duration = Duration::from_millis(20); +/// Whether a jail still has a VMM running in it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Reaped { + /// Nothing is running in the jail: either nothing was, or it has exited. + Clear, + /// A VMM is still running in the jail and could not be reaped. + /// + /// The caller must not remove the chroot: doing so would pull the rootfs + /// out from under a live process, and would destroy the only handle for + /// identifying that process on a later sweep. + StillRunning { + /// The VMM that is still running. + pid: u32, + }, +} + /// Kill the VMM confined to `jail_root`, if one is still running. /// -/// Returns the pid that was reaped. Best effort: a VMM that cannot be -/// identified or killed is reported and left alone, because the alternative -/// to leaving an unidentified process alone is killing the wrong one. -pub fn reap_jailed_vmm(jail_root: &Utf8Path) -> Option { - let pid = find_jailed_vmm(jail_root)?; +/// Best effort about *which* process it touches: a VMM that cannot be +/// identified is left alone, because the alternative to leaving an +/// unidentified process alone is killing the wrong one. Never best effort +/// about what it reports, because the caller decides whether to delete a +/// directory based on the answer. +pub fn reap_jailed_vmm(jail_root: &Utf8Path) -> Reaped { + let Some(pid) = find_jailed_vmm(jail_root) else { + return Reaped::Clear; + }; // Pin the pid before signalling it. A pid found by scanning `/proc` can // exit and have its number recycled before the signal lands, and this runs @@ -48,7 +68,7 @@ pub fn reap_jailed_vmm(jail_root: &Utf8Path) -> Option { let pidfd = match pidfd_open(pid) { Ok(Some(pidfd)) => pidfd, // Already gone, which is the common case and not a failure. - Ok(None) => return None, + Ok(None) => return Reaped::Clear, Err(e) => { // Silence here is what wedges a runner: the orphan keeps the // benchmark CPUs, the cgroup cannot be removed, and nothing says @@ -56,29 +76,30 @@ pub fn reap_jailed_vmm(jail_root: &Utf8Path) -> Option { eprintln!( "Warning: cannot pin orphaned VMM (pid {pid}) in {jail_root} to reap it: {e}. It is still running and still holds the benchmark CPUs." ); - return None; + return Reaped::StillRunning { pid }; }, }; - // Re-check now that the pid cannot change underneath us. + // Re-check now that the pid cannot change underneath us. If it is no + // longer the jail's VMM, the jail is clear whatever else is true. if !is_jailed_vmm(pid, jail_root) { - return None; + return Reaped::Clear; } if let Err(e) = pidfd_kill(&pidfd) { eprintln!("Warning: failed to kill orphaned VMM (pid {pid}) in {jail_root}: {e}"); - return None; + return Reaped::StillRunning { pid }; } if wait_for_exit(pid) { eprintln!("Warning: reaped orphaned VMM (pid {pid}) left behind in {jail_root}"); - Some(pid) + Reaped::Clear } else { eprintln!( "Warning: orphaned VMM (pid {pid}) in {jail_root} did not exit within {} seconds", REAP_TIMEOUT.as_secs() ); - None + Reaped::StillRunning { pid } } } @@ -305,10 +326,22 @@ mod tests { } #[test] - fn reaping_an_unjailed_directory_kills_nothing() { + fn reaping_an_unjailed_directory_kills_nothing_and_reports_clear() { let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); - assert_eq!(reap_jailed_vmm(&root), None); + assert_eq!(reap_jailed_vmm(&root), Reaped::Clear); + } + + #[test] + fn a_still_running_vmm_carries_its_pid() { + // The caller keys the decision not to delete a directory off this, so + // the variant has to name the process it is refusing to abandon. + let still = Reaped::StillRunning { pid: 4242 }; + assert_ne!(still, Reaped::Clear); + match still { + Reaped::StillRunning { pid } => assert_eq!(pid, 4242), + Reaped::Clear => panic!("expected StillRunning"), + } } } diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index 85decb8f1..20c2ddaf2 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -13,6 +13,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use crate::error::JailError; use crate::jail::VmId; +use crate::jail::reap::Reaped; /// Subdirectory of the state directory used as the jailer's chroot base. const CHROOT_BASE: &str = "jail"; @@ -157,6 +158,11 @@ pub fn sweep_jails(jail_parent: &Utf8Path) -> Result { }; let mut swept = 0; + // The first failure is remembered but does not abandon the rest: one jail + // whose cgroup will not go away must not leave every other stale jail + // unreaped, with its chroot and cgroup still in place. + let mut failure = None; + for entry in entries.flatten() { if !entry.file_type().is_ok_and(|file_type| file_type.is_dir()) { continue; @@ -164,10 +170,19 @@ pub fn sweep_jails(jail_parent: &Utf8Path) -> Result { let vm_id = VmId::from_chroot_name(entry.file_name().to_string_lossy().into_owned()); let jail_dir = jail_parent.join(vm_id.as_str()); - // Reap before removing. Pulling the rootfs out from under a process - // that is still running leaves it running anyway, so the process goes - // first and the directory second. - super::reap::reap_jailed_vmm(&jail_dir.join(JAIL_ROOT)); + // Reap before removing, and only remove once the jail is clear. + // Deleting the tree under a live VMM would not stop it, and it would + // destroy the only handle for identifying that process later: without + // the directory the next sweep never sees this id, never removes its + // cgroup, and the cgroup leaks for good. + if let Reaped::StillRunning { pid } = + super::reap::reap_jailed_vmm(&jail_dir.join(JAIL_ROOT)) + { + eprintln!( + "Warning: leaving stale jail {jail_dir} in place because VMM pid {pid} is still running. It still holds the benchmark CPUs through cgroup {vm_id}, so runs will not be isolated until it is gone." + ); + continue; + } // A chroot that will not go away costs disk. Worth a warning, not // worth refusing to run. @@ -180,9 +195,17 @@ pub fn sweep_jails(jail_parent: &Utf8Path) -> Result { // exclusive benchmark CPUs, so leaving it makes every later job's // cpuset write fail. It shares the chroot's name by construction, and // failing to remove it is reported rather than swallowed. - super::cgroup::remove_stale_cgroup(&vm_id)?; + if let Err(e) = super::cgroup::remove_stale_cgroup(&vm_id) + && failure.is_none() + { + failure = Some(e); + } + } + + match failure { + Some(e) => Err(e), + None => Ok(swept), } - Ok(swept) } #[cfg(test)] From d936d35b57bc724147c2429b2dcdb6e5c9e3ae53 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 20:38:22 +0000 Subject: [PATCH 28/91] Read back the cpuset so Applied means the kernel agreed apply_cpuset wrote cpuset.cpus and cpuset.mems and trusted both. Under cgroup v2 a write that overlaps a sibling's exclusive set or reaches past the parent's effective set is accepted and then silently narrowed, possibly to nothing, in which case the VMM simply inherits the parent's CPUs and the run reports a number measured on the whole machine. This was the only fidelity mechanism here still taking a write on trust: the partition mode reads back, cgroup placement reads back, and the commit that introduced this distinction existed precisely to separate applied from half-applied. cpuset.cpus.effective is now read back and compared as a set, since the kernel may render the same set differently from the way it was written. A narrowed or emptied set is an error naming both what was asked for and what was granted. Two variants also claimed confinement without performing a write: an empty benchmark core set, and a layout with no isolation. Neither is reachable today, both were one refactor from being live. Every outcome that is not Applied now carries its reason, so a claim of confinement cannot be made without a verified write behind it. Adds the degrade-path coverage that CI cannot reach, since no hosted runner withholds cpuset delegation: an undelegated controller, an honored write, an equivalent rendering, a narrowed set, and an emptied one. --- plus/bencher_runner/src/error.rs | 10 ++ plus/bencher_runner/src/firecracker/mod.rs | 4 +- plus/bencher_runner/src/jail/cgroup.rs | 182 +++++++++++++++++++-- plus/bencher_runner/src/local_isolation.rs | 6 +- 4 files changed, 186 insertions(+), 16 deletions(-) diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index 0a7a6ebbf..b50684b7d 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -138,6 +138,16 @@ pub enum JailError { )] NetnsNotDistinct { path: Utf8PathBuf }, + #[cfg(target_os = "linux")] + #[error( + "The kernel narrowed the cgroup cpuset at {path}: asked for cpus {requested}, got {effective}. The benchmark would not have run on the cores it claims." + )] + CpusetNarrowed { + path: Utf8PathBuf, + requested: String, + effective: String, + }, + #[cfg(target_os = "linux")] #[error( "Failed to remove the stale cgroup {path}: {source}. It still owns the exclusive benchmark CPUs, so no run can be isolated until it is gone." diff --git a/plus/bencher_runner/src/firecracker/mod.rs b/plus/bencher_runner/src/firecracker/mod.rs index f3ef8eb05..11a8ed2bc 100644 --- a/plus/bencher_runner/src/firecracker/mod.rs +++ b/plus/bencher_runner/src/firecracker/mod.rs @@ -156,9 +156,9 @@ pub fn run_firecracker( } Some(cg) }, - Cpuset::ControllerUnavailable => { + Cpuset::Unavailable(reason) => { eprintln!( - "Warning: the cpuset controller is not delegated to this cgroup, so this run has no CPU isolation and its numbers carry more variance" + "Warning: this run has no CPU isolation ({reason}), so its numbers carry more variance" ); None }, diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index 106d28002..7f234fda4 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -166,18 +166,18 @@ impl CgroupManager { /// isolation, which the caller degrades on rather than failing. pub fn apply_cpuset(&self, layout: &CpuLayout) -> Result { if !layout.has_isolation() { - // No meaningful isolation possible (single core or overlapping sets) - return Ok(Cpuset::Applied); + // Single core, or overlapping sets: there is nothing to confine to. + return Ok(Cpuset::Unavailable("the CPU layout offers no isolation")); } let cpuset = layout.benchmark_cpuset(); if cpuset.is_empty() { - return Ok(Cpuset::Applied); + return Ok(Cpuset::Unavailable("the benchmark core set is empty")); } let path = self.cgroup_path.join("cpuset.cpus"); if !path.exists() { - return Ok(Cpuset::ControllerUnavailable); + return Ok(Cpuset::Unavailable(UNDELEGATED)); } if let Err(e) = fs::write(&path, &cpuset) { return classify_cpuset_error(path, e); @@ -195,7 +195,43 @@ impl CgroupManager { return classify_cpuset_error(mems_path, e); } - Ok(Cpuset::Applied) + self.verify_cpuset(&cpuset) + } + + /// Confirm the kernel actually gave the cgroup the cores that were asked + /// for. + /// + /// A successful write proves nothing here. Under cgroup v2 a `cpuset.cpus` + /// that overlaps a sibling's exclusive set, or reaches past the parent's + /// effective set, is accepted and then silently narrowed, possibly to + /// nothing at all, in which case the VMM simply inherits the parent's + /// CPUs. The whole point of separating applied from half-applied is lost + /// if the applied case is taken on trust, so the effective set is read + /// back and has to match exactly. Every other fidelity mechanism here + /// already reads back: the partition mode does, and so does cgroup + /// placement. + fn verify_cpuset(&self, requested: &str) -> Result { + let path = self.cgroup_path.join("cpuset.cpus.effective"); + let effective = match fs::read_to_string(&path) { + Ok(effective) => effective, + // Nothing to read back means nothing was delegated, the same + // conclusion the write path draws. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Ok(Cpuset::Unavailable(UNDELEGATED)); + }, + Err(e) => return Err(JailError::ReadCgroup { path, source: e }.into()), + }; + + if parse_cpuset(&effective) == parse_cpuset(requested) { + Ok(Cpuset::Applied) + } else { + Err(JailError::CpusetNarrowed { + path, + requested: requested.to_owned(), + effective: effective.trim().to_owned(), + } + .into()) + } } /// Apply I/O bandwidth limits. @@ -361,14 +397,20 @@ pub(crate) fn effective_mems(cgroup: &Utf8Path) -> String { } } +/// Why a run has no CPU isolation, when it has none. +const UNDELEGATED: &str = "the cpuset controller is not delegated to this cgroup"; + /// Whether the cpuset actually confined the VMM to the benchmark cores. +/// +/// Every variant that is not [`Self::Applied`] carries the reason, so no +/// variant can claim confinement without a verified write behind it. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Cpuset { - /// The cgroup confines the VMM to the benchmark cores. + /// The cgroup confines the VMM to the benchmark cores, read back and + /// confirmed. Applied, - /// The host does not delegate the cpuset controller, so there is nothing - /// to write and no CPU isolation to be had. - ControllerUnavailable, + /// There is no CPU isolation to be had, for the reason given. + Unavailable(&'static str), } /// Decide whether a failed cpuset write is an absent controller or a refusal. @@ -379,7 +421,7 @@ pub enum Cpuset { /// have. fn classify_cpuset_error(path: Utf8PathBuf, error: std::io::Error) -> Result { if error.kind() == std::io::ErrorKind::NotFound { - Ok(Cpuset::ControllerUnavailable) + Ok(Cpuset::Unavailable(UNDELEGATED)) } else { Err(JailError::WriteCgroup { path, @@ -389,6 +431,29 @@ fn classify_cpuset_error(path: Utf8PathBuf, error: std::io::Error) -> Result std::collections::BTreeSet { + let mut cpus = std::collections::BTreeSet::new(); + for group in cpuset.trim().split(',').filter(|group| !group.is_empty()) { + match group.split_once('-') { + Some((start, end)) => { + if let (Ok(start), Ok(end)) = (start.trim().parse(), end.trim().parse::()) { + cpus.extend(start..=end); + } + }, + None => { + if let Ok(cpu) = group.trim().parse() { + cpus.insert(cpu); + } + }, + } + } + cpus +} + /// How long to keep trying to remove a stale cgroup. /// /// `rmdir` fails while the cgroup still holds a process, and the reap that @@ -486,6 +551,103 @@ mod tests { assert_eq!(missing_required_controller("cpu memory"), Some("pids")); } + /// A stand-in cgroup tree with the cpuset controller delegated. + /// + /// `effective` is what the kernel would report back after the write, which + /// is the whole point: a real kernel may narrow it silently. + fn cpuset_tree(effective: &str) -> (tempfile::TempDir, CgroupManager) { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + fs::write(root.join("cpuset.cpus"), "").unwrap(); + fs::write(root.join("cpuset.mems"), "").unwrap(); + fs::write(root.join("cpuset.cpus.effective"), effective).unwrap(); + (dir, CgroupManager::detached(root)) + } + + #[test] + fn a_delegated_cpuset_that_the_kernel_honors_is_applied() { + let (_dir, manager) = cpuset_tree("2-7\n"); + let layout = CpuLayout::with_core_count(8); + + assert_eq!(manager.apply_cpuset(&layout).unwrap(), Cpuset::Applied); + assert_eq!( + fs::read_to_string(manager.path().join("cpuset.cpus")).unwrap(), + "2-7" + ); + } + + #[test] + fn an_equivalent_rendering_still_counts_as_applied() { + // The kernel is free to render the same set differently from the way + // it was written, so the comparison is over sets and not strings. + let (_dir, manager) = cpuset_tree("2,3,4,5,6,7\n"); + let layout = CpuLayout::with_core_count(8); + + assert_eq!(manager.apply_cpuset(&layout).unwrap(), Cpuset::Applied); + } + + #[test] + fn an_undelegated_cpuset_controller_degrades() { + // A host that does not delegate cpuset creates its cgroup fine and + // then has no cpuset.cpus to write. That is a declared absence of + // isolation, not a failure, and no CI runner reaches it. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let manager = CgroupManager::detached(root); + let layout = CpuLayout::with_core_count(8); + + assert_eq!( + manager.apply_cpuset(&layout).unwrap(), + Cpuset::Unavailable(UNDELEGATED) + ); + } + + #[test] + fn a_silently_narrowed_cpuset_is_an_error() { + // The kernel accepts a cpuset that overlaps a sibling's exclusive set + // and then narrows it. A successful write proves nothing. + let (_dir, manager) = cpuset_tree("2-3\n"); + let layout = CpuLayout::with_core_count(8); + + let err = manager.apply_cpuset(&layout).unwrap_err().to_string(); + + assert!(err.contains("2-7"), "names what was asked for: {err}"); + assert!(err.contains("2-3"), "names what was granted: {err}"); + } + + #[test] + fn an_emptied_cpuset_is_an_error() { + // The worst case: narrowed to nothing, so the VMM inherits the + // parent's CPUs and the run silently measures the whole machine. + let (_dir, manager) = cpuset_tree("\n"); + let layout = CpuLayout::with_core_count(8); + + manager.apply_cpuset(&layout).unwrap_err(); + } + + #[test] + fn a_layout_with_no_isolation_claims_nothing() { + // Every variant that is not Applied has to carry a reason, so no + // path can report confinement without a verified write behind it. + let (_dir, manager) = cpuset_tree("0\n"); + let layout = CpuLayout::with_core_count(1); + + assert!(matches!( + manager.apply_cpuset(&layout).unwrap(), + Cpuset::Unavailable(_) + )); + } + + #[test] + fn parse_cpuset_reads_kernel_cpu_lists() { + assert_eq!(parse_cpuset("2-7"), (2..=7).collect()); + assert_eq!(parse_cpuset("2,3,4,5,6,7\n"), (2..=7).collect()); + assert_eq!(parse_cpuset("0-1,4,6-7"), [0, 1, 4, 6, 7].into()); + assert_eq!(parse_cpuset("3"), [3].into()); + assert!(parse_cpuset("").is_empty()); + assert!(parse_cpuset("\n").is_empty()); + } + #[test] fn procs_contains_pid_matches_whole_lines() { assert!(procs_contains_pid("7\n70\n701\n", 7)); diff --git a/plus/bencher_runner/src/local_isolation.rs b/plus/bencher_runner/src/local_isolation.rs index e909cb525..86ca7e929 100644 --- a/plus/bencher_runner/src/local_isolation.rs +++ b/plus/bencher_runner/src/local_isolation.rs @@ -66,10 +66,8 @@ impl LocalIsolation { // cpuset degrades its numbers rather than falsifying them. match cgroup.apply_cpuset(layout) { Ok(crate::jail::Cpuset::Applied) => {}, - Ok(crate::jail::Cpuset::ControllerUnavailable) => { - eprintln!( - "Warning: the cpuset controller is not delegated, so this local run has no CPU isolation" - ); + Ok(crate::jail::Cpuset::Unavailable(reason)) => { + eprintln!("Warning: this local run has no CPU isolation ({reason})"); }, Err(e) => eprintln!("Warning: failed to apply cpuset for local run: {e}"), } From ba9a3f6d34a447c43e4ae775de000e407b4b1f0f Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 21:26:31 +0000 Subject: [PATCH 29/91] Correct the reason a stale jail matters, and read back cpuset.mems Six places claimed a leftover cgroup owns the benchmark CPUs exclusively and so makes the next job's cpuset write fail. That is wrong. cpuset.cpus.exclusive and cpuset.cpus.partition are written only on the bencher parent, by the tuning partition; the per-VM cgroups write cpuset.cpus and cpuset.mems and nothing else. Two sibling cgroups can both hold 2-7, a leftover claims nothing, and nothing downstream is rejected because of it. The correction matters because the read-back added for the cpuset does not backstop this after all. An orphan does not narrow the new job's effective set, so the cpuset applies cleanly and verifies cleanly while a stray Firecracker runs untrusted guest code on the very same cores. The harm is contention, which is invisible in the result. Also reads back cpuset.mems, the last write on the fidelity path still taken on trust. Low risk, since the value derives from the parent's effective set and is already a subset of it, but it is written exactly the same way as cpuset.cpus and belongs to the class this change set out to eliminate. --- plus/bencher_runner/src/error.rs | 8 +- plus/bencher_runner/src/jail/cgroup.rs | 124 ++++++++++++++++-------- plus/bencher_runner/src/jail/reap.rs | 14 ++- tasks/test_runner/src/task/scenarios.rs | 2 +- 4 files changed, 104 insertions(+), 44 deletions(-) diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index b50684b7d..82a9d2e11 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -138,6 +138,12 @@ pub enum JailError { )] NetnsNotDistinct { path: Utf8PathBuf }, + #[cfg(target_os = "linux")] + #[error( + "A stale jail at {path} still has VMM pid {pid} running in it. It is executing untrusted guest code on the benchmark cores, so any measurement taken now is contended. The jail is left in place until it can be reaped." + )] + JailStillRunning { path: Utf8PathBuf, pid: u32 }, + #[cfg(target_os = "linux")] #[error( "The kernel narrowed the cgroup cpuset at {path}: asked for cpus {requested}, got {effective}. The benchmark would not have run on the cores it claims." @@ -150,7 +156,7 @@ pub enum JailError { #[cfg(target_os = "linux")] #[error( - "Failed to remove the stale cgroup {path}: {source}. It still owns the exclusive benchmark CPUs, so no run can be isolated until it is gone." + "Failed to remove the stale cgroup {path}: {source}. Stale cgroups accumulate under the parent, and one that cannot be removed usually means something is still running in it." )] StaleCgroup { path: Utf8PathBuf, diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index 7f234fda4..d5368159a 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -195,43 +195,53 @@ impl CgroupManager { return classify_cpuset_error(mems_path, e); } - self.verify_cpuset(&cpuset) + self.verify_cpuset(&cpuset, &mems) } /// Confirm the kernel actually gave the cgroup the cores that were asked /// for. /// - /// A successful write proves nothing here. Under cgroup v2 a `cpuset.cpus` - /// that overlaps a sibling's exclusive set, or reaches past the parent's - /// effective set, is accepted and then silently narrowed, possibly to - /// nothing at all, in which case the VMM simply inherits the parent's - /// CPUs. The whole point of separating applied from half-applied is lost - /// if the applied case is taken on trust, so the effective set is read - /// back and has to match exactly. Every other fidelity mechanism here - /// already reads back: the partition mode does, and so does cgroup - /// placement. - fn verify_cpuset(&self, requested: &str) -> Result { - let path = self.cgroup_path.join("cpuset.cpus.effective"); - let effective = match fs::read_to_string(&path) { - Ok(effective) => effective, - // Nothing to read back means nothing was delegated, the same - // conclusion the write path draws. - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - return Ok(Cpuset::Unavailable(UNDELEGATED)); - }, - Err(e) => return Err(JailError::ReadCgroup { path, source: e }.into()), - }; - - if parse_cpuset(&effective) == parse_cpuset(requested) { - Ok(Cpuset::Applied) - } else { - Err(JailError::CpusetNarrowed { - path, - requested: requested.to_owned(), - effective: effective.trim().to_owned(), + /// A successful write proves nothing here. Under cgroup v2 the effective + /// set is the written set intersected with the parent's effective set, so + /// a `cpuset.cpus` that reaches past the parent is accepted and then + /// silently narrowed, possibly to nothing at all, in which case the VMM + /// simply inherits the parent's CPUs. (An exclusive sibling narrows a set + /// the same way, though these cgroups never claim exclusivity.) The whole + /// point of separating applied from half-applied is lost if the applied + /// case is taken on trust, so both effective sets are read back and have + /// to match. Every other fidelity mechanism here already reads back: the + /// partition mode does, and so does cgroup placement. + fn verify_cpuset(&self, cpus: &str, mems: &str) -> Result { + // Memory nodes as well as cpus. The mems value is derived from the + // parent's effective set, so narrowing is unlikely, but it is written + // exactly the same way and would otherwise be the last write on this + // path still taken on trust. + for (file, requested) in [ + ("cpuset.cpus.effective", cpus), + ("cpuset.mems.effective", mems), + ] { + let path = self.cgroup_path.join(file); + let effective = match fs::read_to_string(&path) { + Ok(effective) => effective, + // Nothing to read back means nothing was delegated, the same + // conclusion the write path draws. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Ok(Cpuset::Unavailable(UNDELEGATED)); + }, + Err(e) => return Err(JailError::ReadCgroup { path, source: e }.into()), + }; + + if parse_cpuset(&effective) != parse_cpuset(requested) { + return Err(JailError::CpusetNarrowed { + path, + requested: requested.to_owned(), + effective: effective.trim().to_owned(), + } + .into()); } - .into()) } + + Ok(Cpuset::Applied) } /// Apply I/O bandwidth limits. @@ -470,8 +480,9 @@ const REMOVE_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50 /// /// Must run after the VMM in it has been reaped: `rmdir` on a cgroup that /// still holds a process fails, which is what forces that ordering. A cgroup -/// that survives is worth shouting about, because it holds the exclusive -/// benchmark CPUs and the next job's cpuset will be rejected because of it. +/// that survives is worth reporting, not because it claims anything (these +/// cgroups set no exclusive cpuset, so a leftover blocks nothing) but because +/// the usual reason `rmdir` fails is that something is still running in it. pub(crate) fn remove_stale_cgroup(vm_id: &VmId) -> Result<(), JailError> { let path = Utf8PathBuf::from(CGROUP_ROOT) .join(BENCHER_CGROUP_BASE) @@ -490,10 +501,9 @@ pub(crate) fn remove_stale_cgroup(vm_id: &VmId) -> Result<(), JailError> { // Someone else got there first, which is the outcome either way. Err(_) if !path.exists() => return Ok(()), Err(e) if std::time::Instant::now() >= deadline => { - // Reported rather than warned. The leftover still owns the - // exclusive benchmark CPUs, so every later job's cpuset would - // be rejected; failing here means the next job sweeps again - // instead of inheriting a host that can never isolate. + // Reported rather than warned, because failing here means the + // next job sweeps again rather than inheriting a host nobody + // is looking at. return Err(JailError::StaleCgroup { path, source: e }); }, Err(_) => std::thread::sleep(REMOVE_INTERVAL), @@ -556,11 +566,22 @@ mod tests { /// `effective` is what the kernel would report back after the write, which /// is the whole point: a real kernel may narrow it silently. fn cpuset_tree(effective: &str) -> (tempfile::TempDir, CgroupManager) { + // The mems written here derive from a parent with no + // `cpuset.mems.effective`, which falls back to node 0. + cpuset_tree_with_mems(effective, "0") + } + + /// A stand-in tree where the effective memory nodes can also be chosen. + fn cpuset_tree_with_mems( + effective_cpus: &str, + effective_mems: &str, + ) -> (tempfile::TempDir, CgroupManager) { let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); fs::write(root.join("cpuset.cpus"), "").unwrap(); fs::write(root.join("cpuset.mems"), "").unwrap(); - fs::write(root.join("cpuset.cpus.effective"), effective).unwrap(); + fs::write(root.join("cpuset.cpus.effective"), effective_cpus).unwrap(); + fs::write(root.join("cpuset.mems.effective"), effective_mems).unwrap(); (dir, CgroupManager::detached(root)) } @@ -586,6 +607,31 @@ mod tests { assert_eq!(manager.apply_cpuset(&layout).unwrap(), Cpuset::Applied); } + #[test] + fn a_narrowed_memory_node_set_is_an_error() { + // The last write on this path that used to be taken on trust. + let (_dir, manager) = cpuset_tree_with_mems("2-7\n", "\n"); + let layout = CpuLayout::with_core_count(8); + + manager.apply_cpuset(&layout).unwrap_err(); + } + + #[test] + fn an_undelegated_memory_node_set_degrades() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + fs::write(root.join("cpuset.cpus"), "").unwrap(); + fs::write(root.join("cpuset.mems"), "").unwrap(); + fs::write(root.join("cpuset.cpus.effective"), "2-7\n").unwrap(); + let manager = CgroupManager::detached(root); + let layout = CpuLayout::with_core_count(8); + + assert_eq!( + manager.apply_cpuset(&layout).unwrap(), + Cpuset::Unavailable(UNDELEGATED) + ); + } + #[test] fn an_undelegated_cpuset_controller_degrades() { // A host that does not delegate cpuset creates its cgroup fine and @@ -604,8 +650,8 @@ mod tests { #[test] fn a_silently_narrowed_cpuset_is_an_error() { - // The kernel accepts a cpuset that overlaps a sibling's exclusive set - // and then narrows it. A successful write proves nothing. + // The kernel intersects the written set with the parent's effective + // set and reports the result. A successful write proves nothing. let (_dir, manager) = cpuset_tree("2-3\n"); let layout = CpuLayout::with_core_count(8); diff --git a/plus/bencher_runner/src/jail/reap.rs b/plus/bencher_runner/src/jail/reap.rs index bdc77d422..70c47df21 100644 --- a/plus/bencher_runner/src/jail/reap.rs +++ b/plus/bencher_runner/src/jail/reap.rs @@ -6,9 +6,17 @@ //! reparented and keeps running, holding the benchmark cores through its //! cgroup. Reclaiming only the disk leaves the more damaging half behind. //! -//! The consequence is not leakage but wrong numbers that look right. An -//! orphaned cgroup still owns the exclusive benchmark CPUs, so the next job's -//! cpuset write is rejected and, before this, the job reported success anyway. +//! The consequence is not leakage but wrong numbers that look right, and +//! nothing downstream catches it. The per-VM cgroups set only `cpuset.cpus` +//! and `cpuset.mems`, never `cpuset.cpus.exclusive`, so a leftover cgroup +//! claims nothing and does not narrow the next job's effective set: the next +//! job's cpuset applies cleanly and verifies cleanly while the stray VMM runs +//! untrusted guest code on the very same cores. The harm is contention, which +//! is invisible in the result. +//! +//! That is why a jail that cannot be cleared fails the job rather than merely +//! warning: it is the one remaining path where the runner would knowingly +//! emit a number it has reason to distrust. //! //! Killing a process the runner does not own is a destructive capability, so //! the target is identified as narrowly as possible: only a process whose root diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index 09c5b30c8..e31929797 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -2770,7 +2770,7 @@ fn run_runner_after_orphan( // wrong reason. if cgroup_existed && cgroup.exists() { bail!( - "The orphaned cgroup {cgroup} survived the next job. It still owns the exclusive benchmark CPUs, so no later run can be isolated." + "The orphaned cgroup {cgroup} survived the next job, so the sweep never removed it. Stale cgroups accumulate, and one that will not go away usually means its VMM is still running." ); } From c1318a914ba1f9a918436468c5b1560357b8219e Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 28 Jul 2026 21:27:32 +0000 Subject: [PATCH 30/91] Fail the job when a stale jail still has a VMM running This was the last path where the runner would knowingly emit a number it has reason to distrust. A stray VMM executes untrusted guest code on the benchmark cores, and nothing catches it: these cgroups claim no exclusive cpuset, so the new job's cpuset applies and verifies cleanly while being contended the whole time. The spec's own rule covers it, since a cgroup that does not contain the VMM is a silent lie about where the benchmark ran, and a stray VMM on those cores is the same lie by another route. Fatal to the job, not to the runner. The sweep already propagates and preparation does not latch on failure, so each later job re-attempts: a transient cause self-heals on the next job, and a persistent one fails loudly every time rather than quietly producing bad data. Both halves are now tested rather than reasoned about, including that three consecutive sweeps each report the pid and the jail, and that removing the cause lets the very next preparation succeed with no restart. Every surviving jail is warned about while the first becomes the error, so an operator sees each one rather than only the first. The sweep takes the reap as a parameter so the skip-removal branch can be tested. That branch is the one preventing a destructive action, and it only runs when a real VMM survives a real kill; manufacturing that would test fault injection rather than this code. --- plus/bencher_runner/src/jail/mod.rs | 26 ++++++ plus/bencher_runner/src/jail/state.rs | 111 ++++++++++++++++++++++++-- 2 files changed, 129 insertions(+), 8 deletions(-) diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index e008d515d..b748e1ea3 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -467,6 +467,32 @@ mod tests { ); } + #[cfg(target_os = "linux")] + #[test] + fn a_failure_is_not_latched_and_self_heals() { + // Fatal to the job, not to the runner. A host that cannot be prepared + // has to fail every job that needs a jail, and recover on its own the + // moment the cause goes away, rather than wedging until a restart. + let dir = tempfile::tempdir().unwrap(); + let root = camino::Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let state_dir = root.join("state"); + // A populated directory the runner did not create is refused. + std::fs::create_dir_all(state_dir.join("someone-elses-data")).unwrap(); + + let mut host = HostPreparation::new(); + for attempt in 1..=3 { + assert!( + host.ensure(&state_dir, JailUser::default()).is_err(), + "attempt {attempt} must fail" + ); + } + + // Remove the cause and the very next job succeeds, with no restart. + std::fs::remove_dir(state_dir.join("someone-elses-data")).unwrap(); + host.ensure(&state_dir, JailUser::default()).unwrap(); + assert!(state_dir.join("jail").is_dir()); + } + #[cfg(target_os = "linux")] #[test] fn the_jail_user_rejects_root() { diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index 20c2ddaf2..c085040a3 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -153,6 +153,19 @@ const RUNNER_ENTRIES: [&str; 2] = [CHROOT_BASE, LOCK_FILE]; /// Non-directory entries are left alone: the jailer only ever creates /// directories here, so anything else was put there by someone else. pub fn sweep_jails(jail_parent: &Utf8Path) -> Result { + sweep_jails_with(jail_parent, super::reap::reap_jailed_vmm) +} + +/// The sweep, with the reap injectable. +/// +/// The branch that refuses to remove a directory is the one preventing a +/// destructive action, and it only runs when a real VMM survives a real kill. +/// Manufacturing that would be testing fault injection rather than this code, +/// so the reap is a parameter and the tests supply the answer. +fn sweep_jails_with(jail_parent: &Utf8Path, reap: R) -> Result +where + R: Fn(&Utf8Path) -> Reaped, +{ let Ok(entries) = fs::read_dir(jail_parent) else { return Ok(0); }; @@ -175,12 +188,25 @@ pub fn sweep_jails(jail_parent: &Utf8Path) -> Result { // destroy the only handle for identifying that process later: without // the directory the next sweep never sees this id, never removes its // cgroup, and the cgroup leaks for good. - if let Reaped::StillRunning { pid } = - super::reap::reap_jailed_vmm(&jail_dir.join(JAIL_ROOT)) - { + if let Reaped::StillRunning { pid } = reap(&jail_dir.join(JAIL_ROOT)) { + // Fatal to the job, not to the runner. A stray VMM runs untrusted + // guest code on the benchmark cores, and nothing downstream + // catches it: these cgroups claim no exclusive cpuset, so the next + // job's cpuset applies and verifies cleanly while being contended + // the whole time. Refusing to measure is the only honest answer. + // + // Every surviving jail is reported and the first becomes the + // error, so an operator sees each one on every attempt rather than + // once. eprintln!( - "Warning: leaving stale jail {jail_dir} in place because VMM pid {pid} is still running. It still holds the benchmark CPUs through cgroup {vm_id}, so runs will not be isolated until it is gone." + "Warning: leaving stale jail {jail_dir} in place because VMM pid {pid} is still running on the benchmark cores." ); + if failure.is_none() { + failure = Some(JailError::JailStillRunning { + path: jail_dir.clone(), + pid, + }); + } continue; } @@ -191,10 +217,11 @@ pub fn sweep_jails(jail_parent: &Utf8Path) -> Result { Err(e) => eprintln!("Warning: failed to sweep stale jail {jail_dir}: {e}"), } - // The cgroup is the half that corrupts later runs: it holds the - // exclusive benchmark CPUs, so leaving it makes every later job's - // cpuset write fail. It shares the chroot's name by construction, and - // failing to remove it is reported rather than swallowed. + // The cgroup shares the chroot's name by construction, so it is + // removed alongside it. A leftover claims nothing, since these cgroups + // set no exclusive cpuset, but they accumulate under the parent and a + // removal that fails usually means something is still running in one. + // Reported rather than swallowed for that reason. if let Err(e) = super::cgroup::remove_stale_cgroup(&vm_id) && failure.is_none() { @@ -363,6 +390,74 @@ mod tests { assert!(note.exists(), "non-directory entries are not the sweep's"); } + #[test] + fn a_jail_whose_vmm_survives_is_left_in_place() { + // Removing the tree would not stop the VMM, and it would destroy the + // only handle for identifying that process on a later sweep. + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")); + state.create().unwrap(); + let live = VmId::from_chroot_name("live".to_owned()); + let dead = VmId::from_chroot_name("dead".to_owned()); + fs::create_dir_all(state.jail_root(&live)).unwrap(); + fs::create_dir_all(state.jail_root(&dead)).unwrap(); + + let err = sweep_jails_with(&state.jail_parent(), |jail_root| { + if jail_root.as_str().contains("live") { + Reaped::StillRunning { pid: 4242 } + } else { + Reaped::Clear + } + }) + .unwrap_err(); + + assert!( + state.jail_dir(&live).exists(), + "a jail with a live VMM must not be removed" + ); + assert!( + !state.jail_dir(&dead).exists(), + "one unreapable jail must not abandon the rest of the sweep" + ); + let message = err.to_string(); + assert!(message.contains("4242"), "names the pid: {message}"); + assert!(message.contains("live"), "names the jail: {message}"); + } + + #[test] + fn a_surviving_vmm_fails_every_attempt_not_just_the_first() { + // A host that can never clear a jail has to tell the operator on every + // job, not once. Nothing latches, so the sweep is re-attempted and + // reports again. + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")); + state.create().unwrap(); + let live = VmId::from_chroot_name("live".to_owned()); + fs::create_dir_all(state.jail_root(&live)).unwrap(); + + let stuck = |_jail_root: &Utf8Path| Reaped::StillRunning { pid: 7 }; + for attempt in 1..=3 { + let err = sweep_jails_with(&state.jail_parent(), stuck).unwrap_err(); + assert!( + err.to_string().contains('7'), + "attempt {attempt} must report the pid" + ); + assert!(state.jail_dir(&live).exists()); + } + } + + #[test] + fn a_cleared_jail_is_still_swept() { + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")); + state.create().unwrap(); + fs::create_dir_all(state.jail_root(&VmId::from_chroot_name("one".to_owned()))).unwrap(); + + let swept = sweep_jails_with(&state.jail_parent(), |_jail_root| Reaped::Clear).unwrap(); + + assert_eq!(swept, 1); + } + #[test] fn sweep_missing_parent_is_zero() { let (_dir, root) = temp_root(); From 607d79fe552f59b9e02587ed0bed0e2aaa8f3921 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Wed, 29 Jul 2026 01:45:01 +0000 Subject: [PATCH 31/91] Fail fast when the jailer exits before the API socket appears The readiness wait still burned its full five seconds when the jailer had already died. A bad --netns, an unwritable --chroot-base-dir, or a refused mknod makes it exit immediately, and polling a socket that will never appear then reported SocketNotReady, pointing at Firecracker rather than at the jailer's own stderr. That is the same shape as the sun_path failure, where a deterministic error was retried until it looked like something else, and SocketUnusable was added to stop exactly that: this finishes the job it started. The readiness loop moves to the process, which is what holds the child, and the client keeps the error taxonomy in a single non-looping attempt. --- plus/bencher_runner/src/firecracker/client.rs | 67 ++++++++----------- plus/bencher_runner/src/firecracker/error.rs | 12 ++++ .../bencher_runner/src/firecracker/process.rs | 32 ++++++++- 3 files changed, 70 insertions(+), 41 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/client.rs b/plus/bencher_runner/src/firecracker/client.rs index c83986faf..611ffe8f0 100644 --- a/plus/bencher_runner/src/firecracker/client.rs +++ b/plus/bencher_runner/src/firecracker/client.rs @@ -29,47 +29,38 @@ impl FirecrackerClient { } } - /// Wait for the Firecracker API socket to become ready. + /// Try the API socket once. /// - /// Only failures that a not-yet-listening VMM actually produces are - /// retried. Anything else fails immediately: an unusable path is not going - /// to become usable, and retrying it for the whole timeout turns a precise - /// error into a timeout that points at Firecracker instead of at the - /// cause. An over-long socket path is rejected by the standard library - /// before any syscall, which is exactly the case this distinction exists - /// to surface. - pub fn wait_for_ready(&self, timeout: Duration) -> Result<(), FirecrackerError> { - let start = std::time::Instant::now(); - let poll_interval = Duration::from_millis(50); - - while start.elapsed() < timeout { - match UnixStream::connect(&self.socket_path) { - Ok(mut stream) => { - drop(stream.set_read_timeout(Some(Duration::from_secs(1)))); - drop(stream.set_write_timeout(Some(Duration::from_secs(1)))); - - let request = "GET / HTTP/1.1\r\nHost: localhost\r\nAccept: */*\r\n\r\n"; - if stream.write_all(request.as_bytes()).is_ok() { - let mut buf = [0u8; 256]; - if let Ok(n) = stream.read(&mut buf) - && n > 0 - { - return Ok(()); - } + /// `Ok(true)` once Firecracker is answering, `Ok(false)` while it is not + /// listening yet, and an error when the address itself cannot be used. + /// Only failures a not-yet-listening VMM actually produces are worth + /// retrying: an unusable path never becomes usable, and retrying it for a + /// whole timeout turns a precise error into one that points at Firecracker + /// instead of at the cause. An over-long socket path is rejected by the + /// standard library before any syscall, which is exactly that case. + pub fn try_ready(&self) -> Result { + match UnixStream::connect(&self.socket_path) { + Ok(mut stream) => { + drop(stream.set_read_timeout(Some(Duration::from_secs(1)))); + drop(stream.set_write_timeout(Some(Duration::from_secs(1)))); + + let request = "GET / HTTP/1.1\r\nHost: localhost\r\nAccept: */*\r\n\r\n"; + if stream.write_all(request.as_bytes()).is_ok() { + let mut buf = [0u8; 256]; + if let Ok(n) = stream.read(&mut buf) + && n > 0 + { + return Ok(true); } - }, - Err(e) if is_not_listening_yet(&e) => {}, - Err(e) => { - return Err(FirecrackerError::SocketUnusable { - path: self.socket_path.clone(), - source: e, - }); - }, - } - std::thread::sleep(poll_interval); + } + Ok(false) + }, + Err(e) if is_not_listening_yet(&e) => Ok(false), + Err(e) => Err(FirecrackerError::SocketUnusable { + path: self.socket_path.clone(), + source: e, + }), } - - Err(FirecrackerError::SocketNotReady(timeout)) } /// Configure the machine (vCPUs, memory). diff --git a/plus/bencher_runner/src/firecracker/error.rs b/plus/bencher_runner/src/firecracker/error.rs index a21632ee7..adf990594 100644 --- a/plus/bencher_runner/src/firecracker/error.rs +++ b/plus/bencher_runner/src/firecracker/error.rs @@ -52,6 +52,18 @@ pub enum FirecrackerError { #[error("Firecracker API socket not ready after {0:?}")] SocketNotReady(std::time::Duration), + /// The jailer exited before Firecracker started serving its API. + /// + /// Distinct from a timeout: the process is gone, so waiting cannot help, + /// and the reason is on stderr under the `[firecracker]` prefix. + #[error( + "The jailer exited ({status}) before the Firecracker API socket appeared; its diagnostics are above, prefixed [firecracker]" + )] + JailerExited { + /// How the jailer exited. + status: String, + }, + /// The API socket address itself cannot be used. /// /// Distinct from [`Self::SocketNotReady`]: waiting will not help, so the diff --git a/plus/bencher_runner/src/firecracker/process.rs b/plus/bencher_runner/src/firecracker/process.rs index 5c0e466c1..2d773eea8 100644 --- a/plus/bencher_runner/src/firecracker/process.rs +++ b/plus/bencher_runner/src/firecracker/process.rs @@ -140,18 +140,44 @@ impl FirecrackerProcess { } }); - let process = Self { + let mut process = Self { child, api_socket: api_socket.clone(), stderr_thread: Some(stderr_thread), }; - // Wait for the API socket to become ready - process.client().wait_for_ready(Duration::from_secs(5))?; + process.wait_for_ready(Duration::from_secs(5))?; Ok(process) } + /// Wait for the API socket, giving up the moment the jailer dies. + /// + /// Watching the child is what keeps a jailer that failed outright from + /// presenting as a socket timeout. A bad `--netns`, an unwritable + /// `--chroot-base-dir`, or a refused `mknod` makes it exit immediately, + /// and polling for a socket that will never appear would report + /// `SocketNotReady` and point at Firecracker instead of at the jailer's + /// own diagnostics, which are already on stderr. + fn wait_for_ready(&mut self, timeout: Duration) -> Result<(), FirecrackerError> { + let start = std::time::Instant::now(); + let poll_interval = Duration::from_millis(50); + + while start.elapsed() < timeout { + if self.client().try_ready()? { + return Ok(()); + } + if let Ok(Some(status)) = self.child.try_wait() { + return Err(FirecrackerError::JailerExited { + status: status.to_string(), + }); + } + std::thread::sleep(poll_interval); + } + + Err(FirecrackerError::SocketNotReady(timeout)) + } + /// Get a client for the Firecracker REST API. pub fn client(&self) -> FirecrackerClient { FirecrackerClient::new(self.api_socket.socket()) From a6b8fcb362d7976c484353d678ed42faffadadf2 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Wed, 29 Jul 2026 01:45:14 +0000 Subject: [PATCH 32/91] Reap every process in a jail, not just the first The reap killed the first process whose root inode matched, reported the jail clear, and the sweep then deleted the tree. A second process sharing that root would have survived and had the directory removed out from under it. Unreachable today, since neither --daemonize nor --new-pid-ns is passed and the jailer execs in place as a single process. But the caller deletes a directory tree based on this answer, so the invariant is load-bearing for a destructive operation and nothing asserted it. It now rescans after each reap and loops, bounded, so the guarantee is enforced rather than assumed. Also stops two successes from reading as failures. Removing a stale cgroup and reaping an orphaned VMM both printed under a Warning prefix, which matters more than it sounds on a path whose entire job is telling an operator that something went wrong. --- plus/bencher_runner/src/jail/cgroup.rs | 2 +- plus/bencher_runner/src/jail/reap.rs | 34 +++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index d5368159a..fe4c6690f 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -495,7 +495,7 @@ pub(crate) fn remove_stale_cgroup(vm_id: &VmId) -> Result<(), JailError> { loop { match fs::remove_dir(&path) { Ok(()) => { - eprintln!("Warning: removed stale cgroup {path} left by a previous runner"); + eprintln!("Removed stale cgroup {path} left by a previous runner"); return Ok(()); }, // Someone else got there first, which is the outcome either way. diff --git a/plus/bencher_runner/src/jail/reap.rs b/plus/bencher_runner/src/jail/reap.rs index 70c47df21..e9764a7fb 100644 --- a/plus/bencher_runner/src/jail/reap.rs +++ b/plus/bencher_runner/src/jail/reap.rs @@ -33,6 +33,12 @@ use std::time::{Duration, Instant}; use camino::Utf8Path; +/// How many processes to reap from one jail before giving up. +/// +/// A jail holds one VMM, so this is a bound on a loop that should run once, +/// not an expectation. +const MAX_JAILED_PROCESSES: usize = 64; + /// How long to wait for a killed VMM to disappear. const REAP_TIMEOUT: Duration = Duration::from_secs(5); @@ -63,10 +69,30 @@ pub enum Reaped { /// about what it reports, because the caller decides whether to delete a /// directory based on the answer. pub fn reap_jailed_vmm(jail_root: &Utf8Path) -> Reaped { - let Some(pid) = find_jailed_vmm(jail_root) else { - return Reaped::Clear; - }; + // Rescan after each reap rather than assuming one process per jail. That + // assumption holds today, since neither `--daemonize` nor `--new-pid-ns` + // is passed and the jailer execs in place as a single process, but the + // caller deletes a directory tree based on this answer. An invariant that + // load-bearing is worth enforcing rather than trusting, and a survivor + // would otherwise have the tree removed out from under it. + for _ in 0..MAX_JAILED_PROCESSES { + let Some(pid) = find_jailed_vmm(jail_root) else { + return Reaped::Clear; + }; + if let Reaped::StillRunning { pid } = reap_one(pid, jail_root) { + return Reaped::StillRunning { pid }; + } + } + + // Something keeps appearing in this jail. Report it rather than looping. + match find_jailed_vmm(jail_root) { + Some(pid) => Reaped::StillRunning { pid }, + None => Reaped::Clear, + } +} +/// Kill one process known to be confined to `jail_root`. +fn reap_one(pid: u32, jail_root: &Utf8Path) -> Reaped { // Pin the pid before signalling it. A pid found by scanning `/proc` can // exit and have its number recycled before the signal lands, and this runs // as root, so the signal would go to whatever inherited the number. A @@ -100,7 +126,7 @@ pub fn reap_jailed_vmm(jail_root: &Utf8Path) -> Reaped { } if wait_for_exit(pid) { - eprintln!("Warning: reaped orphaned VMM (pid {pid}) left behind in {jail_root}"); + eprintln!("Reaped orphaned VMM (pid {pid}) left behind in {jail_root}"); Reaped::Clear } else { eprintln!( From e3f04612ae648ce143564a402774de0513d82188 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Wed, 29 Jul 2026 01:45:27 +0000 Subject: [PATCH 33/91] Take the image pull out from under the jail lock The lock moved ahead of the workspace setup when preparation became lazy, so the OCI pull, the unpack, and the init staging all ran under the exclusive per-state-directory lock. None of them touches the chroot. For the daemon that changed nothing, since it is serial, but two concurrent runner run invocations on one host would have serialized on the image download. They can and now do happen before the lock, which leaves only the work that genuinely needs the jail inside it: the kernel copy, the ext4 build, and the run. Host preparation still comes first, so a host that cannot jail at all fails before anything is downloaded rather than after. The kernel image is also handed read access rather than ownership. Firecracker only reads it, so it stays owned by root, which means the VMM cannot write it and cannot chmod it into something it can. Zero impact given the per-job chroot, but out of step with the rest of a change about least privilege. The mode is set explicitly, since a bundled write or a copy from the host can land at 0600 and leave the VMM unable to read its own kernel. Removes the non-Linux prepare_host_once stub, a leftover of the design that HostPreparation replaced. It had no Linux counterpart and no callers, and being pub meant no lint fired. --- plus/bencher_runner/src/jail/chroot.rs | 19 ++++++- plus/bencher_runner/src/jail/mod.rs | 11 ---- plus/bencher_runner/src/vm.rs | 74 +++++++++++++++----------- 3 files changed, 59 insertions(+), 45 deletions(-) diff --git a/plus/bencher_runner/src/jail/chroot.rs b/plus/bencher_runner/src/jail/chroot.rs index b6fa33e8c..ce4a9df3f 100644 --- a/plus/bencher_runner/src/jail/chroot.rs +++ b/plus/bencher_runner/src/jail/chroot.rs @@ -73,9 +73,24 @@ impl Drop for JailDir { /// /// The jailer chowns the chroot root and the device nodes it makes, but that /// chown is not recursive: files the runner placed inside keep the ownership -/// they were created with, which is root. Every artifact Firecracker touches +/// they were created with, which is root. Every artifact Firecracker *writes* /// has to be handed over explicitly, and getting it wrong produces an opaque -/// boot failure, so each one is checked. +/// boot failure, so each one is checked. Anything it only reads gets +/// [`grant_jail_read`] instead. +/// Let the jailed VMM read a file without giving it away. +/// +/// Firecracker only ever reads the kernel image, so it gets read permission +/// and nothing more: the file stays owned by root, which means the VMM cannot +/// write it and cannot chmod it into something it can write. The mode is set +/// explicitly rather than inherited, because a bundled write or a copy from +/// the host can land at 0600 and leave the VMM unable to read its own kernel. +pub fn grant_jail_read(path: &Utf8Path) -> Result<(), JailError> { + fs::set_permissions(path, fs::Permissions::from_mode(0o644)).map_err(|e| JailError::ChownJail { + path: path.to_owned(), + source: e, + }) +} + pub fn chown_to_jail(path: &Utf8Path, jail_user: JailUser) -> Result<(), JailError> { chown(path, Some(jail_user.uid()), Some(jail_user.gid())).map_err(|e| JailError::ChownJail { path: path.to_owned(), diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index b748e1ea3..432c150e9 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -317,17 +317,6 @@ fn lookup_name_in(database: &str, id: u32) -> Option { }) } -/// Prepare the host for jailed execution. -/// -/// The jail is Linux-only, as is the VM executor it protects. -#[cfg(not(target_os = "linux"))] -pub fn prepare_host_once( - _state_dir: &camino::Utf8Path, - _jail_user: JailUser, -) -> Result<(), crate::error::JailError> { - Ok(()) -} - /// Resource limits for the Firecracker microVM process. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ResourceLimits { diff --git a/plus/bencher_runner/src/vm.rs b/plus/bencher_runner/src/vm.rs index 165d8717a..215ac3563 100644 --- a/plus/bencher_runner/src/vm.rs +++ b/plus/bencher_runner/src/vm.rs @@ -34,13 +34,45 @@ pub fn vm_execute( let state_dir = StateDir::new(config.state_dir.clone()); // Prepare the host on demand, before the first jail this process builds. - // Must come before the lock is taken: preparation takes the same lock, and - // `flock` is per open file description, so nesting would block on itself. + // Must come before the job lock is taken: preparation takes the same lock, + // and `flock` is per open file description, so nesting would block on + // itself. It is also the cheap check, so a host that cannot jail at all + // fails here rather than after pulling an image. host.ensure(state_dir.path(), config.jail_user)?; - // Held for the whole job. Another runner's sweep removes every chroot it - // finds, so it must not run while this one is live. Declared before the - // jail guard so the lock outlives the teardown it protects. + // Everything that does not touch the jail happens before the lock. The + // image pull and unpack are the slow part of a job and need nothing from + // the chroot, so holding an exclusive per-state-directory lock across them + // would serialize concurrent `runner run` invocations on the download + // rather than on the jail. + let workspace = prepare_oci_workspace(config)?; + let work_dir = &workspace.work_dir; + let unpack_dir = &workspace.unpack_dir; + let oci_config = workspace.oci_config; + + let command = oci_config.command; + let working_dir = &oci_config.working_dir; + let env = oci_config.env; + + // Write command config for the VM + println!("Writing init config..."); + write_init_config( + unpack_dir, + &command, + working_dir, + &env, + config.file_paths.as_deref(), + config.max_output_size, + )?; + + // Step 5: Install init binary + println!("Installing init binary..."); + install_init_binary(unpack_dir)?; + + // Held from here to the end of the job. Another runner's sweep removes + // every chroot it finds, so it must not run while this one is live. + // Declared before the jail guard so the lock outlives the teardown it + // protects. let _lock = JailLock::acquire(state_dir.path())?; // Rebuilt per job rather than once per daemon lifetime: the handle lives @@ -56,11 +88,6 @@ pub fn vm_execute( let jail = JailPaths::new(jail_dir.root())?; println!(" Jail: {}", jail.root()); - let workspace = prepare_oci_workspace(config)?; - let work_dir = &workspace.work_dir; - let unpack_dir = &workspace.unpack_dir; - let oci_config = workspace.oci_config; - // Everything Firecracker reads has to be inside the chroot, so the kernel // lands in the jail root whatever its source: bundled, supplied by the // job, or found on the host. @@ -76,25 +103,6 @@ pub fn vm_execute( copy_file(&find_kernel()?, kernel_dest)?; } - let command = oci_config.command; - let working_dir = &oci_config.working_dir; - let env = oci_config.env; - - // Write command config for the VM - println!("Writing init config..."); - write_init_config( - unpack_dir, - &command, - working_dir, - &env, - config.file_paths.as_deref(), - config.max_output_size, - )?; - - // Step 5: Install init binary - println!("Installing init binary..."); - install_init_binary(unpack_dir)?; - // Step 6: Create the ext4 rootfs directly in the jail root let rootfs_dest = jail.rootfs().host().as_path(); println!( @@ -104,10 +112,12 @@ pub fn vm_execute( bencher_rootfs::create_ext4_with_size(unpack_dir, rootfs_dest, config.disk.to_mib())?; // The jailer chowns the chroot root and the device nodes it creates, but - // not what the runner placed inside, so hand over each artifact - // explicitly: Firecracker writes the rootfs and reads the kernel. + // not what the runner placed inside, so each artifact is handed over + // explicitly. The rootfs is written by Firecracker and is given away; the + // kernel is only read, so it stays owned by root and merely becomes + // readable. chroot::chown_to_jail(rootfs_dest, config.jail_user)?; - chroot::chown_to_jail(kernel_dest, config.jail_user)?; + chroot::grant_jail_read(kernel_dest)?; // Step 7-8: Build Firecracker config and run the microVM let fc_config = build_firecracker_config(config, work_dir, vm_id, &state_dir, jail, netns)?; From 42a6e0e91661f3da646ba6363742653e90b3c622 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Wed, 29 Jul 2026 01:45:43 +0000 Subject: [PATCH 34/91] Test the stacked-mount unwind, and stop the sweep tests reading host state The unwind loop is the whole reason clear() exists, and it was the one path in that function with no coverage. Its failure wedges every sandboxed job on the host until an operator loops umount by hand, which was demonstrated on a real kernel earlier in this work. A scenario now stacks two bind mounts on the namespace handle before the job runs and asserts the job still succeeds. Verified on a real kernel that the setup produces exactly the wedge: after a single detach the unlink fails with EBUSY and creating the placeholder fails with EPERM, while looping the detach recovers. If the loop regresses, the scenario goes red. The setup hook it needed was free, which was the point of giving Scenario a Default. sweep_jails_with parameterized the reap and then called the real cgroup removal, which stats /sys/fs/cgroup on whatever machine runs the tests. Those tests passed only because their ids happen not to exist on a dev box, which is host state leaking into a unit test; the removal is now injected for the same reason the reap already was. The probe loop also drains the child's pipes while it runs. Nothing read them during the loop, so a runner chatty enough to fill the 64 KiB buffer would block on its own output until the probe timed out. --- plus/bencher_runner/src/jail/state.rs | 64 +++++++++++---- tasks/test_runner/src/task/scenarios.rs | 101 +++++++++++++++++++++++- 2 files changed, 145 insertions(+), 20 deletions(-) diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index c085040a3..63d1725e2 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -153,18 +153,32 @@ const RUNNER_ENTRIES: [&str; 2] = [CHROOT_BASE, LOCK_FILE]; /// Non-directory entries are left alone: the jailer only ever creates /// directories here, so anything else was put there by someone else. pub fn sweep_jails(jail_parent: &Utf8Path) -> Result { - sweep_jails_with(jail_parent, super::reap::reap_jailed_vmm) + sweep_jails_with( + jail_parent, + super::reap::reap_jailed_vmm, + super::cgroup::remove_stale_cgroup, + ) } -/// The sweep, with the reap injectable. +/// The sweep, with the reap and the cgroup removal injectable. /// /// The branch that refuses to remove a directory is the one preventing a /// destructive action, and it only runs when a real VMM survives a real kill. /// Manufacturing that would be testing fault injection rather than this code, /// so the reap is a parameter and the tests supply the answer. -fn sweep_jails_with(jail_parent: &Utf8Path, reap: R) -> Result +/// +/// The cgroup removal is a parameter for a different reason: it reaches into +/// `/sys/fs/cgroup` on the machine running the tests, and a unit test that +/// passes only because a given id happens not to exist on a dev box is +/// reading host state, not this code. +fn sweep_jails_with( + jail_parent: &Utf8Path, + reap: R, + remove_cgroup: C, +) -> Result where R: Fn(&Utf8Path) -> Reaped, + C: Fn(&VmId) -> Result<(), JailError>, { let Ok(entries) = fs::read_dir(jail_parent) else { return Ok(0); @@ -222,7 +236,7 @@ where // set no exclusive cpuset, but they accumulate under the parent and a // removal that fails usually means something is still running in one. // Reported rather than swallowed for that reason. - if let Err(e) = super::cgroup::remove_stale_cgroup(&vm_id) + if let Err(e) = remove_cgroup(&vm_id) && failure.is_none() { failure = Some(e); @@ -357,7 +371,10 @@ mod tests { .unwrap(); fs::create_dir_all(state.jail_dir(&VmId::from_chroot_name("two".to_owned()))).unwrap(); - assert_eq!(sweep_jails(&state.jail_parent()).unwrap(), 2); + assert_eq!( + sweep_jails_with(&state.jail_parent(), |_j| Reaped::Clear, |_v| Ok(())).unwrap(), + 2 + ); assert!( !state .jail_dir(&VmId::from_chroot_name("one".to_owned())) @@ -381,7 +398,10 @@ mod tests { fs::write(¬e, b"not a jail").unwrap(); fs::create_dir_all(state.jail_dir(&VmId::from_chroot_name("stale".to_owned()))).unwrap(); - assert_eq!(sweep_jails(&state.jail_parent()).unwrap(), 1); + assert_eq!( + sweep_jails_with(&state.jail_parent(), |_j| Reaped::Clear, |_v| Ok(())).unwrap(), + 1 + ); assert!( !state .jail_dir(&VmId::from_chroot_name("stale".to_owned())) @@ -402,13 +422,17 @@ mod tests { fs::create_dir_all(state.jail_root(&live)).unwrap(); fs::create_dir_all(state.jail_root(&dead)).unwrap(); - let err = sweep_jails_with(&state.jail_parent(), |jail_root| { - if jail_root.as_str().contains("live") { - Reaped::StillRunning { pid: 4242 } - } else { - Reaped::Clear - } - }) + let err = sweep_jails_with( + &state.jail_parent(), + |jail_root| { + if jail_root.as_str().contains("live") { + Reaped::StillRunning { pid: 4242 } + } else { + Reaped::Clear + } + }, + |_vm_id| Ok(()), + ) .unwrap_err(); assert!( @@ -437,7 +461,7 @@ mod tests { let stuck = |_jail_root: &Utf8Path| Reaped::StillRunning { pid: 7 }; for attempt in 1..=3 { - let err = sweep_jails_with(&state.jail_parent(), stuck).unwrap_err(); + let err = sweep_jails_with(&state.jail_parent(), stuck, |_vm_id| Ok(())).unwrap_err(); assert!( err.to_string().contains('7'), "attempt {attempt} must report the pid" @@ -453,7 +477,12 @@ mod tests { state.create().unwrap(); fs::create_dir_all(state.jail_root(&VmId::from_chroot_name("one".to_owned()))).unwrap(); - let swept = sweep_jails_with(&state.jail_parent(), |_jail_root| Reaped::Clear).unwrap(); + let swept = sweep_jails_with( + &state.jail_parent(), + |_jail_root| Reaped::Clear, + |_vm_id| Ok(()), + ) + .unwrap(); assert_eq!(swept, 1); } @@ -461,6 +490,9 @@ mod tests { #[test] fn sweep_missing_parent_is_zero() { let (_dir, root) = temp_root(); - assert_eq!(sweep_jails(&root.join("nope")).unwrap(), 0); + assert_eq!( + sweep_jails_with(&root.join("nope"), |_j| Reaped::Clear, |_v| Ok(())).unwrap(), + 0 + ); } } diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index e31929797..edaa2c4e5 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -60,6 +60,8 @@ struct Scenario { cancel_after_secs: Option, /// Whether to use `--sandbox firecracker` (default: true). sandboxed: bool, + /// If set, run before the runner starts, to put the host in some state. + setup: Option Result<()>>, /// If set, a host-side check run while the runner is executing. probe: Option, /// Kill the runner once its VMM is up so nothing unwinds, then run the @@ -79,6 +81,7 @@ impl Default for Scenario { dockerfile: "", extra_args: &[], cancel_after_secs: None, + setup: None, probe: None, orphan_then_rerun: false, // Sandboxed is the interesting case and the overwhelming majority, @@ -245,6 +248,10 @@ fn run_scenario(scenario: &Scenario, runner_bin: &Utf8Path) -> Result<()> { let image_path = build_test_image(scenario.name, scenario.dockerfile) .with_context(|| format!("Failed to build image for {}", scenario.name))?; + if let Some(setup) = scenario.setup { + setup().with_context(|| format!("Setup failed for {}", scenario.name))?; + } + // Every scenario gets its own state directory, so jail assertions are // scoped to the scenario and never touch a real runner's state. let state_dir = scenario_state_dir(); @@ -2474,6 +2481,16 @@ CMD ["sh", "-c", "echo JAIL_CONFINEMENT_a7f3b2c9 && sleep 5"]"#, }, ..Scenario::default() }, + Scenario { + name: "jail_netns_recovers_from_stacked_mounts", + description: "A job succeeds against a network namespace handle carrying stacked mounts", + dockerfile: r#"FROM busybox +CMD ["echo", "JAIL_NETNS_a7f3b2c9"]"#, + setup: Some(stack_netns_mounts), + extra_args: &["--timeout", "120"], + validate: |output| assert_job_succeeded(output, "JAIL_NETNS_a7f3b2c9"), + ..Scenario::default() + }, Scenario { name: "jail_sweep_reclaims_orphan", description: "A chroot orphaned by a runner that never unwound is swept by the next job", @@ -2494,6 +2511,43 @@ CMD ["sh", "-c", "echo JAIL_SWEEP_a7f3b2c9 && sleep 10"]"#, ] } +/// Stack extra bind mounts on the network namespace handle. +/// +/// Recreating the handle bind mounts over it, and a bind mount over a file +/// reports no error, so mounts stack. Against a stacked handle a single +/// detach leaves one behind, the unlink then fails with EBUSY, and creating +/// the placeholder fails with EPERM even as root: every sandboxed job on the +/// host fails until an operator loops `umount` by hand. The unwind loop exists +/// for exactly this, and nothing else exercises it. +fn stack_netns_mounts() -> Result<()> { + let handle = "/run/netns/bencher-jail"; + fs::create_dir_all("/run/netns").context("Failed to create the netns directory")?; + if !Utf8Path::new(handle).exists() { + fs::File::create(handle).context("Failed to create the netns handle")?; + } + + for _ in 0..2 { + let status = Command::new("unshare") + .args(["--net", "sh", "-c"]) + .arg(format!("mount --bind /proc/self/ns/net {handle}")) + .status() + .context("Failed to run unshare to stack a netns mount")?; + anyhow::ensure!(status.success(), "Failed to stack a netns mount"); + } + + let stacked = fs::read_to_string("/proc/self/mountinfo") + .context("Failed to read mountinfo")? + .lines() + .filter(|line| line.contains(&format!(" {handle} "))) + .count(); + anyhow::ensure!( + stacked >= 2, + "Expected at least two stacked mounts on {handle}, found {stacked}" + ); + println!(" stacked {stacked} mounts on {handle}"); + Ok(()) +} + /// Assert the runner actually completed the job. /// /// A confinement scenario that asserts only confinement passes vacuously when @@ -2790,6 +2844,41 @@ fn is_firecracker(pid: u32) -> bool { fs::read_to_string(format!("/proc/{pid}/comm")).is_ok_and(|comm| comm.trim() == "firecracker") } +/// Reader threads draining a child's piped output. +struct DrainedOutput { + stdout: std::thread::JoinHandle, + stderr: std::thread::JoinHandle, +} + +impl DrainedOutput { + /// Wait for both readers and return what they collected. + fn join(self) -> (String, String) { + let stdout = self.stdout.join().unwrap_or_default(); + let stderr = self.stderr.join().unwrap_or_default(); + (stdout, stderr) + } +} + +/// Start reading a child's stdout and stderr so neither pipe can fill. +fn drain_output(child: &mut std::process::Child) -> DrainedOutput { + fn reader( + stream: Option, + ) -> std::thread::JoinHandle { + std::thread::spawn(move || { + let mut buffer = String::new(); + if let Some(mut stream) = stream { + drop(stream.read_to_string(&mut buffer)); + } + buffer + }) + } + + DrainedOutput { + stdout: reader(child.stdout.take()), + stderr: reader(child.stderr.take()), + } +} + /// Send a signal to a process, ignoring the result. fn kill_pid(pid: u32, signal: libc::c_int) { #[expect( @@ -2821,6 +2910,11 @@ fn run_runner_with_probe( .stderr(std::process::Stdio::piped()) .spawn()?; + // Drain both pipes while the probe runs. Nothing reads them during the + // loop otherwise, so a runner chatty enough to fill the 64 KiB pipe buffer + // blocks on its own output until the probe times out. + let readers = drain_output(&mut child); + let deadline = std::time::Instant::now() + PROBE_TIMEOUT; let mut observed = None; loop { @@ -2843,15 +2937,14 @@ fn run_runner_with_probe( std::thread::sleep(PROBE_INTERVAL); } - let output = child.wait_with_output()?; - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + let status = child.wait()?; + let (stdout, stderr) = readers.join(); match observed { Some(Ok(())) => Ok(ScenarioOutput { stdout, stderr, - exit_code: output.status.code().unwrap_or(-1), + exit_code: status.code().unwrap_or(-1), }), Some(Err(e)) => Err(e).with_context(|| format!("stdout: {stdout}\nstderr: {stderr}")), None => bail!( From c233e2630e45208a0cbd4b736f28be3d46c9422c Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Wed, 29 Jul 2026 03:35:25 +0000 Subject: [PATCH 35/91] Assert cgroup placement rather than noticing its absence Placement before exec is the centrepiece of this change: it is what fixed the cpuset being applied after the VMM was already running. It had unit tests for its helpers and nothing exercising it end to end, while the scenario named for confinement quietly asserted only the uid half, because a missing cgroup printed a NOTE and returned Ok. That is the third green in this PR that asserted less than it looked like. A missing cgroup is now a failure naming what was not exercised and what a host needs to exercise it, and the scenario additionally requires the runner's own report that it pinned the VMM to the benchmark cores, which it prints only after creating the cgroup, writing the cpuset, and reading the effective set back. Together they make a vacuous pass impossible: the probe cannot pass without a cgroup to check membership against, and validation cannot pass without the runner saying it built one. Verified on a real kernel that a two-vCPU host, which is what the hosted runners are, does reach this: cpuset is delegated on a systemd cgroup v2 host, a two-core layout yields one housekeeping and one benchmark core, and apply_cpuset against a real cgroup returns Applied with the effective set matching exactly. A unit test pins the two-core premise, since if that ever stopped holding the coverage would disappear silently again. --- plus/bencher_runner/src/cpu.rs | 20 ++++++++++ tasks/test_runner/src/task/scenarios.rs | 50 +++++++++++++++++++------ 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/plus/bencher_runner/src/cpu.rs b/plus/bencher_runner/src/cpu.rs index d8dae6a7f..bfab91ea7 100644 --- a/plus/bencher_runner/src/cpu.rs +++ b/plus/bencher_runner/src/cpu.rs @@ -489,6 +489,26 @@ mod tests { assert_eq!(format_cpumask(&[64, 1]), "1,00000000,00000002"); } + #[test] + fn two_cores_is_enough_for_isolation() { + // The scenario suite runs on two-vCPU hosted runners, and the runner + // only builds a cgroup when the layout offers isolation. If this ever + // stopped holding, cgroup placement would silently go unexercised in + // CI, which is the whole reason the scenario asserts it. + let layout = CpuLayout::with_cpu_ids(vec![0, 1]); + + assert!(layout.has_isolation()); + assert_eq!(layout.housekeeping, vec![0]); + assert_eq!(layout.benchmark, vec![1]); + } + + #[test] + fn one_core_offers_no_isolation() { + let layout = CpuLayout::with_cpu_ids(vec![0]); + + assert!(!layout.has_isolation()); + } + #[test] fn benchmark_cpuset_string() { let layout = CpuLayout::with_core_count(8); diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index edaa2c4e5..898c8af16 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -2477,6 +2477,7 @@ CMD ["sh", "-c", "echo JAIL_CONFINEMENT_a7f3b2c9 && sleep 5"]"#, // booted a guest, so without this the scenario stays green // while the product is broken. assert_job_succeeded(output, "JAIL_CONFINEMENT_a7f3b2c9")?; + assert_cpu_isolation_applied(output)?; assert_no_chroot_remains(&scenario_state_dir()) }, ..Scenario::default() @@ -2511,6 +2512,25 @@ CMD ["sh", "-c", "echo JAIL_SWEEP_a7f3b2c9 && sleep 10"]"#, ] } +/// Assert the runner reported that it confined the VMM to the benchmark cores. +/// +/// The runner prints this only after creating the cgroup, writing the cpuset, +/// and reading the effective set back, so it is the runner's own statement +/// that a cgroup exists for the probe to have checked membership against. +/// Without it the probe could pass on a host where no cgroup was ever made. +fn assert_cpu_isolation_applied(output: &ScenarioOutput) -> Result<()> { + const PINNED: &str = "CPU isolation: Firecracker pinned to cores"; + if output.stdout.contains(PINNED) { + return Ok(()); + } + bail!( + "The runner never reported pinning the VMM to benchmark cores, so no cgroup was created \ + and cgroup placement went unexercised by this run. Expected {PINNED:?}.\nstdout: {}\nstderr: {}", + output.stdout, + output.stderr + ) +} + /// Stack extra bind mounts on the network namespace handle. /// /// Recreating the handle bind mounts over it, and a bind mount over a file @@ -2723,20 +2743,28 @@ fn jail_root_uid(jail_root: &Utf8Path) -> Option { /// first time the process is observable. fn check_cgroup_membership(vm_id: &str, pid: u32) -> Result<()> { let procs_path = format!("/sys/fs/cgroup/bencher/{vm_id}/cgroup.procs"); - // No cgroup means no isolation was possible on this host, which is a - // declared limitation rather than a confinement failure. Say so out loud: - // a confinement check that quietly asserts nothing is exactly the kind of - // green that must never be invisible. - let Ok(procs) = fs::read_to_string(&procs_path) else { - println!( - " NOTE: {procs_path} is unreadable, so cgroup placement was NOT verified for this run" - ); - return Ok(()); - }; + + // A missing cgroup is a failure, not a note. Placement before exec is the + // centrepiece of the jail: it is what fixed the cpuset being applied after + // the VMM was already running. Letting its absence pass quietly is how a + // scenario named for confinement ends up asserting only the uid half of + // it, which is a green that means less than it looks like. + let procs = fs::read_to_string(&procs_path).with_context(|| { + format!( + "No cgroup at {procs_path}, so cgroup placement was not exercised at all. \ + The runner creates one whenever its CPU layout offers isolation, which needs \ + two or more online CPUs and the cpuset controller delegated to this cgroup tree." + ) + })?; + if procs.lines().any(|line| line.trim() == pid.to_string()) { Ok(()) } else { - bail!("The VMM (pid {pid}) is not in {procs_path}, which holds: {procs:?}") + bail!( + "The VMM (pid {pid}) is not in {procs_path}, which holds: {procs:?}. \ + Placement happens in pre_exec, before the jailer starts, so membership must \ + already hold the first time the process is visible." + ) } } From 489cd52af942e3be794ba5473d31667f15984000 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Wed, 29 Jul 2026 03:35:40 +0000 Subject: [PATCH 36/91] Re-sweep after a failed teardown, widen the startup budget, fix three diagnostics A chroot that Drop could not remove leaked until restart. The sweep runs once per process, and Drop has nowhere to report, so in a long-lived daemon a failed teardown left a VMM binary and a full guest rootfs behind with only an eprintln. Each jail now carries a signal it raises on failure, owned by the runner rather than global, and preparation sweeps again when it is set. Lazy preparation is unaffected. The API socket budget was five seconds, chosen when Firecracker started on its own. It now also covers the jailer building a chroot, copying a multi-megabyte exec file into it, mknod, chown, pivot_root, and setns, on a host that may be busy running someone else's benchmark. Thirty seconds costs nothing on the failure path, since a jailer that dies is detected when it exits rather than at the deadline. Three diagnostics that would send an operator the wrong way. A doc block ran into the next function's, leaving chown_to_jail undocumented and grant_jail_read carrying a description of the opposite behaviour. grant_jail_read reported a chmod failure as an ownership failure, when it deliberately leaves the file owned by root, so it gets its own variant. And VsockListener::new claimed it bound using the host view while the body binds the socket view; getting that view wrong is the entire failure the three path types exist to prevent, so a doc pointing the wrong way is worse than cosmetic. --- plus/bencher_runner/src/error.rs | 7 ++ .../bencher_runner/src/firecracker/process.rs | 15 +++- plus/bencher_runner/src/firecracker/vsock.rs | 7 +- plus/bencher_runner/src/jail/chroot.rs | 52 ++++++++----- plus/bencher_runner/src/jail/mod.rs | 76 ++++++++++++++++++- plus/bencher_runner/src/vm.rs | 2 +- 6 files changed, 135 insertions(+), 24 deletions(-) diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index 82a9d2e11..632f5c985 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -187,6 +187,13 @@ pub enum JailError { source: std::io::Error, }, + #[cfg(target_os = "linux")] + #[error("Failed to make {path} readable by the jailed VMM: {source}")] + ChmodJail { + path: Utf8PathBuf, + source: std::io::Error, + }, + #[cfg(target_os = "linux")] #[error("Failed to hand {path} to the jail uid and gid: {source}")] ChownJail { diff --git a/plus/bencher_runner/src/firecracker/process.rs b/plus/bencher_runner/src/firecracker/process.rs index 2d773eea8..ed91b856e 100644 --- a/plus/bencher_runner/src/firecracker/process.rs +++ b/plus/bencher_runner/src/firecracker/process.rs @@ -13,6 +13,19 @@ use crate::firecracker::config::{Action, ActionType}; use crate::firecracker::error::FirecrackerError; use crate::jail::{JailFile, JailUser, VmId}; +/// How long to wait for the Firecracker API socket to appear. +/// +/// This budget used to cover Firecracker starting up on its own. It now also +/// has to cover the jailer building a chroot, copying a multi-megabyte exec +/// file into it, creating device nodes, chowning, `pivot_root`, and `setns`, +/// on a host that may be busy running someone else's benchmark. Five seconds +/// left no margin for that. +/// +/// Widening costs nothing on the failure path: a jailer that dies is detected +/// the moment it exits rather than at the deadline, so the only thing this +/// affects is how patient the runner is with a slow host. +const API_SOCKET_TIMEOUT: Duration = Duration::from_secs(30); + /// Everything needed to spawn the VMM under the jailer. #[derive(Debug)] pub struct JailedSpawn<'a> { @@ -146,7 +159,7 @@ impl FirecrackerProcess { stderr_thread: Some(stderr_thread), }; - process.wait_for_ready(Duration::from_secs(5))?; + process.wait_for_ready(API_SOCKET_TIMEOUT)?; Ok(process) } diff --git a/plus/bencher_runner/src/firecracker/vsock.rs b/plus/bencher_runner/src/firecracker/vsock.rs index e8c0b976c..3a59b6ce2 100644 --- a/plus/bencher_runner/src/firecracker/vsock.rs +++ b/plus/bencher_runner/src/firecracker/vsock.rs @@ -75,9 +75,10 @@ pub struct VsockListener { impl VsockListener { /// Create vsock listeners for all expected ports. /// - /// Creates Unix listeners at `{vsock_uds_path}_{port}` for each port, - /// using the host view of the path: the runner binds them from outside - /// the chroot. These must be created before the VM boots. + /// Creates Unix listeners at `{vsock}_{port}` for each port. Binding uses + /// the socket view, which is the only view short enough for `sun_path`; + /// unlinking and ownership use the host view, which does not depend on a + /// descriptor staying open. These must be created before the VM boots. pub fn new(vsock: &JailFile) -> Result { let socket = vsock.socket(); let stdout_path = socket.with_suffix(&format!("_{}", ports::STDOUT)); diff --git a/plus/bencher_runner/src/jail/chroot.rs b/plus/bencher_runner/src/jail/chroot.rs index ce4a9df3f..f923f8825 100644 --- a/plus/bencher_runner/src/jail/chroot.rs +++ b/plus/bencher_runner/src/jail/chroot.rs @@ -14,7 +14,7 @@ use std::os::unix::fs::{PermissionsExt as _, chown}; use camino::{Utf8Path, Utf8PathBuf}; use crate::error::JailError; -use crate::jail::{JailUser, StateDir, VmId}; +use crate::jail::{JailUser, ReclaimFailed, StateDir, VmId}; /// A job's chroot tree, removed when this value is dropped. /// @@ -25,11 +25,16 @@ use crate::jail::{JailUser, StateDir, VmId}; pub struct JailDir { dir: Utf8PathBuf, root: Utf8PathBuf, + reclaim_failed: ReclaimFailed, } impl JailDir { /// Create the chroot tree for `vm_id` at mode 0700. - pub fn create(state: &StateDir, vm_id: &VmId) -> Result { + pub fn create( + state: &StateDir, + vm_id: &VmId, + reclaim_failed: ReclaimFailed, + ) -> Result { let dir = state.jail_dir(vm_id); let root = state.jail_root(vm_id); @@ -49,7 +54,11 @@ impl JailDir { })?; } - Ok(Self { dir, root }) + Ok(Self { + dir, + root, + reclaim_failed, + }) } /// The chroot root, which becomes `/` inside the jail. @@ -64,19 +73,15 @@ impl Drop for JailDir { if let Err(e) = fs::remove_dir_all(&self.dir) && e.kind() != std::io::ErrorKind::NotFound { - eprintln!("Warning: failed to remove jail {}: {e}", self.dir); + eprintln!( + "Warning: failed to remove jail {}: {e}. It holds a VMM binary and a full guest rootfs; the next job will sweep it.", + self.dir + ); + self.reclaim_failed.set(); } } } -/// Hand a file the runner placed inside the chroot to the jail uid and gid. -/// -/// The jailer chowns the chroot root and the device nodes it makes, but that -/// chown is not recursive: files the runner placed inside keep the ownership -/// they were created with, which is root. Every artifact Firecracker *writes* -/// has to be handed over explicitly, and getting it wrong produces an opaque -/// boot failure, so each one is checked. Anything it only reads gets -/// [`grant_jail_read`] instead. /// Let the jailed VMM read a file without giving it away. /// /// Firecracker only ever reads the kernel image, so it gets read permission @@ -85,12 +90,23 @@ impl Drop for JailDir { /// explicitly rather than inherited, because a bundled write or a copy from /// the host can land at 0600 and leave the VMM unable to read its own kernel. pub fn grant_jail_read(path: &Utf8Path) -> Result<(), JailError> { - fs::set_permissions(path, fs::Permissions::from_mode(0o644)).map_err(|e| JailError::ChownJail { + // Reported as a mode failure, not an ownership one. This function + // deliberately leaves the file owned by root, so an operator sent looking + // at ownership would be chasing the opposite of what went wrong. + fs::set_permissions(path, fs::Permissions::from_mode(0o644)).map_err(|e| JailError::ChmodJail { path: path.to_owned(), source: e, }) } +/// Hand a file the runner placed inside the chroot to the jail uid and gid. +/// +/// The jailer chowns the chroot root and the device nodes it makes, but that +/// chown is not recursive: files the runner placed inside keep the ownership +/// they were created with, which is root. Every artifact Firecracker *writes* +/// has to be handed over explicitly, and getting it wrong produces an opaque +/// boot failure, so each one is checked. Anything it only reads gets +/// [`grant_jail_read`] instead. pub fn chown_to_jail(path: &Utf8Path, jail_user: JailUser) -> Result<(), JailError> { chown(path, Some(jail_user.uid()), Some(jail_user.gid())).map_err(|e| JailError::ChownJail { path: path.to_owned(), @@ -119,7 +135,7 @@ mod tests { fn create_builds_a_private_chroot_tree() { let (_dir, state) = state_in_tmpdir(); - let jail = JailDir::create(&state, &vm_id()).unwrap(); + let jail = JailDir::create(&state, &vm_id(), ReclaimFailed::default()).unwrap(); assert_eq!(jail.root(), state.jail_root(&vm_id())); assert!(jail.root().is_dir()); @@ -134,7 +150,7 @@ mod tests { let (_dir, state) = state_in_tmpdir(); fs::create_dir_all(state.jail_root(&vm_id())).unwrap(); - JailDir::create(&state, &vm_id()).unwrap(); + JailDir::create(&state, &vm_id(), ReclaimFailed::default()).unwrap(); } #[test] @@ -142,7 +158,7 @@ mod tests { let (_dir, state) = state_in_tmpdir(); { - let jail = JailDir::create(&state, &vm_id()).unwrap(); + let jail = JailDir::create(&state, &vm_id(), ReclaimFailed::default()).unwrap(); fs::write(jail.root().join("rootfs.ext4"), b"guest").unwrap(); fs::create_dir_all(jail.root().join("dev")).unwrap(); } @@ -163,13 +179,13 @@ mod tests { // impossible to create. fs::write(state.jail_dir(&vm_id()), b"in the way").unwrap(); - JailDir::create(&state, &vm_id()).unwrap_err(); + JailDir::create(&state, &vm_id(), ReclaimFailed::default()).unwrap_err(); } #[test] fn drop_tolerates_an_already_removed_tree() { let (_dir, state) = state_in_tmpdir(); - let jail = JailDir::create(&state, &vm_id()).unwrap(); + let jail = JailDir::create(&state, &vm_id(), ReclaimFailed::default()).unwrap(); fs::remove_dir_all(state.jail_dir(&vm_id())).unwrap(); drop(jail); } diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 432c150e9..0bcf27624 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -34,6 +34,9 @@ pub use paths::{ChrootPath, HostPath, JailFile, JailPaths, SocketPath}; #[cfg(target_os = "linux")] pub use state::StateDir; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + use serde::{Deserialize, Serialize}; /// Default location of the runner's persistent state directory. @@ -171,6 +174,36 @@ pub struct HostPreparation { expect(dead_code, reason = "host preparation is Linux-only") )] prepared: bool, + /// Set when a job's teardown could not reclaim its chroot. + reclaim_failed: ReclaimFailed, +} + +/// Shared signal that a jail could not be reclaimed. +/// +/// `Drop` has nowhere to report a failure, and a chroot that outlives its job +/// holds a copy of the VMM binary and a full guest rootfs. Because the sweep +/// otherwise runs once per process, a long-lived daemon would carry that leak +/// until a restart. Setting this makes the next job sweep again, which is the +/// mechanism that already exists for exactly this. +/// +/// Owned by the runner's [`HostPreparation`] and cloned into each jail, never +/// global. +#[derive(Debug, Clone, Default)] +pub struct ReclaimFailed(Arc); + +impl ReclaimFailed { + /// Record that a jail could not be reclaimed. + pub fn set(&self) { + self.0.store(true, Ordering::SeqCst); + } + + /// Consume the signal, reporting whether it was set. + /// + /// Only the jail reads it, and the jail is Linux-only. + #[cfg(target_os = "linux")] + fn take(&self) -> bool { + self.0.swap(false, Ordering::SeqCst) + } } impl HostPreparation { @@ -180,6 +213,12 @@ impl HostPreparation { Self::default() } + /// A handle each jail uses to report that it could not be reclaimed. + #[must_use] + pub fn reclaim_signal(&self) -> ReclaimFailed { + self.reclaim_failed.clone() + } + /// Prepare the host for jailed execution, at most once. /// /// Called on demand, immediately before the first job builds a jail, never @@ -201,7 +240,9 @@ impl HostPreparation { state_dir: &camino::Utf8Path, jail_user: JailUser, ) -> Result<(), crate::error::JailError> { - if self.prepared { + // A jail that could not be reclaimed earns another sweep, whatever + // this process has already done. + if self.prepared && !self.reclaim_failed.take() { return Ok(()); } prepare_host(state_dir, jail_user)?; @@ -482,6 +523,39 @@ mod tests { assert!(state_dir.join("jail").is_dir()); } + #[cfg(target_os = "linux")] + #[test] + fn a_jail_that_could_not_be_reclaimed_earns_another_sweep() { + // The sweep otherwise runs once per process, so a teardown that failed + // in a long-lived daemon would leak a chroot holding a VMM copy and a + // full guest rootfs until a restart. Drop has nowhere to report, so it + // raises this instead. + let dir = tempfile::tempdir().unwrap(); + let root = camino::Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let state_dir = root.join("state"); + + let mut host = HostPreparation::new(); + host.ensure(&state_dir, JailUser::default()).unwrap(); + + // Already prepared: a second job does not redo the work. + std::fs::remove_dir_all(&state_dir).unwrap(); + host.ensure(&state_dir, JailUser::default()).unwrap(); + assert!(!state_dir.exists(), "preparation happens at most once"); + + // A jail that could not be reclaimed changes that. + host.reclaim_signal().set(); + host.ensure(&state_dir, JailUser::default()).unwrap(); + assert!( + state_dir.join("jail").is_dir(), + "a failed teardown must earn another sweep" + ); + + // And the signal is consumed, not sticky. + std::fs::remove_dir_all(&state_dir).unwrap(); + host.ensure(&state_dir, JailUser::default()).unwrap(); + assert!(!state_dir.exists(), "the signal is consumed once"); + } + #[cfg(target_os = "linux")] #[test] fn the_jail_user_rejects_root() { diff --git a/plus/bencher_runner/src/vm.rs b/plus/bencher_runner/src/vm.rs index 215ac3563..1ea730607 100644 --- a/plus/bencher_runner/src/vm.rs +++ b/plus/bencher_runner/src/vm.rs @@ -84,7 +84,7 @@ pub fn vm_execute( // before any of them exist. Dropping this guard removes the chroot tree, // which is what the workspace temp directory used to cover. let vm_id = VmId::new(); - let jail_dir = JailDir::create(&state_dir, &vm_id)?; + let jail_dir = JailDir::create(&state_dir, &vm_id, host.reclaim_signal())?; let jail = JailPaths::new(jail_dir.root())?; println!(" Jail: {}", jail.root()); From 85d920385c4af693c776a645a415b293dd793d44 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Wed, 29 Jul 2026 04:11:05 +0000 Subject: [PATCH 37/91] Skip non-UTF-8 jail names, own only what we created, say what isolation was lost A non-UTF-8 directory name was lossily converted and the path rebuilt from the result, so everything downstream worked on a different file than the one on disk: the reap stats a path that does not exist and reports the jail clear, so a live VMM is neither reaped nor mentioned, the removal degrades to a warning, and the cgroup removal targets a name nobody created. Planting one needs write access to a 0700 root-owned tree, so it is not a live threat, but skipping is one line and it also makes the swept count honest. The runner only ever writes UTF-8 names here, so anything else is not ours to touch. CgroupManager claimed ownership even when the cgroup already existed, so Drop would rmdir something it had not created. Fresh ids make that unlikely, but the already-exists branch is there precisely for when the id is not fresh. The no-isolation warning also overstated the loss. vCPU thread pinning is gated on the layout rather than on the cgroup, so it still happens; what goes is the cgroup's hard confinement, which is what stops other work being scheduled onto the benchmark cores, along with its metrics and swap control. An operator reading the old line would have concluded more was gone than was. --- plus/bencher_runner/src/firecracker/mod.rs | 9 ++++- plus/bencher_runner/src/jail/cgroup.rs | 41 +++++++++++++++++++--- plus/bencher_runner/src/jail/state.rs | 16 ++++++++- 3 files changed, 60 insertions(+), 6 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/mod.rs b/plus/bencher_runner/src/firecracker/mod.rs index 11a8ed2bc..beaffdce1 100644 --- a/plus/bencher_runner/src/firecracker/mod.rs +++ b/plus/bencher_runner/src/firecracker/mod.rs @@ -157,8 +157,15 @@ pub fn run_firecracker( Some(cg) }, Cpuset::Unavailable(reason) => { + // Precise about what is lost. The vCPU threads are + // still pinned to the benchmark cores further + // down, which is gated on the layout and not on + // the cgroup, so what goes is the cgroup's hard + // confinement (nothing stops other work being + // scheduled onto those cores) along with its + // metrics and swap control. eprintln!( - "Warning: this run has no CPU isolation ({reason}), so its numbers carry more variance" + "Warning: this run has no cgroup cpuset ({reason}), so nothing keeps other work off the benchmark cores and its numbers carry more variance; vCPU threads are still pinned to them" ); None }, diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index fe4c6690f..881574d46 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -44,17 +44,24 @@ impl CgroupManager { // the tuning cpuset partition at startup. Self::enable_controllers(&parent)?; - // Create this run's cgroup - if !cgroup_path.exists() { + // Create this run's cgroup, and remember whether we are the ones who + // made it. `Drop` removes what this created, so claiming ownership of + // a cgroup that was already there would have it rmdir something + // belonging to whoever did create it. Fresh ids make that unlikely, + // but this branch exists precisely for when the id is not fresh. + let created = if cgroup_path.exists() { + false + } else { fs::create_dir_all(&cgroup_path).map_err(|e| JailError::CreateCgroup { path: cgroup_path.clone(), source: e, })?; - } + true + }; Ok(Self { cgroup_path, - created: true, + created, }) } @@ -694,6 +701,32 @@ mod tests { assert!(parse_cpuset("\n").is_empty()); } + #[test] + fn a_cgroup_that_already_existed_is_not_ours_to_remove() { + // Drop removes what this created. Claiming a cgroup that was already + // there would have it rmdir something belonging to whoever did. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + let ours = CgroupManager { + cgroup_path: root.join("ours"), + created: true, + }; + let theirs = CgroupManager { + cgroup_path: root.join("theirs"), + created: false, + }; + fs::create_dir_all(ours.path()).unwrap(); + fs::create_dir_all(theirs.path()).unwrap(); + let theirs_path = theirs.path().to_owned(); + + drop(ours); + drop(theirs); + + assert!(!root.join("ours").exists(), "we remove what we created"); + assert!(theirs_path.exists(), "we leave what we did not create"); + } + #[test] fn procs_contains_pid_matches_whole_lines() { assert!(procs_contains_pid("7\n70\n701\n", 7)); diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index 63d1725e2..240b14d89 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -194,7 +194,21 @@ where if !entry.file_type().is_ok_and(|file_type| file_type.is_dir()) { continue; } - let vm_id = VmId::from_chroot_name(entry.file_name().to_string_lossy().into_owned()); + // Skipped rather than lossily converted. A lossy name rebuilds into a + // path naming a different file, and everything downstream then works + // on the wrong one: the reap stats a path that does not exist and + // reports the jail clear, so a live VMM is neither reaped nor + // mentioned, and the cgroup removal targets a name nobody created. + // The runner only ever creates UTF-8 names here, so anything else is + // not ours to touch. + let file_name = entry.file_name(); + let Some(name) = file_name.to_str() else { + eprintln!( + "Warning: skipping an entry with a non-UTF-8 name under {jail_parent}; the runner did not create it" + ); + continue; + }; + let vm_id = VmId::from_chroot_name(name.to_owned()); let jail_dir = jail_parent.join(vm_id.as_str()); // Reap before removing, and only remove once the jail is clear. From e1069438f67f801611d678775324826b4bfedf1c Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Wed, 29 Jul 2026 04:11:19 +0000 Subject: [PATCH 38/91] Drain the orphan path's pipes, refuse to build as root, note the socket lifetime The orphan scenario polls for up to three minutes while the runner pulls an image, unpacks it, and builds an ext4, with both pipes undrained. That is more than enough output to fill a 64 KiB buffer and block the runner, and it would surface as "No jailed VMM appeared", pointing at the sweep rather than at the pipe. The probe path already drained for exactly this reason. If BENCHER_RUNNER_BIN ever failed to arrive, the harness fell through to cargo as root and left the target directory and cargo cache root-owned, silently, which is the precise outcome the build-then-elevate split exists to prevent. Sudo does pass it through on both runner images, so this is belt and braces, but it now refuses with the two commands to run instead. Verified: as root without the variable it errors and leaves zero root-owned files; with the variable it proceeds and still runs no cargo. Records why SocketPath does not borrow. Borrowing would let the compiler enforce the descriptor's lifetime instead of the doc asking for it, and the failure it guards against is the worst one in that module, but the lifetime would ripple through JailFile, the job config, the jailer spawn, and everything owning them. The comment says the invariant is deliberate and unenforced rather than overlooked, and names what holds the line meanwhile. --- plus/bencher_runner/src/jail/paths.rs | 15 +++++++++++++ tasks/test_runner/src/task/scenarios.rs | 28 +++++++++++++++++++++---- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/plus/bencher_runner/src/jail/paths.rs b/plus/bencher_runner/src/jail/paths.rs index 646c9af8d..3ed9a5ac1 100644 --- a/plus/bencher_runner/src/jail/paths.rs +++ b/plus/bencher_runner/src/jail/paths.rs @@ -87,6 +87,21 @@ impl std::fmt::Display for ChrootPath { /// A path the runner may hand to `bind` or `connect`. /// +/// The value names a descriptor held open by the [`JailPaths`] it came from, +/// and it is only valid while that value is alive. That invariant is upheld by +/// ordering rather than by the compiler: `SocketPath` could borrow, which would +/// make a use-after-drop impossible, but the lifetime would ripple through +/// [`JailFile`], the Firecracker job config, the jailer spawn, and everything +/// that owns them. It is deliberately not enforced, not overlooked, and it is +/// a reasonable follow-up. +/// +/// Two things hold the line in the meantime. Only `bind` and `connect` take +/// this view, because they are the only callers subject to the length limit; +/// everything else, unlinking above all, takes the host view, which cannot go +/// stale. And a test drops the paths, claims the released descriptor number +/// with another directory, and asserts the same string no longer names the +/// jail, so the hazard is at least pinned. +/// /// The type makes the length limit unforgeable: every value has been checked /// against `sun_path`, so a path that would not fit is reported when the jail /// is built, naming the limit and the offending string, rather than surfacing diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index 898c8af16..5894598ba 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -2386,6 +2386,19 @@ fn ensure_runner_bin() -> Result { return Ok(path); } + // Falling through to cargo as root is the exact outcome the + // build-then-elevate split exists to prevent: it leaves the target + // directory and the cargo cache owned by root, and it does so silently. + // Sudo does pass the variable through on both runner images, so this is + // belt and braces, but a loud failure beats a root-owned cache. + anyhow::ensure!( + !is_root(), + "Running as root without {RUNNER_BIN_ENV} set. Building here would run cargo as root and \ + leave the target directory and cargo cache root-owned. Build unprivileged first:\n\ + \x20 cargo test-runner scenarios --build-only\n\ + \x20 sudo {RUNNER_BIN_ENV}=./target/debug/runner ./target/debug/test_runner scenarios" + ); + let workspace_root = super::workspace_root(); let target_triple = super::musl_target_triple()?; @@ -2792,6 +2805,13 @@ fn run_runner_after_orphan( .stderr(std::process::Stdio::piped()) .spawn()?; + // Drain both pipes for the same reason the probe path does. This polls for + // up to three minutes while the runner pulls an image, unpacks it, and + // builds an ext4, which is more than enough output to fill a 64 KiB pipe + // and block the runner. It would surface as "No jailed VMM appeared", + // which points at the sweep rather than at the pipe. + let readers = drain_output(&mut child); + // Wait for a real orphan: a chroot with a VMM running in it, not just an // empty directory created microseconds before the kill. let deadline = std::time::Instant::now() + PROBE_TIMEOUT; @@ -2808,16 +2828,16 @@ fn run_runner_after_orphan( }; let Some((vm_id, jail_root, vmm_pid)) = orphan else { - let output = child.wait_with_output()?; + drop(child.wait()); + let (stdout, stderr) = readers.join(); bail!( - "No jailed VMM appeared within {PROBE_TIMEOUT:?}, so nothing was orphaned and the sweep is untested.\nstdout: {}\nstderr: {}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), + "No jailed VMM appeared within {PROBE_TIMEOUT:?}, so nothing was orphaned and the sweep is untested.\nstdout: {stdout}\nstderr: {stderr}" ); }; kill_pid(child.id(), libc::SIGKILL); drop(child.wait()); + drop(readers.join()); if !jail_root.exists() { bail!( From fd942ed77e40fc7690c51610de161c63ce4d02a2 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:02:54 +0000 Subject: [PATCH 39/91] Name the jail lock file once The lock file name was spelled in two places, and both have to agree for the state directory to recognize a directory the runner created: it counts the lock among the entries that mark a directory as its own, so a drift between the two spellings would have the runner refuse a state directory holding nothing but its own lock. The lock module owns the name and the state directory reads it from there. --- plus/bencher_runner/src/jail/lock.rs | 7 ++++++- plus/bencher_runner/src/jail/state.rs | 7 +------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plus/bencher_runner/src/jail/lock.rs b/plus/bencher_runner/src/jail/lock.rs index c0b641970..2ab72a2ad 100644 --- a/plus/bencher_runner/src/jail/lock.rs +++ b/plus/bencher_runner/src/jail/lock.rs @@ -26,7 +26,12 @@ use crate::error::JailError; /// It lives beside the chroot base rather than inside it, so the sweep (which /// only removes directories under `/jail/firecracker`) can never /// reach it. -const LOCK_FILE: &str = ".lock"; +/// +/// Defined once and shared with the state directory, which counts it among the +/// entries that mark a directory as the runner's own. Two spellings of the same +/// name would leave a directory holding only this file unrecognized, and the +/// runner would refuse a state directory it created itself. +pub(super) const LOCK_FILE: &str = ".lock"; /// Holds the jail lock for as long as it is alive. /// diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index 240b14d89..dda41f99d 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -13,6 +13,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use crate::error::JailError; use crate::jail::VmId; +use crate::jail::lock::LOCK_FILE; use crate::jail::reap::Reaped; /// Subdirectory of the state directory used as the jailer's chroot base. @@ -21,12 +22,6 @@ const CHROOT_BASE: &str = "jail"; /// The chroot directory inside a jail, which the jailer makes `/`. const JAIL_ROOT: &str = "root"; -/// The jail lock file, which lives beside the chroot base. -/// -/// Named here as well as in the lock module so the state directory knows -/// which of its entries it created. -const LOCK_FILE: &str = ".lock"; - /// The `--exec-file` base name the jailer derives the chroot layout from. /// /// The jailer builds `///root`, so the From 26f991fa8df8eb0ca2aeefd5fc172ece3f2ecd4d Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:03:12 +0000 Subject: [PATCH 40/91] Remove a stale jail's cgroup before the chroot that names it The sweep removed the chroot first and then the cgroup, which defeats the reasoning the sweep already states: the chroot must survive a jail it cannot clear, because the directory name is the only handle a later sweep has for finding that jail again. That reasoning holds just as well when the cgroup removal is what fails. With the old order, a cgroup whose `rmdir` the kernel refuses had its directory removed anyway, so the next sweep never saw the id, never retried, and whatever was still in that cgroup kept running on the benchmark cores while later jobs measured through it. The window is real: a runner killed after the jailer is placed in its cgroup but before it pivots into the chroot leaves a process that the reap cannot identify, since its root is still `/`, so the jail reads as clear while the cgroup is not. The cgroup now goes first, and the chroot only once the cgroup is gone. --- plus/bencher_runner/src/jail/state.rs | 71 ++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index dda41f99d..e312525a5 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -233,23 +233,31 @@ where continue; } + // The cgroup goes first, and the chroot only once the cgroup is gone. + // The two are named by the same id, and the directory is the only + // handle a later sweep has for finding the cgroup again, so removing + // the directory while the cgroup survives strands that cgroup for + // good: the next sweep never sees the id, never retries the removal, + // and something may still be running on the benchmark cores under it. + // A leftover cgroup claims nothing, since these cgroups set no + // exclusive cpuset, but a removal that fails usually means something is + // still in it, which is why it is reported rather than swallowed. + if let Err(e) = remove_cgroup(&vm_id) { + eprintln!( + "Warning: leaving stale jail {jail_dir} in place because its cgroup could not be removed: {e}" + ); + if failure.is_none() { + failure = Some(e); + } + continue; + } + // A chroot that will not go away costs disk. Worth a warning, not // worth refusing to run. match fs::remove_dir_all(&jail_dir) { Ok(()) => swept += 1, Err(e) => eprintln!("Warning: failed to sweep stale jail {jail_dir}: {e}"), } - - // The cgroup shares the chroot's name by construction, so it is - // removed alongside it. A leftover claims nothing, since these cgroups - // set no exclusive cpuset, but they accumulate under the parent and a - // removal that fails usually means something is still running in one. - // Reported rather than swallowed for that reason. - if let Err(e) = remove_cgroup(&vm_id) - && failure.is_none() - { - failure = Some(e); - } } match failure { @@ -479,6 +487,47 @@ mod tests { } } + #[test] + fn a_jail_whose_cgroup_survives_keeps_the_chroot_that_names_it() { + // The chroot name is the only handle a later sweep has for finding the + // cgroup, so a directory removed while its cgroup survives strands + // that cgroup for good: nothing ever sees the id again. One stuck + // cgroup must still not abandon the rest of the sweep. + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")); + state.create().unwrap(); + let stuck = VmId::from_chroot_name("stuck".to_owned()); + let clear = VmId::from_chroot_name("clear".to_owned()); + fs::create_dir_all(state.jail_root(&stuck)).unwrap(); + fs::create_dir_all(state.jail_root(&clear)).unwrap(); + + let err = sweep_jails_with( + &state.jail_parent(), + |_jail_root| Reaped::Clear, + |vm_id| { + if vm_id.as_str() == "stuck" { + Err(JailError::StaleCgroup { + path: Utf8PathBuf::from("/sys/fs/cgroup/bencher/stuck"), + source: std::io::Error::from(std::io::ErrorKind::DirectoryNotEmpty), + }) + } else { + Ok(()) + } + }, + ) + .unwrap_err(); + + assert!( + state.jail_dir(&stuck).exists(), + "the chroot names the cgroup that has to be retried" + ); + assert!( + !state.jail_dir(&clear).exists(), + "one stuck cgroup must not abandon the rest of the sweep" + ); + assert!(err.to_string().contains("stuck"), "names the cgroup: {err}"); + } + #[test] fn a_cleared_jail_is_still_swept() { let (_dir, root) = temp_root(); From f27b689a8be190ffcf540182bce590579f9c869a Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:05:27 +0000 Subject: [PATCH 41/91] Hold a job's chroot when its cgroup could not be removed Teardown had the same gap the sweep did, on the ordinary path rather than the crash path. A cgroup whose `rmdir` the kernel refuses only warned, and the chroot guard then removed the directory regardless, which is the one thing that makes such a cgroup unreachable: the directory name is what a later sweep works backwards from, so nothing would ever look for that cgroup again. The reclaim signal now covers both. A refused `rmdir` raises it, which holds the chroot and earns the next job a sweep, and that sweep reclaims the cgroup and the directory together. The signal is shared by construction rather than inferred: the cgroup and the chroot of one job hold the same handle, and the cgroup is torn down first because the VM run owns it and returns before the chroot guard drops. A non-sandboxed run has no chroot naming its cgroup and no sweep that walks it, so it is explicit about holding a signal nothing reads. --- plus/bencher_runner/src/firecracker/mod.rs | 8 +- plus/bencher_runner/src/jail/cgroup.rs | 85 +++++++++++++++++++--- plus/bencher_runner/src/jail/chroot.rs | 37 ++++++++++ plus/bencher_runner/src/jail/mod.rs | 23 ++++++ plus/bencher_runner/src/local_isolation.rs | 5 +- plus/bencher_runner/src/vm.rs | 15 +++- 6 files changed, 156 insertions(+), 17 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/mod.rs b/plus/bencher_runner/src/firecracker/mod.rs index beaffdce1..954d8931b 100644 --- a/plus/bencher_runner/src/firecracker/mod.rs +++ b/plus/bencher_runner/src/firecracker/mod.rs @@ -32,7 +32,7 @@ use std::time::{Duration, Instant}; use camino::Utf8PathBuf; use crate::cpu::CpuLayout; -use crate::jail::{CgroupManager, Cpuset, JailPaths, JailUser, VmId}; +use crate::jail::{CgroupManager, Cpuset, JailPaths, JailUser, ReclaimFailed, VmId}; use crate::metrics::{self, RunMetrics}; pub use error::FirecrackerError; @@ -73,6 +73,10 @@ pub struct FirecrackerJobConfig { pub chroot_base_dir: Utf8PathBuf, /// Handle of the empty network namespace the VMM joins. pub netns: Utf8PathBuf, + /// Shared with the chroot guard of the same id: a cgroup this job cannot + /// remove has to hold that chroot, which is the only handle a later sweep + /// has for finding the cgroup again. + pub reclaim_failed: ReclaimFailed, /// Number of vCPUs. pub vcpus: u8, /// Memory size in MiB. @@ -131,7 +135,7 @@ pub fn run_firecracker( // Step 0: Create cgroup with cpuset if CPU layout is provided let cgroup = if let Some(layout) = &config.cpu_layout { if layout.has_isolation() { - match CgroupManager::new(vm_id) { + match CgroupManager::new(vm_id, config.reclaim_failed.clone()) { Ok(cg) => { // A cgroup that exists but does not confine the VMM to the // benchmark cores would report a number measured somewhere diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index 881574d46..bc79b69a1 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -9,7 +9,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use crate::RunnerError; use crate::cpu::CpuLayout; use crate::error::JailError; -use crate::jail::{ResourceLimits, VmId}; +use crate::jail::{ReclaimFailed, ResourceLimits, VmId}; /// Default cgroup v2 mount point. const CGROUP_ROOT: &str = "/sys/fs/cgroup"; @@ -21,11 +21,19 @@ pub(crate) const BENCHER_CGROUP_BASE: &str = "bencher"; pub struct CgroupManager { cgroup_path: Utf8PathBuf, created: bool, + /// Raised when this cgroup could not be removed, so a later sweep retries + /// it and the chroot that names it is held until then. + reclaim_failed: ReclaimFailed, } impl CgroupManager { /// Create a new cgroup for the given microVM. - pub fn new(vm_id: &VmId) -> Result { + /// + /// The signal is shared with the chroot of the same id: a cgroup that + /// cannot be removed has to keep that directory alive, because the + /// directory name is the only handle a later sweep has for finding this + /// cgroup again. + pub fn new(vm_id: &VmId, reclaim_failed: ReclaimFailed) -> Result { let cgroup_path = Utf8PathBuf::from(CGROUP_ROOT) .join(BENCHER_CGROUP_BASE) .join(vm_id.as_str()); @@ -62,6 +70,7 @@ impl CgroupManager { Ok(Self { cgroup_path, created, + reclaim_failed, }) } @@ -75,6 +84,7 @@ impl CgroupManager { Self { cgroup_path, created: false, + reclaim_failed: ReclaimFailed::unwatched(), } } @@ -382,11 +392,19 @@ impl CgroupManager { } /// Clean up the cgroup. + /// + /// A `rmdir` the kernel refuses is raised on the reclaim signal, not just + /// logged: it means something is still in this cgroup, and the only way to + /// get to it later is through the chroot of the same id, so the signal both + /// holds that directory and earns the next job a sweep. pub fn cleanup(&mut self) -> Result<(), RunnerError> { if self.created && self.cgroup_path.exists() { if let Err(e) = fs::remove_dir(&self.cgroup_path) { - // Log but don't fail - cgroup might still have processes - eprintln!("Warning: failed to remove cgroup {}: {e}", self.cgroup_path); + eprintln!( + "Warning: failed to remove cgroup {}: {e}. Something is still in it, so the next job sweeps it along with the jail that names it.", + self.cgroup_path + ); + self.reclaim_failed.set(); } else { self.created = false; } @@ -711,10 +729,12 @@ mod tests { let ours = CgroupManager { cgroup_path: root.join("ours"), created: true, + reclaim_failed: ReclaimFailed::unwatched(), }; let theirs = CgroupManager { cgroup_path: root.join("theirs"), created: false, + reclaim_failed: ReclaimFailed::unwatched(), }; fs::create_dir_all(ours.path()).unwrap(); fs::create_dir_all(theirs.path()).unwrap(); @@ -738,13 +758,57 @@ mod tests { } #[test] - fn open_procs_reports_a_missing_cgroup() { + fn a_cgroup_that_will_not_go_away_raises_the_reclaim_signal() { + // The kernel refuses `rmdir` while a cgroup still holds a process, and + // a non-empty ordinary directory refuses it the same way. Warning alone + // would let the chroot that names this cgroup be removed, leaving + // nothing for a later sweep to find it by. let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); - let manager = CgroupManager { - cgroup_path: root.join("absent"), - created: false, + let reclaim_failed = ReclaimFailed::default(); + let mut manager = CgroupManager { + cgroup_path: root.join("stuck"), + created: true, + reclaim_failed: reclaim_failed.clone(), }; + fs::create_dir_all(manager.path()).unwrap(); + fs::write(manager.path().join("cgroup.procs"), "42\n").unwrap(); + + manager.cleanup().unwrap(); + + assert!( + reclaim_failed.is_set(), + "a cgroup that outlives its job has to earn another sweep" + ); + assert!(manager.path().exists()); + } + + #[test] + fn a_removed_cgroup_leaves_the_signal_alone() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let reclaim_failed = ReclaimFailed::default(); + let mut manager = CgroupManager { + cgroup_path: root.join("gone"), + created: true, + reclaim_failed: reclaim_failed.clone(), + }; + fs::create_dir_all(manager.path()).unwrap(); + + manager.cleanup().unwrap(); + + assert!(!manager.path().exists()); + assert!( + !reclaim_failed.is_set(), + "a clean teardown must not hold the chroot back" + ); + } + + #[test] + fn open_procs_reports_a_missing_cgroup() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let manager = CgroupManager::detached(root.join("absent")); manager.open_procs().unwrap_err(); manager.contains_pid(1).unwrap_err(); @@ -755,10 +819,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); fs::write(root.join("cgroup.procs"), "123\n456\n").unwrap(); - let manager = CgroupManager { - cgroup_path: root, - created: false, - }; + let manager = CgroupManager::detached(root); assert!(manager.contains_pid(456).unwrap()); assert!(!manager.contains_pid(789).unwrap()); diff --git a/plus/bencher_runner/src/jail/chroot.rs b/plus/bencher_runner/src/jail/chroot.rs index f923f8825..fb4c463c5 100644 --- a/plus/bencher_runner/src/jail/chroot.rs +++ b/plus/bencher_runner/src/jail/chroot.rs @@ -21,6 +21,10 @@ use crate::jail::{JailUser, ReclaimFailed, StateDir, VmId}; /// The jailer cleans up nothing by design, so teardown is the runner's job. /// `Drop` covers completion, timeout, cancellation, and every error return; /// the sweep in `prepare_host` covers the exits that never unwind. +/// +/// The cgroup's teardown runs first, which is what makes the reclaim signal +/// meaningful here: `run_firecracker` owns the cgroup and returns before this +/// guard is dropped. #[derive(Debug)] pub struct JailDir { dir: Utf8PathBuf, @@ -70,6 +74,20 @@ impl JailDir { impl Drop for JailDir { fn drop(&mut self) { + // A cgroup this job could not remove keeps the chroot alive. The two + // are named by the same id, and this directory is the only handle a + // later sweep has for finding that cgroup again, so removing it here + // would strand the cgroup for good with whatever is still in it. The + // same signal already earns the next job a sweep, which reclaims both + // in the right order. + if self.reclaim_failed.is_set() { + eprintln!( + "Warning: leaving jail {} in place because its cgroup could not be removed. The directory names that cgroup, so the next job sweeps both.", + self.dir + ); + return; + } + if let Err(e) = fs::remove_dir_all(&self.dir) && e.kind() != std::io::ErrorKind::NotFound { @@ -182,6 +200,25 @@ mod tests { JailDir::create(&state, &vm_id(), ReclaimFailed::default()).unwrap_err(); } + #[test] + fn a_surviving_cgroup_holds_the_chroot_that_names_it() { + // The cgroup is torn down first, and a removal it could not finish + // raises this. Removing the chroot anyway would leave nothing for a + // later sweep to find the cgroup by. + let (_dir, state) = state_in_tmpdir(); + let reclaim_failed = ReclaimFailed::default(); + let jail = JailDir::create(&state, &vm_id(), reclaim_failed.clone()).unwrap(); + fs::write(jail.root().join("rootfs.ext4"), b"guest").unwrap(); + + reclaim_failed.set(); + drop(jail); + + assert!( + state.jail_dir(&vm_id()).exists(), + "the chroot names the cgroup that still has to be removed" + ); + } + #[test] fn drop_tolerates_an_already_removed_tree() { let (_dir, state) = state_in_tmpdir(); diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 0bcf27624..785c64713 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -192,6 +192,18 @@ pub struct HostPreparation { pub struct ReclaimFailed(Arc); impl ReclaimFailed { + /// A signal nothing reads, for a cgroup no sweep can find again. + /// + /// A non-sandboxed run has no chroot, and its cgroup is not named by any + /// directory the sweep walks, so there is no handle for a later sweep to + /// work from and nothing a raised signal could change. Named rather than + /// defaulted so the call site says which of the two it is. + #[cfg(target_os = "linux")] + #[must_use] + pub fn unwatched() -> Self { + Self::default() + } + /// Record that a jail could not be reclaimed. pub fn set(&self) { self.0.store(true, Ordering::SeqCst); @@ -204,6 +216,17 @@ impl ReclaimFailed { fn take(&self) -> bool { self.0.swap(false, Ordering::SeqCst) } + + /// Whether the signal is set, without consuming it. + /// + /// Read by the chroot teardown, which must not remove a directory whose + /// cgroup is still there: the directory name is the only handle a later + /// sweep has for finding that cgroup again. + #[cfg(target_os = "linux")] + #[must_use] + pub(crate) fn is_set(&self) -> bool { + self.0.load(Ordering::SeqCst) + } } impl HostPreparation { diff --git a/plus/bencher_runner/src/local_isolation.rs b/plus/bencher_runner/src/local_isolation.rs index 86ca7e929..753806a0e 100644 --- a/plus/bencher_runner/src/local_isolation.rs +++ b/plus/bencher_runner/src/local_isolation.rs @@ -59,7 +59,10 @@ impl LocalIsolation { let benchmark = layout.benchmark.clone(); let run_id = crate::jail::VmId::from_chroot_name(format!("local-{}", uuid::Uuid::new_v4())); - let cgroup = match crate::jail::CgroupManager::new(&run_id) { + // No chroot names this cgroup, and no sweep walks it, so a teardown it + // cannot finish has nothing to hand the work to. + let reclaim_failed = crate::jail::ReclaimFailed::unwatched(); + let cgroup = match crate::jail::CgroupManager::new(&run_id, reclaim_failed) { Ok(cgroup) => { // Best effort here, unlike the sandboxed path: a local run // makes no confinement claim to begin with, so losing the diff --git a/plus/bencher_runner/src/vm.rs b/plus/bencher_runner/src/vm.rs index 1ea730607..1d6002e9f 100644 --- a/plus/bencher_runner/src/vm.rs +++ b/plus/bencher_runner/src/vm.rs @@ -9,7 +9,8 @@ use camino::{Utf8Path, Utf8PathBuf}; use crate::error::RunnerError; use crate::jail::{ - HostPreparation, JailDir, JailLock, JailPaths, StateDir, VmId, chroot, netns, state, + HostPreparation, JailDir, JailLock, JailPaths, ReclaimFailed, StateDir, VmId, chroot, netns, + state, }; use crate::run::{RunOutput, prepare_oci_workspace}; @@ -120,7 +121,15 @@ pub fn vm_execute( chroot::grant_jail_read(kernel_dest)?; // Step 7-8: Build Firecracker config and run the microVM - let fc_config = build_firecracker_config(config, work_dir, vm_id, &state_dir, jail, netns)?; + let fc_config = build_firecracker_config( + config, + work_dir, + vm_id, + &state_dir, + jail, + netns, + host.reclaim_signal(), + )?; let run_output = run_firecracker(&fc_config, cancel_flag)?; @@ -135,6 +144,7 @@ fn build_firecracker_config( state_dir: &StateDir, jail: JailPaths, netns: Utf8PathBuf, + reclaim_failed: ReclaimFailed, ) -> Result { // The jailer copies `--exec-file` into the chroot itself and rejects a // multiply linked file, so Firecracker is staged outside the jail and is @@ -181,6 +191,7 @@ fn build_firecracker_config( jail_user: config.jail_user, chroot_base_dir: state_dir.chroot_base(), netns, + reclaim_failed, vcpus, memory_mib, boot_args: config.kernel_cmdline.clone(), From 0f9d2ef3e8a31866a8080833ca242badfbf6a162 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:07:58 +0000 Subject: [PATCH 42/91] Refuse to build a jail without root, and say so Sandboxed Jobs need a root Runner, which is the release's breaking change, but only the test harnesses said so. The product left an operator upgrading to infer it from a permission error on the state directory or a bare EPERM out of `unshare`, neither of which mentions root or the flag that avoids needing it. Host preparation now checks the effective uid first and reports what the release notes report, including the `--danger-allow-no-sandbox` escape hatch and what giving up the sandbox actually costs. The check goes where the requirement is, not at startup: a Runner that serves only non-sandboxed Specs still comes up unprivileged, and nothing latches, so every Job that needs a jail says it again rather than only the first one. The uid is a parameter of the preparation so the tests can reach what follows it without running as root, the same way the sweep's reap already is. --- plus/bencher_runner/src/error.rs | 11 +++ plus/bencher_runner/src/jail/mod.rs | 135 +++++++++++++++++++++++----- 2 files changed, 126 insertions(+), 20 deletions(-) diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index 632f5c985..ba9e7103c 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -104,6 +104,17 @@ pub enum JailError { #[error("The network namespace thread panicked")] NetnsThread, + #[cfg(target_os = "linux")] + #[error( + "A Runner executing sandboxed Jobs must run as root, but this one is running as uid {euid}. \ + Building the sandbox needs privileges that enabling KVM does not grant: mknod for the chroot's /dev/kvm, \ + chown to hand the guest images to the sandbox user, pivot_root, and setns to join a network namespace. \ + A world-readable /dev/kvm is enough to use KVM unprivileged but not to build the sandbox around it. \ + Start the Runner as root, or start it with --danger-allow-no-sandbox and assign it only Specs with no Sandbox, \ + which gives up the microVM itself and not just its confinement." + )] + NotRoot { euid: u32 }, + #[cfg(target_os = "linux")] #[error("Failed to open the jail lock {path}: {source}")] OpenJailLock { diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 785c64713..7d8317953 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -257,18 +257,36 @@ impl HostPreparation { /// needing a restart. That matters most for a sweep that could not reclaim /// a stale cgroup: the orphan may simply not have exited yet, and a /// latched failure would leave every later job failing with no retry. + /// + /// Requires root, and says so by name rather than letting the operator + /// infer it from a permission error several layers down. #[cfg(target_os = "linux")] pub fn ensure( &mut self, state_dir: &camino::Utf8Path, jail_user: JailUser, + ) -> Result<(), crate::error::JailError> { + self.ensure_as(current_euid(), state_dir, jail_user) + } + + /// Prepare the host, with the effective uid supplied. + /// + /// The uid is a parameter for the same reason the sweep's reap is one: the + /// check refuses every uid but root, so a test that had to be root to reach + /// anything past it would be exercising the harness rather than this. + #[cfg(target_os = "linux")] + fn ensure_as( + &mut self, + euid: u32, + state_dir: &camino::Utf8Path, + jail_user: JailUser, ) -> Result<(), crate::error::JailError> { // A jail that could not be reclaimed earns another sweep, whatever // this process has already done. if self.prepared && !self.reclaim_failed.take() { return Ok(()); } - prepare_host(state_dir, jail_user)?; + prepare_host(euid, state_dir, jail_user)?; self.prepared = true; Ok(()) } @@ -299,9 +317,15 @@ impl HostPreparation { #[cfg(target_os = "linux")] #[expect(clippy::print_stdout, reason = "host preparation reports what it did")] fn prepare_host( + euid: u32, state_dir: &camino::Utf8Path, jail_user: JailUser, ) -> Result<(), crate::error::JailError> { + // Checked first, and by name. Without it the most likely upgrade failure + // surfaces as a permission error on a directory, or a bare EPERM out of + // `unshare`, neither of which mentions root or the flag that avoids it. + check_root(euid)?; + let state = StateDir::new(state_dir.to_owned()); state.create()?; @@ -317,6 +341,32 @@ fn prepare_host( Ok(()) } +/// Refuse to build a jail without the privileges building one needs. +/// +/// The failure an operator actually hits on upgrade is this one, so it says +/// what the release notes say rather than leaving them to infer it from a +/// permission error several layers down. +#[cfg(target_os = "linux")] +fn check_root(euid: u32) -> Result<(), crate::error::JailError> { + if euid == 0 { + Ok(()) + } else { + Err(crate::error::JailError::NotRoot { euid }) + } +} + +/// The effective uid of this process. +#[cfg(target_os = "linux")] +#[expect( + unsafe_code, + reason = "geteuid has no std wrapper and cannot fail or touch memory" +)] +fn current_euid() -> u32 { + // SAFETY: `geteuid` takes no arguments, returns a plain integer, and is + // documented as always succeeding. + unsafe { libc::geteuid() } +} + /// Warn when the jail uid or gid belongs to a named account. /// /// The jailer needs no passwd entry, so a name resolving here is the cheap @@ -473,11 +523,54 @@ impl ResourceLimits { } } -#[cfg(test)] +// Everything the jail prepares is Linux-only, and so is every test of it. +#[cfg(all(test, target_os = "linux"))] mod tests { use super::*; - #[cfg(target_os = "linux")] + /// The only uid that can build a jail. + /// + /// Supplied rather than inherited from the test process, which is usually + /// not root and would otherwise never reach the preparation being tested. + const ROOT_EUID: u32 = 0; + + #[test] + fn a_runner_that_is_not_root_is_refused_by_name() { + // The upgrade failure an operator actually hits. Without this it + // surfaces as a permission error on a directory, or a bare EPERM out of + // `unshare`, neither of which mentions root or the flag that avoids it. + let dir = tempfile::tempdir().unwrap(); + let root = camino::Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let state_dir = root.join("state"); + + let mut host = HostPreparation::new(); + // Nothing latches, so every job says it again rather than only the + // first one. + for attempt in 1..=3 { + let err = host + .ensure_as(1000, &state_dir, JailUser::default()) + .unwrap_err(); + let message = err.to_string(); + assert!( + message.contains("1000"), + "attempt {attempt} must name the uid: {message}" + ); + assert!( + message.contains("root"), + "attempt {attempt} must name root: {message}" + ); + assert!( + message.contains("--danger-allow-no-sandbox"), + "attempt {attempt} must name the escape hatch: {message}" + ); + } + + assert!( + !state_dir.exists(), + "a refused runner must not have touched the state directory" + ); + } + #[test] fn the_default_jail_user_is_outside_the_allocated_ranges() { // systemd-homed takes 60001-60513 and DynamicUser takes 61184-65519. @@ -489,7 +582,6 @@ mod tests { } } - #[cfg(target_os = "linux")] #[test] fn preparation_is_lazy_and_happens_at_most_once() { // A daemon that prepared at startup would need root just to come up, @@ -501,26 +593,29 @@ mod tests { assert!(!state_dir.exists(), "startup has not prepared anything"); let mut host = HostPreparation::new(); - host.ensure(&state_dir, JailUser::default()).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); assert!(state_dir.join("jail").is_dir(), "the first job prepares"); // A second job must not redo it: proven by removing the tree and // seeing that it is not rebuilt. Owning the token is what makes this // independent of every other test in the process. std::fs::remove_dir_all(&state_dir).unwrap(); - host.ensure(&state_dir, JailUser::default()).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); assert!(!state_dir.exists(), "preparation happens at most once"); // A different runner prepares its own host. let mut other = HostPreparation::new(); - other.ensure(&state_dir, JailUser::default()).unwrap(); + other + .ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); assert!( state_dir.join("jail").is_dir(), "a fresh token prepares again" ); } - #[cfg(target_os = "linux")] #[test] fn a_failure_is_not_latched_and_self_heals() { // Fatal to the job, not to the runner. A host that cannot be prepared @@ -535,18 +630,19 @@ mod tests { let mut host = HostPreparation::new(); for attempt in 1..=3 { assert!( - host.ensure(&state_dir, JailUser::default()).is_err(), + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .is_err(), "attempt {attempt} must fail" ); } // Remove the cause and the very next job succeeds, with no restart. std::fs::remove_dir(state_dir.join("someone-elses-data")).unwrap(); - host.ensure(&state_dir, JailUser::default()).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); assert!(state_dir.join("jail").is_dir()); } - #[cfg(target_os = "linux")] #[test] fn a_jail_that_could_not_be_reclaimed_earns_another_sweep() { // The sweep otherwise runs once per process, so a teardown that failed @@ -558,16 +654,19 @@ mod tests { let state_dir = root.join("state"); let mut host = HostPreparation::new(); - host.ensure(&state_dir, JailUser::default()).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); // Already prepared: a second job does not redo the work. std::fs::remove_dir_all(&state_dir).unwrap(); - host.ensure(&state_dir, JailUser::default()).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); assert!(!state_dir.exists(), "preparation happens at most once"); // A jail that could not be reclaimed changes that. host.reclaim_signal().set(); - host.ensure(&state_dir, JailUser::default()).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); assert!( state_dir.join("jail").is_dir(), "a failed teardown must earn another sweep" @@ -575,11 +674,11 @@ mod tests { // And the signal is consumed, not sticky. std::fs::remove_dir_all(&state_dir).unwrap(); - host.ensure(&state_dir, JailUser::default()).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); assert!(!state_dir.exists(), "the signal is consumed once"); } - #[cfg(target_os = "linux")] #[test] fn the_jail_user_rejects_root() { // Untrusted code against a root VMM is the one thing the confinement @@ -593,7 +692,6 @@ mod tests { assert_eq!(user.gid(), 5678); } - #[cfg(target_os = "linux")] #[test] fn the_default_jail_user_is_unprivileged() { let default = JailUser::default(); @@ -602,7 +700,6 @@ mod tests { JailUser::new(default.uid(), default.gid()).unwrap(); } - #[cfg(target_os = "linux")] #[test] fn a_named_account_is_found_by_id() { let passwd = "root:x:0:0:root:/root:/bin/bash\nbuild:x:61016:61016:CI build user:/home/build:/bin/sh\n"; @@ -611,7 +708,6 @@ mod tests { assert_eq!(lookup_name_in(passwd, 0).as_deref(), Some("root")); } - #[cfg(target_os = "linux")] #[test] fn an_unallocated_id_has_no_name() { let passwd = @@ -620,7 +716,6 @@ mod tests { assert_eq!(lookup_name_in(passwd, 61016), None); } - #[cfg(target_os = "linux")] #[test] fn malformed_records_are_skipped() { let passwd = "\nnot-a-record\nshort:x\nbuild:x:61016:61016::/home/build:/bin/sh\n"; From 9d01ecd543c400f6f5c8ae71740fde4675f265be Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:08:14 +0000 Subject: [PATCH 43/91] Delete the resource limits nothing applies `ResourceLimits` and the cgroup writes that consumed it had no caller outside their own tests. Nothing built one: the VM cgroup is created for cpuset placement and swap control, the memory and vCPU bounds the microVM actually has come from the Firecracker machine config, and the guest cannot exceed them from inside. Keeping a second, unreachable way to bound the same things invited the belief that jobs were being limited by it, and its tests read as coverage of a live mechanism. The I/O throttling and block device discovery go with it, along with the `add_self` placement that the pre-exec write replaced. --- plus/bencher_runner/src/jail/cgroup.rs | 104 +-------------- plus/bencher_runner/src/jail/mod.rs | 170 ------------------------- plus/bencher_runner/src/lib.rs | 5 +- 3 files changed, 2 insertions(+), 277 deletions(-) diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index bc79b69a1..0187e9eac 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -9,7 +9,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use crate::RunnerError; use crate::cpu::CpuLayout; use crate::error::JailError; -use crate::jail::{ReclaimFailed, ResourceLimits, VmId}; +use crate::jail::{ReclaimFailed, VmId}; /// Default cgroup v2 mount point. const CGROUP_ROOT: &str = "/sys/fs/cgroup"; @@ -123,40 +123,6 @@ impl CgroupManager { Ok(()) } - /// Apply resource limits to this cgroup. - pub fn apply_limits(&self, limits: &ResourceLimits) -> Result<(), RunnerError> { - // CPU limit - if let Some(quota) = limits.cpu_quota_us { - let cpu_max = format!("{quota} {}", limits.cpu_period_us); - self.write_file("cpu.max", &cpu_max)?; - } - - // Memory limit - if let Some(bytes) = limits.memory_bytes { - self.write_file("memory.max", &bytes.to_string())?; - - // Disable swap to ensure benchmark memory measurements are accurate - // and to prevent swap thrashing from affecting benchmark results. - drop(self.disable_swap()); - } - - // OOM group kill: when the cgroup hits its memory limit, kill ALL processes - // in the group together. This prevents partial kills that leave orphan processes. - drop(self.write_file("memory.oom.group", "1")); - - // PIDs limit - self.write_file("pids.max", &limits.max_procs.to_string())?; - - // I/O limits - applied to all block devices - // Note: This requires knowing the device major:minor. We attempt to - // discover common devices, but this may not work in all configuration. - if limits.io_read_bps.is_some() || limits.io_write_bps.is_some() { - self.apply_io_limits(limits); - } - - Ok(()) - } - /// Apply CPU pinning via cpuset controller. /// /// Restricts processes in this cgroup to run only on the specified CPUs. @@ -261,68 +227,6 @@ impl CgroupManager { Ok(Cpuset::Applied) } - /// Apply I/O bandwidth limits. - /// - /// Attempts to apply io.max limits to discovered block devices. - /// The io.max format is: "MAJ:MIN rbps=BYTES wbps=BYTES" - fn apply_io_limits(&self, limits: &ResourceLimits) { - use std::fmt::Write as _; - - // Try to find block devices to apply limits to - let devices = Self::discover_block_devices(); - - if devices.is_empty() { - // No devices found, skip I/O limits silently - return; - } - - let read_limit = limits - .io_read_bps - .map_or("max".to_owned(), |v| v.to_string()); - let write_limit = limits - .io_write_bps - .map_or("max".to_owned(), |v| v.to_string()); - - let mut io_max_content = String::new(); - for (major, minor) in devices { - // Format: "MAJ:MIN rbps=BYTES wbps=BYTES" - let _unused = writeln!( - io_max_content, - "{major}:{minor} rbps={read_limit} wbps={write_limit}" - ); - } - - // Try to write io.max - may fail if io controller is not available - let path = self.cgroup_path.join("io.max"); - if let Err(e) = fs::write(&path, &io_max_content) { - // Log warning but don't fail - io controller may not be available - eprintln!("Warning: failed to set io.max (io controller may not be available): {e}"); - } - } - - /// Discover block devices on the system. - /// - /// Returns a list of (major, minor) device numbers for block devices. - fn discover_block_devices() -> Vec<(u32, u32)> { - let mut devices = Vec::new(); - - // Try to read /sys/block to find block devices - if let Ok(entries) = fs::read_dir("/sys/block") { - for entry in entries.flatten() { - let dev_path = entry.path().join("dev"); - if let Ok(content) = fs::read_to_string(&dev_path) - && let Some((major_str, minor_str)) = content.trim().split_once(':') - && let (Ok(major), Ok(minor)) = - (major_str.parse::(), minor_str.parse::()) - { - devices.push((major, minor)); - } - } - } - - devices - } - /// Disable swap for this cgroup. /// /// Keeps benchmark memory resident: swap thrashing adds run-to-run @@ -331,12 +235,6 @@ impl CgroupManager { self.write_file("memory.swap.max", "0") } - /// Add the current process to this cgroup. - pub fn add_self(&self) -> Result<(), RunnerError> { - let pid = std::process::id(); - self.write_file("cgroup.procs", &pid.to_string()) - } - /// Open this cgroup's `cgroup.procs` for writing. /// /// The descriptor is opened before the fork so the `pre_exec` closure that diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 7d8317953..64a030052 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -37,8 +37,6 @@ pub use state::StateDir; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use serde::{Deserialize, Serialize}; - /// Default location of the runner's persistent state directory. pub const DEFAULT_STATE_DIR: &str = "/var/lib/bencher-runner"; @@ -431,98 +429,6 @@ fn lookup_name_in(database: &str, id: u32) -> Option { }) } -/// Resource limits for the Firecracker microVM process. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ResourceLimits { - /// Maximum CPU time in microseconds per second. - /// E.g., 100000 = 100ms per 100ms period = 1 full CPU. - #[serde(default)] - pub cpu_quota_us: Option, - - /// CPU period in microseconds (default: 100000 = 100ms). - #[serde(default = "default_cpu_period")] - pub cpu_period_us: u64, - - /// Maximum memory in bytes. - #[serde(default)] - pub memory_bytes: Option, - - /// Maximum number of open file descriptors. - #[serde(default = "default_max_fds")] - pub max_fds: u64, - - /// Maximum number of processes/threads. - #[serde(default = "default_max_procs")] - pub max_procs: u64, - - /// Maximum I/O read bandwidth in bytes per second. - /// Applied via cgroup v2 io.max. - #[serde(default)] - pub io_read_bps: Option, - - /// Maximum I/O write bandwidth in bytes per second. - /// Applied via cgroup v2 io.max. - #[serde(default)] - pub io_write_bps: Option, -} - -const fn default_cpu_period() -> u64 { - 100_000 // 100ms -} - -const fn default_max_fds() -> u64 { - 1024 -} - -const fn default_max_procs() -> u64 { - 64 -} - -impl Default for ResourceLimits { - fn default() -> Self { - Self { - cpu_quota_us: None, - cpu_period_us: default_cpu_period(), - memory_bytes: None, - max_fds: default_max_fds(), - max_procs: default_max_procs(), - io_read_bps: None, - io_write_bps: None, - } - } -} - -impl ResourceLimits { - /// Set CPU limit as a fraction of CPUs (e.g., 0.5 = half a CPU, 2.0 = 2 CPUs). - #[must_use] - #[expect( - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - clippy::cast_precision_loss, - reason = "CPU fraction to microsecond quota conversion" - )] - pub fn with_cpu_limit(mut self, cpus: f64) -> Self { - let quota = (cpus * self.cpu_period_us as f64) as u64; - self.cpu_quota_us = Some(quota); - self - } - - /// Set memory limit in bytes. - #[must_use] - pub fn with_memory_limit(mut self, bytes: u64) -> Self { - self.memory_bytes = Some(bytes); - self - } - - /// Set I/O bandwidth limits in bytes per second. - #[must_use] - pub fn with_io_limits(mut self, read_bps: u64, write_bps: u64) -> Self { - self.io_read_bps = Some(read_bps); - self.io_write_bps = Some(write_bps); - self - } -} - // Everything the jail prepares is Linux-only, and so is every test of it. #[cfg(all(test, target_os = "linux"))] mod tests { @@ -723,80 +629,4 @@ mod tests { assert_eq!(lookup_name_in(passwd, 61016).as_deref(), Some("build")); assert_eq!(lookup_name_in("", 61016), None); } - - #[test] - fn resource_limits_defaults() { - let limits = ResourceLimits::default(); - assert_eq!(limits.cpu_quota_us, None); - assert_eq!(limits.cpu_period_us, 100_000); - assert_eq!(limits.memory_bytes, None); - assert_eq!(limits.max_fds, 1024); - assert_eq!(limits.max_procs, 64); - assert_eq!(limits.io_read_bps, None); - assert_eq!(limits.io_write_bps, None); - } - - #[test] - fn with_cpu_limit_one_cpu() { - let limits = ResourceLimits::default().with_cpu_limit(1.0); - assert_eq!(limits.cpu_quota_us, Some(100_000)); - } - - #[test] - fn with_cpu_limit_half_cpu() { - let limits = ResourceLimits::default().with_cpu_limit(0.5); - assert_eq!(limits.cpu_quota_us, Some(50_000)); - } - - #[test] - fn with_cpu_limit_two_cpus() { - let limits = ResourceLimits::default().with_cpu_limit(2.0); - assert_eq!(limits.cpu_quota_us, Some(200_000)); - } - - #[test] - fn with_memory_limit() { - let limits = ResourceLimits::default().with_memory_limit(1024 * 1024 * 512); - assert_eq!(limits.memory_bytes, Some(0x2000_0000)); - } - - #[test] - fn with_io_limits() { - let limits = ResourceLimits::default().with_io_limits(1_000_000, 500_000); - assert_eq!(limits.io_read_bps, Some(1_000_000)); - assert_eq!(limits.io_write_bps, Some(500_000)); - } - - #[test] - fn builder_chain() { - let limits = ResourceLimits::default() - .with_cpu_limit(2.0) - .with_memory_limit(1024) - .with_io_limits(100, 200); - assert_eq!(limits.cpu_quota_us, Some(200_000)); - assert_eq!(limits.memory_bytes, Some(1024)); - assert_eq!(limits.io_read_bps, Some(100)); - assert_eq!(limits.io_write_bps, Some(200)); - } - - #[test] - fn serde_round_trip() { - let limits = ResourceLimits::default() - .with_cpu_limit(1.5) - .with_memory_limit(2048); - let json = serde_json::to_string(&limits).unwrap(); - let parsed: ResourceLimits = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed.cpu_quota_us, limits.cpu_quota_us); - assert_eq!(parsed.memory_bytes, Some(2048)); - assert_eq!(parsed.cpu_period_us, 100_000); - } - - #[test] - fn serde_deserialize_minimal() { - let json = "{}"; - let limits: ResourceLimits = serde_json::from_str(json).unwrap(); - assert_eq!(limits.cpu_quota_us, None); - assert_eq!(limits.cpu_period_us, 100_000); - assert_eq!(limits.max_procs, 64); - } } diff --git a/plus/bencher_runner/src/lib.rs b/plus/bencher_runner/src/lib.rs index 0fae74702..5fd12296e 100644 --- a/plus/bencher_runner/src/lib.rs +++ b/plus/bencher_runner/src/lib.rs @@ -62,10 +62,7 @@ pub use config::Config; #[cfg(feature = "plus")] pub use error::{ConfigError, ExecutionError, JailError, RunnerError}; #[cfg(feature = "plus")] -pub use jail::{ - DEFAULT_JAIL_GID, DEFAULT_JAIL_UID, DEFAULT_STATE_DIR, HostPreparation, JailUser, - ResourceLimits, -}; +pub use jail::{DEFAULT_JAIL_GID, DEFAULT_JAIL_UID, DEFAULT_STATE_DIR, HostPreparation, JailUser}; #[cfg(feature = "plus")] pub use log_level::SandboxLogLevel; #[cfg(feature = "plus")] From e8af769b616450a7a857dbbb52ee88c8976ea5d9 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:08:24 +0000 Subject: [PATCH 44/91] Refuse a state directory whose contents cannot be read The guard that keeps the runner from chmodding a directory it does not own treated every `read_dir` failure as "not ours yet, so create it", which is the one conclusion a failed read does not support. A path that is really a file, or a directory whose contents cannot be listed, fell through to the 0700 tightening that the guard exists to prevent. Absence is now the only failure that means the directory is the runner's to create; anything else is reported, since the runner cannot establish ownership. --- plus/bencher_runner/src/error.rs | 8 ++++++ plus/bencher_runner/src/jail/state.rs | 39 ++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index ba9e7103c..1cfcfcf79 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -75,6 +75,14 @@ pub enum JailError { source: std::io::Error, }, + #[error( + "Failed to read the runner state directory {path}: {source}. The runner cannot tell whether this directory is its own, and it will not tighten the permissions of one that might not be." + )] + ReadStateDir { + path: Utf8PathBuf, + source: std::io::Error, + }, + #[cfg(target_os = "linux")] #[error("Failed to create network namespace directory {path}: {source}")] NetnsDir { diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index e312525a5..39999aa52 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -57,11 +57,24 @@ impl StateDir { /// populated one is only ours if it already carries something the runner /// put there. Without this, `--state-dir /var/lib` would be chmodded to /// 0700 and take the host down with it. + /// + /// A read that fails for any reason other than absence is refused rather + /// than treated as an empty directory. It is not evidence that the + /// directory is empty, and the chmod that follows is the thing this guard + /// exists to keep off a directory that is not the runner's: a path that is + /// really a file, or one whose contents cannot be listed, would otherwise + /// be taken on the strength of a failed check. fn check_root_is_ours(&self) -> Result<(), JailError> { - let Ok(entries) = fs::read_dir(&self.root) else { - // Missing, or unreadable: creating it is the next step and will - // report the real error. - return Ok(()); + let entries = match fs::read_dir(&self.root) { + Ok(entries) => entries, + // Missing: creating it is the next step. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => { + return Err(JailError::ReadStateDir { + path: self.root.clone(), + source: e, + }); + }, }; let mut populated = false; for entry in entries.flatten() { @@ -341,6 +354,24 @@ mod tests { assert_ne!(mode & 0o777, 0o700, "a refused root must not be chmodded"); } + #[test] + fn a_root_that_cannot_be_read_is_not_assumed_to_be_ours() { + // A failed read is not an empty directory. A file where the state + // directory should be reads back `ENOTDIR`, the same way an unlistable + // directory reads back `EACCES`, and neither says the path is the + // runner's to chmod. + let (_dir, root) = temp_root(); + let not_a_dir = root.join("state"); + fs::write(¬_a_dir, b"operator note").unwrap(); + + let err = StateDir::new(not_a_dir).create().unwrap_err(); + + assert!( + matches!(err, JailError::ReadStateDir { .. }), + "a read that failed is reported, not swallowed: {err}" + ); + } + #[test] fn an_empty_directory_is_ours_to_take() { let (_dir, root) = temp_root(); From a629391dbd78477ee787abe264ee8bf772f600fa Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:08:41 +0000 Subject: [PATCH 45/91] Refuse a jail whose scan and kill disagree The bounded rescan is a guard on a loop that should run once, not a verdict, and reaching the end of it was reported without comment. A healthy jail holds exactly one process: Firecracker does not fork and the jailer execs in place. So exhausting the bound means either the scan disagrees with the kill, which is a bug that should be surfaced where a maintainer will look, or something is spawning into the jail, which is precisely when the sweep must refuse rather than proceed. Neither says the jail is clear, so it is reported as still running and the reason is said out loud. Nothing else would catch it. The cgroup removal that follows in the sweep refuses while anything is still in the cgroup, but a run with no CPU layout, or one whose host does not delegate cpuset, has no cgroup at all. Failing the job is recoverable, since nothing latches and the next job sweeps again; measuring through a jail that is still occupied is not, and that is the outcome this module exists to prevent. The scan and the kill become parameters so both paths are tested without manufacturing sixty-four jailed processes. --- plus/bencher_runner/src/jail/reap.rs | 116 +++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 5 deletions(-) diff --git a/plus/bencher_runner/src/jail/reap.rs b/plus/bencher_runner/src/jail/reap.rs index e9764a7fb..201f42722 100644 --- a/plus/bencher_runner/src/jail/reap.rs +++ b/plus/bencher_runner/src/jail/reap.rs @@ -69,6 +69,19 @@ pub enum Reaped { /// about what it reports, because the caller decides whether to delete a /// directory based on the answer. pub fn reap_jailed_vmm(jail_root: &Utf8Path) -> Reaped { + reap_jailed_vmm_with(jail_root, find_jailed_vmm, reap_one) +} + +/// The reap, with the scan and the kill injectable. +/// +/// Both reach into `/proc` and signal processes on the machine running the +/// tests, so the exhaustion path is exercised through parameters rather than by +/// manufacturing sixty-four jailed processes. +fn reap_jailed_vmm_with(jail_root: &Utf8Path, find: F, reap: R) -> Reaped +where + F: Fn(&Utf8Path) -> Option, + R: Fn(u32, &Utf8Path) -> Reaped, +{ // Rescan after each reap rather than assuming one process per jail. That // assumption holds today, since neither `--daemonize` nor `--new-pid-ns` // is passed and the jailer execs in place as a single process, but the @@ -76,17 +89,34 @@ pub fn reap_jailed_vmm(jail_root: &Utf8Path) -> Reaped { // load-bearing is worth enforcing rather than trusting, and a survivor // would otherwise have the tree removed out from under it. for _ in 0..MAX_JAILED_PROCESSES { - let Some(pid) = find_jailed_vmm(jail_root) else { + let Some(pid) = find(jail_root) else { return Reaped::Clear; }; - if let Reaped::StillRunning { pid } = reap_one(pid, jail_root) { + if let Reaped::StillRunning { pid } = reap(pid, jail_root) { return Reaped::StillRunning { pid }; } } - // Something keeps appearing in this jail. Report it rather than looping. - match find_jailed_vmm(jail_root) { - Some(pid) => Reaped::StillRunning { pid }, + // Exhausting the bound is anomalous by construction: Firecracker does not + // fork, the jailer execs in place, and a healthy jail therefore holds + // exactly one process. Reaching here means either the scan and the kill + // disagree, which is a bug and the likelier of the two, or something is + // spawning into this jail, which is precisely when the sweep must refuse. + // Neither says the jail is clear, so it is not reported clear. + // + // Nothing else would catch it. The cgroup removal that follows in the sweep + // refuses while anything is still in the cgroup, but a run with no CPU + // layout, or one whose host does not delegate cpuset, has no cgroup at all. + // Failing the job is recoverable, because nothing latches and the next job + // sweeps again; measuring through a jail that is still occupied is not, and + // that is the one outcome this module exists to prevent. + match find(jail_root) { + Some(pid) => { + eprintln!( + "Warning: gave up scanning {jail_root} after {MAX_JAILED_PROCESSES} passes; pid {pid} still matches it while every reap reported the jail clear." + ); + Reaped::StillRunning { pid } + }, None => Reaped::Clear, } } @@ -367,6 +397,82 @@ mod tests { assert_eq!(reap_jailed_vmm(&root), Reaped::Clear); } + #[test] + fn a_survivor_is_reported_without_exhausting_the_scan() { + // A process the reap could not kill is the answer immediately: the + // caller must not delete the tree under it, and there is nothing to be + // gained by rescanning. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let scans = std::cell::Cell::new(0); + + let reaped = reap_jailed_vmm_with( + &root, + |_jail_root| { + scans.set(scans.get() + 1); + Some(99) + }, + |pid, _jail_root| Reaped::StillRunning { pid }, + ); + + assert_eq!(reaped, Reaped::StillRunning { pid: 99 }); + assert_eq!(scans.get(), 1, "a survivor is the answer on the first pass"); + } + + #[test] + fn a_jail_that_keeps_producing_processes_is_never_called_clear() { + // A healthy jail holds one process, so exhausting the bound is either + // the scan disagreeing with the kill or something spawning into the + // jail. Neither is evidence the jail is clear, and reporting it clear + // would have the sweep delete the tree and the next job measure through + // whatever is left. The cgroup removal cannot be leaned on here: a run + // with no CPU layout has no cgroup to refuse. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let reaps = std::cell::Cell::new(0); + + let reaped = reap_jailed_vmm_with( + &root, + |_jail_root| Some(7), + |_pid, _jail_root| { + reaps.set(reaps.get() + 1); + Reaped::Clear + }, + ); + + assert_eq!(reaped, Reaped::StillRunning { pid: 7 }); + assert_eq!( + reaps.get(), + MAX_JAILED_PROCESSES, + "the loop is bounded rather than endless" + ); + } + + #[test] + fn a_jail_that_empties_on_the_last_pass_is_clear() { + // The bound is a guard, not a verdict: a jail that no longer matches + // anything is clear however many passes it took to get there. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let scans = std::cell::Cell::new(0); + + let reaped = reap_jailed_vmm_with( + &root, + |_jail_root| { + scans.set(scans.get() + 1); + (scans.get() <= MAX_JAILED_PROCESSES).then_some(7) + }, + |_pid, _jail_root| Reaped::Clear, + ); + + assert_eq!(reaped, Reaped::Clear); + assert_eq!( + scans.get(), + MAX_JAILED_PROCESSES + 1, + "the scan after the loop is what decides" + ); + } + #[test] fn a_still_running_vmm_carries_its_pid() { // The caller keys the decision not to delete a directory off this, so From 770b83a5733fd8bd997d65e260a14f33c914b5ed Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:08:49 +0000 Subject: [PATCH 46/91] Require an absolute state directory The state directory reaches the jailer as `--chroot-base-dir`, which the jailer resolves against its own working directory rather than the runner's. A relative `--state-dir` therefore builds the chroot somewhere the runner does not look: the sweep never reaches it, the lock does not protect it, and the mismatch shows up as a jail that cannot be found rather than as a bad argument. It is refused where it is parsed, for both `runner run` and `runner up`, and the docs say so. --- .../docs-reference/runner/de/runner-run.mdx | 1 + .../docs-reference/runner/de/runner-up.mdx | 1 + .../docs-reference/runner/en/runner-run.mdx | 1 + .../docs-reference/runner/en/runner-up.mdx | 1 + .../docs-reference/runner/es/runner-run.mdx | 1 + .../docs-reference/runner/es/runner-up.mdx | 1 + .../docs-reference/runner/fr/runner-run.mdx | 1 + .../docs-reference/runner/fr/runner-up.mdx | 1 + .../docs-reference/runner/ja/runner-run.mdx | 1 + .../docs-reference/runner/ja/runner-up.mdx | 1 + .../docs-reference/runner/ko/runner-run.mdx | 1 + .../docs-reference/runner/ko/runner-up.mdx | 1 + .../docs-reference/runner/pt/runner-run.mdx | 1 + .../docs-reference/runner/pt/runner-up.mdx | 1 + .../docs-reference/runner/ru/runner-run.mdx | 1 + .../docs-reference/runner/ru/runner-up.mdx | 1 + .../docs-reference/runner/zh/runner-run.mdx | 1 + .../docs-reference/runner/zh/runner-up.mdx | 1 + services/runner/src/parser/mod.rs | 45 ++++++++++++++++++- services/runner/src/parser/up.rs | 9 +++- 20 files changed, 68 insertions(+), 4 deletions(-) diff --git a/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx index 949b052f0..fa25b0939 100644 --- a/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx @@ -35,6 +35,7 @@ Standardmäßig wird `300` verwendet. ### `--state-dir ` Das persistente Zustandsverzeichnis für den Runner. +Es muss ein absoluter Pfad sein. Das Jail, das die Sandbox einschließt, wird unter diesem Verzeichnis erstellt, und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor dem ersten Sandbox-Job daraus entfernt. Standardmäßig wird `/var/lib/bencher-runner` verwendet. diff --git a/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx index e0e0bfc9a..0c69da179 100644 --- a/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx @@ -38,6 +38,7 @@ Standardmäßig wird `55` verwendet. ### `--state-dir ` Das persistente Zustandsverzeichnis für den Runner. +Es muss ein absoluter Pfad sein. Das Jail, das die Sandbox einschließt, wird unter diesem Verzeichnis erstellt, und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor dem ersten Sandbox-Job daraus entfernt. Standardmäßig wird `/var/lib/bencher-runner` verwendet. diff --git a/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx index 4addb2cc8..30232dd14 100644 --- a/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx @@ -35,6 +35,7 @@ By default, `300` is used. ### `--state-dir ` The persistent state directory for the Runner. +It must be an absolute path. The jail that confines the sandbox is built under this directory, and jails left behind by an unclean exit are swept from it before the first sandboxed Job runs. By default, `/var/lib/bencher-runner` is used. diff --git a/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx index 310463bee..82759f086 100644 --- a/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx @@ -38,6 +38,7 @@ By default, `55` is used. ### `--state-dir ` The persistent state directory for the Runner. +It must be an absolute path. The jail that confines the sandbox is built under this directory, and jails left behind by an unclean exit are swept from it before the first sandboxed Job runs. By default, `/var/lib/bencher-runner` is used. diff --git a/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx index a02fc9067..36e9b2892 100644 --- a/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx @@ -35,6 +35,7 @@ Por defecto, se usa `300`. ### `--state-dir ` El directorio de estado persistente del Runner. +Debe ser una ruta absoluta. La jaula que confina el sandbox se crea dentro de este directorio, y las jaulas que quedaron tras una salida no limpia se eliminan de él antes del primer Job con sandbox. Por defecto, se usa `/var/lib/bencher-runner`. diff --git a/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx index 874610834..f4de47880 100644 --- a/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx @@ -38,6 +38,7 @@ Por defecto, se usa `55`. ### `--state-dir ` El directorio de estado persistente del Runner. +Debe ser una ruta absoluta. La jaula que confina el sandbox se crea dentro de este directorio, y las jaulas que quedaron tras una salida no limpia se eliminan de él antes del primer Job con sandbox. Por defecto, se usa `/var/lib/bencher-runner`. diff --git a/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx index 9148efa52..6e9ce85bf 100644 --- a/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx @@ -35,6 +35,7 @@ Par défaut, `300` est utilisé. ### `--state-dir ` Le répertoire d'état persistant du Runner. +Il doit s'agir d'un chemin absolu. La prison qui confine le bac à sable est créée dans ce répertoire, et les prisons laissées par un arrêt brutal en sont supprimées avant le premier Job avec bac à sable. Par défaut, `/var/lib/bencher-runner` est utilisé. diff --git a/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx index f0608d490..81c17282f 100644 --- a/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx @@ -38,6 +38,7 @@ Par défaut, `55` est utilisé. ### `--state-dir ` Le répertoire d'état persistant du Runner. +Il doit s'agir d'un chemin absolu. La prison qui confine le bac à sable est créée dans ce répertoire, et les prisons laissées par un arrêt brutal en sont supprimées avant le premier Job avec bac à sable. Par défaut, `/var/lib/bencher-runner` est utilisé. diff --git a/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx index 10cc5cfb6..7f132117f 100644 --- a/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx @@ -35,6 +35,7 @@ Firecracker microVM には `firecracker` を使用します (Linux のみ)。 ### `--state-dir ` Runner の永続的な状態ディレクトリ。 +絶対パスである必要があります。 サンドボックスを閉じ込める jail はこのディレクトリの下に作成され、 異常終了で残った jail は、最初のサンドボックス Job の実行前にここから一掃されます。 デフォルトでは `/var/lib/bencher-runner` が使用されます。 diff --git a/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx index 9f9b4a0ad..1d1c666b1 100644 --- a/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx @@ -37,6 +37,7 @@ Job を待機する際のロングポーリングのタイムアウト (秒)。` ### `--state-dir ` Runner の永続的な状態ディレクトリ。 +絶対パスである必要があります。 サンドボックスを閉じ込める jail はこのディレクトリの下に作成され、 異常終了で残った jail は、最初のサンドボックス Job の実行前にここから一掃されます。 デフォルトでは `/var/lib/bencher-runner` が使用されます。 diff --git a/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx index d2636ddd7..32776997c 100644 --- a/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx @@ -35,6 +35,7 @@ Firecracker microVM(Linux 전용)을 사용하려면 `firecracker`를 사용하 ### `--state-dir ` Runner의 영구 상태 디렉터리입니다. +절대 경로여야 합니다. 샌드박스를 가두는 jail이 이 디렉터리 아래에 생성되며, 비정상 종료로 남겨진 jail은 첫 번째 샌드박스 Job 실행 전에 이곳에서 정리됩니다. 기본적으로 `/var/lib/bencher-runner`가 사용됩니다. diff --git a/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx index be67c7ed1..839a2dc98 100644 --- a/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx @@ -38,6 +38,7 @@ Job을 기다리는 동안의 롱 폴(long-poll) 타임아웃(초)으로, `1`에 ### `--state-dir ` Runner의 영구 상태 디렉터리입니다. +절대 경로여야 합니다. 샌드박스를 가두는 jail이 이 디렉터리 아래에 생성되며, 비정상 종료로 남겨진 jail은 첫 번째 샌드박스 Job 실행 전에 이곳에서 정리됩니다. 기본적으로 `/var/lib/bencher-runner`가 사용됩니다. diff --git a/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx index 7be345481..9f7450f9a 100644 --- a/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx @@ -35,6 +35,7 @@ Por padrão, `300` é usado. ### `--state-dir ` O diretório de estado persistente do Runner. +Deve ser um caminho absoluto. A jail que confina o sandbox é criada sob este diretório, e as jails deixadas por um encerramento não limpo são removidas dele antes do primeiro Job com sandbox. Por padrão, `/var/lib/bencher-runner` é usado. diff --git a/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx index d568f8bc1..7b99709dd 100644 --- a/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx @@ -38,6 +38,7 @@ Por padrão, `55` é usado. ### `--state-dir ` O diretório de estado persistente do Runner. +Deve ser um caminho absoluto. A jail que confina o sandbox é criada sob este diretório, e as jails deixadas por um encerramento não limpo são removidas dele antes do primeiro Job com sandbox. Por padrão, `/var/lib/bencher-runner` é usado. diff --git a/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx index 9257dd18d..f20cf689e 100644 --- a/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx @@ -35,6 +35,7 @@ JWT-токен для аутентификации в реестре при за ### `--state-dir ` Постоянный каталог состояния Runner. +Путь должен быть абсолютным. Изолирующая среда (jail), ограничивающая песочницу, создаётся внутри этого каталога, а среды, оставшиеся после аварийного завершения, удаляются из него перед запуском первого Job с песочницей. По умолчанию используется `/var/lib/bencher-runner`. diff --git a/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx index dfe388977..8f3033422 100644 --- a/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx @@ -38,6 +38,7 @@ UUID или slug Runner, от имени которого работать. ### `--state-dir ` Постоянный каталог состояния Runner. +Путь должен быть абсолютным. Изолирующая среда (jail), ограничивающая песочницу, создаётся внутри этого каталога, а среды, оставшиеся после аварийного завершения, удаляются из него перед запуском первого Job с песочницей. По умолчанию используется `/var/lib/bencher-runner`. diff --git a/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx index 2f6bef16d..fbd1058d2 100644 --- a/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx @@ -35,6 +35,7 @@ runner run --image [OPTIONS] ### `--state-dir ` Runner 的持久状态目录。 +必须是绝对路径。 限制沙箱的 jail 在该目录下创建， 非正常退出遗留的 jail 会在第一个沙箱 Job 运行前从中清除。 默认使用 `/var/lib/bencher-runner`。 diff --git a/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx index 3425c651b..8e750c17f 100644 --- a/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx @@ -37,6 +37,7 @@ runner up [OPTIONS] ### `--state-dir ` Runner 的持久状态目录。 +必须是绝对路径。 限制沙箱的 jail 在该目录下创建， 非正常退出遗留的 jail 会在第一个沙箱 Job 运行前从中清除。 默认使用 `/var/lib/bencher-runner`。 diff --git a/services/runner/src/parser/mod.rs b/services/runner/src/parser/mod.rs index 023171cb8..bdf8642e8 100644 --- a/services/runner/src/parser/mod.rs +++ b/services/runner/src/parser/mod.rs @@ -62,8 +62,13 @@ pub struct CliRun { #[arg(long, default_value = "300")] pub timeout: u64, - /// Persistent state directory for the runner. - #[arg(long, env = "BENCHER_STATE_DIR", default_value = bencher_runner::DEFAULT_STATE_DIR)] + /// Persistent state directory for the runner (absolute path). + #[arg( + long, + env = "BENCHER_STATE_DIR", + default_value = bencher_runner::DEFAULT_STATE_DIR, + value_parser = absolute_state_dir, + )] pub state_dir: Utf8PathBuf, /// Unprivileged uid the jailed sandbox process drops to. @@ -142,3 +147,39 @@ pub struct CliRun { #[arg(long, default_value = "warning", requires = "sandbox")] pub sandbox_log_level: bencher_runner::SandboxLogLevel, } + +/// Require an absolute state directory. +/// +/// The path reaches the jailer as `--chroot-base-dir`, which the jailer resolves +/// against its own working directory rather than the runner's, so a relative +/// value builds the chroot somewhere the runner does not look and the sweep +/// never reaches. +#[cfg(feature = "plus")] +fn absolute_state_dir(arg: &str) -> Result { + let path = Utf8PathBuf::from(arg); + if path.is_absolute() { + Ok(path) + } else { + Err(format!( + "the state directory must be an absolute path, and `{path}` is relative" + )) + } +} + +#[cfg(all(test, feature = "plus"))] +mod tests { + use super::absolute_state_dir; + + #[test] + fn a_relative_state_dir_is_refused() { + // A relative path resolves against the jailer's working directory, not + // the runner's, so it has to be caught before it can be handed over. + assert_eq!( + absolute_state_dir("/var/lib/bencher-runner").unwrap(), + "/var/lib/bencher-runner" + ); + absolute_state_dir("bencher-runner").unwrap_err(); + absolute_state_dir("./bencher-runner").unwrap_err(); + absolute_state_dir("").unwrap_err(); + } +} diff --git a/services/runner/src/parser/up.rs b/services/runner/src/parser/up.rs index 84b700004..9be3daad6 100644 --- a/services/runner/src/parser/up.rs +++ b/services/runner/src/parser/up.rs @@ -23,8 +23,13 @@ pub struct CliUp { #[arg(long, default_value = "55", value_parser = clap::value_parser!(u32).range(1..=900))] pub poll_timeout: u32, - /// Persistent state directory for the runner. - #[arg(long, env = "BENCHER_STATE_DIR", default_value = bencher_runner::DEFAULT_STATE_DIR)] + /// Persistent state directory for the runner (absolute path). + #[arg( + long, + env = "BENCHER_STATE_DIR", + default_value = bencher_runner::DEFAULT_STATE_DIR, + value_parser = super::absolute_state_dir, + )] pub state_dir: Utf8PathBuf, /// Unprivileged uid the jailed sandbox process drops to. From 6ae0d51807b733fc4a3249f08bca95cdb0f5cfff Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:09:05 +0000 Subject: [PATCH 47/91] Punctuate the build script overrides consistently Three of the four environment variable overrides are documented in one list, and one of them already used a colon while its neighbours used a dash. The colon is the house style, so the neighbours follow it. --- plus/bencher_runner/build.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plus/bencher_runner/build.rs b/plus/bencher_runner/build.rs index 3ef0f678c..21a971b1b 100644 --- a/plus/bencher_runner/build.rs +++ b/plus/bencher_runner/build.rs @@ -20,10 +20,10 @@ //! //! # Environment Variable Overrides //! -//! - `BENCHER_INIT_PATH` — path to a pre-built bencher-init binary -//! - `BENCHER_FIRECRACKER_PATH` — path to a pre-built firecracker binary +//! - `BENCHER_INIT_PATH`: path to a pre-built bencher-init binary +//! - `BENCHER_FIRECRACKER_PATH`: path to a pre-built firecracker binary //! - `BENCHER_JAILER_PATH`: path to a pre-built jailer binary -//! - `BENCHER_KERNEL_PATH` — path to a pre-built vmlinux kernel +//! - `BENCHER_KERNEL_PATH`: path to a pre-built vmlinux kernel #![expect( clippy::expect_used, From 6846fa20ae07583ff00a65d15210f553aba2098d Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:58:56 +0000 Subject: [PATCH 48/91] Spend the reclaim signal only on a sweep that finished The signal was consumed before the sweep it asks for had a chance to run, so one failure disabled sweeping for the rest of the daemon's life. Job N could not reclaim its jail and raised the signal. Job N+1 consumed it, swept, and failed on the jail that was still stuck. Job N+2 saw a prepared process and a clear signal, returned early, and no job after it ever swept again. Nothing said so, and the promise directly above the code is that a failure is not remembered so the next job retries. When the reason a jail cannot be reclaimed is a VMM still in its cgroup, that orphan runs untrusted guest code on the benchmark cores while every later job measures through it and reports clean, which is the outcome the reap exists to prevent. The check and the consume are now separate steps with the sweep between them, so the signal survives every attempt that does not finish. Neither existing test covered this, though both look like they do: `a_failure_is_not_latched_and_self_heals` fails before the process is ever marked prepared, so it never enters the latching path, and `a_surviving_vmm_fails_every_attempt_not_just_the_first` drives the sweep directly and never touches the signal. The new test prepares successfully, raises the signal, fails twice, and holds the sweep owed throughout. --- plus/bencher_runner/src/jail/mod.rs | 71 +++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 64a030052..0ddaaa3f9 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -207,12 +207,18 @@ impl ReclaimFailed { self.0.store(true, Ordering::SeqCst); } - /// Consume the signal, reporting whether it was set. + /// Consume the signal. + /// + /// Deliberately not one swap together with [`Self::is_set`]. The signal asks + /// for a sweep, so it is spent only once a sweep has actually finished: + /// spending it up front would let a sweep that failed clear the one thing + /// that makes a later job try again. Jobs are serial, so nothing can raise + /// it between the sweep finishing and this call. /// /// Only the jail reads it, and the jail is Linux-only. #[cfg(target_os = "linux")] - fn take(&self) -> bool { - self.0.swap(false, Ordering::SeqCst) + fn clear(&self) { + self.0.store(false, Ordering::SeqCst); } /// Whether the signal is set, without consuming it. @@ -281,10 +287,16 @@ impl HostPreparation { ) -> Result<(), crate::error::JailError> { // A jail that could not be reclaimed earns another sweep, whatever // this process has already done. - if self.prepared && !self.reclaim_failed.take() { + if self.prepared && !self.reclaim_failed.is_set() { return Ok(()); } prepare_host(euid, state_dir, jail_user)?; + // Spent only now that a sweep has run to completion. Spending it before + // the sweep would disarm the mechanism precisely when it is needed: the + // signal would be gone, this process would still count as prepared, and + // every later job would return early while the jail that could not be + // reclaimed sat there holding the benchmark cores. + self.reclaim_failed.clear(); self.prepared = true; Ok(()) } @@ -585,6 +597,57 @@ mod tests { assert!(!state_dir.exists(), "the signal is consumed once"); } + #[test] + fn a_sweep_that_failed_does_not_disarm_the_next_one() { + // The signal is spent on a sweep that finished, never on one that was + // merely attempted. Consuming it up front costs nothing on the first + // failure and everything on the second: this process still counts as + // prepared, so with the signal gone every later job returns early and + // the jail that could not be reclaimed is never swept again for the + // lifetime of the daemon. If the reason it could not be reclaimed is a + // VMM still in its cgroup, that orphan holds the benchmark cores while + // every later job measures through it and reports clean. + let dir = tempfile::tempdir().unwrap(); + let root = camino::Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let state_dir = root.join("state"); + + let mut host = HostPreparation::new(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + + // A teardown that could not reclaim its jail asks for another sweep. + host.reclaim_signal().set(); + + // Preparation now fails, twice, with the signal still outstanding. + std::fs::remove_dir_all(&state_dir).unwrap(); + std::fs::create_dir_all(state_dir.join("someone-elses-data")).unwrap(); + for attempt in 1..=2 { + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap_err(); + assert!( + host.reclaim_signal().is_set(), + "attempt {attempt} failed, so the sweep it asked for is still owed" + ); + } + + // Remove the cause: the next job must still sweep rather than return + // early on the strength of a signal an earlier failure ate. + std::fs::remove_dir_all(&state_dir).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + assert!( + state_dir.join("jail").is_dir(), + "the owed sweep must survive every failed attempt at it" + ); + + // And it is spent now that one has finished. + assert!(!host.reclaim_signal().is_set()); + std::fs::remove_dir_all(&state_dir).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + assert!(!state_dir.exists(), "a finished sweep spends the signal"); + } + #[test] fn the_jail_user_rejects_root() { // Untrusted code against a root VMM is the one thing the confinement From 913cd4d3def3d3a135fd305bd3a3345521cb2b09 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:59:10 +0000 Subject: [PATCH 49/91] Let the runner own a dedicated filesystem A volume mounted at the state directory arrives with `lost+found` on it, put there by mkfs rather than by anyone, and that was enough to refuse the directory as somebody else's with an error blaming the operator for content they did not create. A dedicated filesystem is the natural home for the chroots, since each holds a copy of the VMM binary and a full guest rootfs, and keeping that traffic off the system disk is the recommended answer to its effect on a run, so the guard was rejecting the setup it exists to serve. What a filesystem creates for itself is now exempt, and only that: a volume carrying `lost+found` beside anything else is still refused. The error also points at using a subdirectory, which works for any populated mount. The listing is no longer flattened, either. Dropping per-entry errors let a `readdir` that broke off partway read as an empty directory, which took the 0700 tightening straight past the guard: exactly the hole the report of a failed read closes two lines above. --- plus/bencher_runner/src/error.rs | 2 +- plus/bencher_runner/src/jail/state.rs | 66 +++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index 1cfcfcf79..1ec615f89 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -65,7 +65,7 @@ pub enum JailError { PrivilegedJailUser { field: &'static str }, #[error( - "The state directory {path} already exists, is not empty, and was not created by the runner. Point --state-dir at a directory the runner owns." + "The state directory {path} already exists, is not empty, and was not created by the runner. Point --state-dir at a directory the runner owns, or at a subdirectory of this one." )] ForeignStateDir { path: Utf8PathBuf }, diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index 39999aa52..f78bb8166 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -62,8 +62,13 @@ impl StateDir { /// than treated as an empty directory. It is not evidence that the /// directory is empty, and the chmod that follows is the thing this guard /// exists to keep off a directory that is not the runner's: a path that is - /// really a file, or one whose contents cannot be listed, would otherwise - /// be taken on the strength of a failed check. + /// really a file, one whose contents cannot be listed, or a listing that + /// breaks off partway would otherwise be taken on the strength of a failed + /// check. A listing that ends early is the same failure as one that never + /// started, so it is reported rather than dropped. + /// + /// What the filesystem itself put there is not somebody else's data. See + /// [`BENIGN_ENTRIES`]. fn check_root_is_ours(&self) -> Result<(), JailError> { let entries = match fs::read_dir(&self.root) { Ok(entries) => entries, @@ -77,12 +82,19 @@ impl StateDir { }, }; let mut populated = false; - for entry in entries.flatten() { - populated = true; + for entry in entries { + let entry = entry.map_err(|e| JailError::ReadStateDir { + path: self.root.clone(), + source: e, + })?; let name = entry.file_name(); if RUNNER_ENTRIES.iter().any(|ours| name == *ours) { return Ok(()); } + if BENIGN_ENTRIES.iter().any(|benign| name == *benign) { + continue; + } + populated = true; } if populated { return Err(JailError::ForeignStateDir { @@ -149,6 +161,20 @@ impl StateDir { /// one that belongs to the host. const RUNNER_ENTRIES: [&str; 2] = [CHROOT_BASE, LOCK_FILE]; +/// Entries that do not make a directory somebody else's. +/// +/// A dedicated filesystem is the natural home for the chroots, since each holds +/// a copy of the VMM binary and a full guest rootfs, and moving that traffic +/// off the system disk is the recommended answer to its effect on a run. A +/// freshly created ext4 volume already contains `lost+found` at its mount +/// point, so counting that as somebody else's data would refuse the exact setup +/// the state directory exists to support, and would do it with an error saying +/// the directory was not created by the runner. +/// +/// Only what the filesystem itself creates belongs here. Anything a person or +/// another program put there is what the guard is for. +const BENIGN_ENTRIES: [&str; 1] = ["lost+found"]; + /// Remove every jail directory under `jail_parent`, returning how many were /// reclaimed. /// @@ -372,6 +398,38 @@ mod tests { ); } + #[test] + fn a_dedicated_filesystem_is_ours_to_take() { + // A freshly created ext4 volume mounted at the state directory holds + // `lost+found`, which the filesystem made, not an operator. Refusing it + // would block the recommended setup with an error blaming the operator + // for a directory they did not populate. + let (_dir, root) = temp_root(); + let volume = root.join("volume"); + fs::create_dir_all(volume.join("lost+found")).unwrap(); + + StateDir::new(volume.clone()).create().unwrap(); + + let mode = fs::metadata(&volume).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o700); + assert!(volume.join("lost+found").exists(), "left where it was"); + } + + #[test] + fn a_benign_entry_does_not_launder_a_populated_directory() { + // The exemption covers what a filesystem creates, not the directory it + // happens to sit in. + let (_dir, root) = temp_root(); + let foreign = root.join("var-lib"); + fs::create_dir_all(foreign.join("lost+found")).unwrap(); + fs::create_dir_all(foreign.join("dpkg")).unwrap(); + + StateDir::new(foreign.clone()).create().unwrap_err(); + + let mode = fs::metadata(&foreign).unwrap().permissions().mode(); + assert_ne!(mode & 0o777, 0o700, "a refused root must not be chmodded"); + } + #[test] fn an_empty_directory_is_ours_to_take() { let (_dir, root) = temp_root(); From cbc396cf26bba0baab4f4b0f05e672e16a4c55d3 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:59:23 +0000 Subject: [PATCH 50/91] Name both ways to give up the sandbox The root refusal sent every operator to `--danger-allow-no-sandbox`, which only `runner up` accepts. A one-shot `runner run --sandbox firecracker` on an unprivileged host hits the same error and was told to pass an argument it rejects; the way out of that one is to omit `--sandbox`. Both are named now, each against the subcommand that has it, and the cost is stated once for both: what is given up is the microVM and not merely its confinement, so the Job runs on the host. The test asserts on both spellings, since the wording is the only thing standing between an operator and a flag that does not exist. --- plus/bencher_runner/src/error.rs | 5 +++-- plus/bencher_runner/src/jail/mod.rs | 11 ++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index 1ec615f89..5ede52d3a 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -118,8 +118,9 @@ pub enum JailError { Building the sandbox needs privileges that enabling KVM does not grant: mknod for the chroot's /dev/kvm, \ chown to hand the guest images to the sandbox user, pivot_root, and setns to join a network namespace. \ A world-readable /dev/kvm is enough to use KVM unprivileged but not to build the sandbox around it. \ - Start the Runner as root, or start it with --danger-allow-no-sandbox and assign it only Specs with no Sandbox, \ - which gives up the microVM itself and not just its confinement." + Run as root, or give up the sandbox: start `runner up` with --danger-allow-no-sandbox and assign it \ + only Specs with no Sandbox, or invoke `runner run` without --sandbox. Either way what is given up is \ + the microVM itself and not just its confinement, so the Job executes directly on the host." )] NotRoot { euid: u32 }, diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 0ddaaa3f9..2c2fa2548 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -477,9 +477,18 @@ mod tests { message.contains("root"), "attempt {attempt} must name root: {message}" ); + // Both escapes, because both subcommands reach this error and each + // has only one of them: `--danger-allow-no-sandbox` exists on + // `runner up`, and a one-shot `runner run` gives up the sandbox by + // omitting `--sandbox`. Naming only the daemon's flag sends a + // `runner run` operator to an argument it does not accept. assert!( message.contains("--danger-allow-no-sandbox"), - "attempt {attempt} must name the escape hatch: {message}" + "attempt {attempt} must name the daemon's escape hatch: {message}" + ); + assert!( + message.contains("without --sandbox"), + "attempt {attempt} must name the one-shot escape hatch: {message}" ); } From 1d96162dadb54192ea887afb71b10de5041afed2 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:59:35 +0000 Subject: [PATCH 51/91] Track which archive entries are still wanted A tar archive may legally carry the same path more than once, and the extraction counted down the number of entries it still wanted. A third match against a two-entry list underflowed and panicked with a message about subtraction rather than about the archive, and two matches on the same entry reached zero and reported success while the other entry was never extracted. It tracks the outstanding names instead, so a duplicate is simply extracted again and the early return means every wanted entry was actually found. --- plus/bencher_runner/build.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/plus/bencher_runner/build.rs b/plus/bencher_runner/build.rs index 21a971b1b..bcbb8c4b1 100644 --- a/plus/bencher_runner/build.rs +++ b/plus/bencher_runner/build.rs @@ -421,7 +421,13 @@ fn download_and_extract_tgz( let gz = flate2::read::GzDecoder::new(archive_bytes.as_slice()); let mut archive = tar::Archive::new(gz); - let mut remaining = wanted.len(); + // The names still to be found, not a count of them. A tar archive may + // legally carry the same path more than once, and counting extractions down + // gets that wrong in both directions: a third match against two wanted + // entries underflows and panics with a message that says nothing about the + // archive, and two matches on the same entry reach zero and report success + // while the other is still missing. + let mut outstanding: Vec<&str> = wanted.iter().map(|(name, _)| name.as_str()).collect(); for entry in archive .entries() .map_err(|e| format!("Failed to read tar entries: {e}"))? @@ -443,8 +449,8 @@ fn download_and_extract_tgz( fs::write(dest, &bytes) .map_err(|e| format!("Failed to write to {}: {e}", dest.display()))?; eprintln!("Extracted '{path}' to: {}", dest.display()); - remaining -= 1; - if remaining == 0 { + outstanding.retain(|name| *name != path); + if outstanding.is_empty() { return Ok(()); } } From a5e7669ee859ee35949da7f46bd54c750efb27a9 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 03:59:35 +0000 Subject: [PATCH 52/91] Say what the scenarios' state directory actually is There is one state directory for the suite, wiped before each scenario, not one per scenario. The consequence the comment draws is right and the reason it gave for it was not. --- tasks/test_runner/src/task/scenarios.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index 5894598ba..8f09fe94b 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -252,8 +252,9 @@ fn run_scenario(scenario: &Scenario, runner_bin: &Utf8Path) -> Result<()> { setup().with_context(|| format!("Setup failed for {}", scenario.name))?; } - // Every scenario gets its own state directory, so jail assertions are - // scoped to the scenario and never touch a real runner's state. + // One state directory for the suite, wiped before each scenario, so jail + // assertions see only this scenario's jails and never touch a real runner's + // state. let state_dir = scenario_state_dir(); drop(fs::remove_dir_all(&state_dir)); From d1a92c1cc5ae595276b5eef58e3a9184628a23e5 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 04:40:55 +0000 Subject: [PATCH 53/91] Report a jail directory the sweep could not read The sweep treated every failure to read its parent directory as an empty one and reported success, and dropped per-entry errors the same way. That is the opposite of the rule the guard one level up follows, where anything but absence is refused rather than taken for an empty directory, and it matters more here: this is the function whose whole job is finding what a previous runner left behind, so "could not look" must never reach the caller as "nothing was there". Absence still means a first run and returns zero. Everything else is now reported, and a listing that breaks off partway is remembered like any other failure while the entries it did yield are still reaped. A `state.create()` immediately before makes this near-unreachable in practice, but the reasoning that makes it unreachable lives in another function. --- plus/bencher_runner/src/error.rs | 8 ++++ plus/bencher_runner/src/jail/state.rs | 56 +++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index 5ede52d3a..2023cbe8f 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -83,6 +83,14 @@ pub enum JailError { source: std::io::Error, }, + #[error( + "Failed to read the jail directory {path}: {source}. The sweep cannot tell what a previous runner left behind, and a directory it could not read is not an empty one." + )] + ReadJailParent { + path: Utf8PathBuf, + source: std::io::Error, + }, + #[cfg(target_os = "linux")] #[error("Failed to create network namespace directory {path}: {source}")] NetnsDir { diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index f78bb8166..d8aa2f586 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -214,8 +214,21 @@ where R: Fn(&Utf8Path) -> Reaped, C: Fn(&VmId) -> Result<(), JailError>, { - let Ok(entries) = fs::read_dir(jail_parent) else { - return Ok(0); + // Absence is the only reading that means there is nothing to sweep, and it + // is the ordinary one: this runs before any jail exists in this process, so + // a parent that is not there yet is a first run. Every other failure is + // reported, because "could not look" must not reach the caller as "nothing + // was there" in the one function whose job is finding what a previous runner + // left behind. The rule `check_root_is_ours` follows, one level up. + let entries = match fs::read_dir(jail_parent) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(0), + Err(e) => { + return Err(JailError::ReadJailParent { + path: jail_parent.to_owned(), + source: e, + }); + }, }; let mut swept = 0; @@ -224,7 +237,26 @@ where // unreaped, with its chroot and cgroup still in place. let mut failure = None; - for entry in entries.flatten() { + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + // A listing that breaks off partway leaves jails unexamined, so it + // is remembered like any other failure rather than passing for a + // sweep that found nothing. What the listing did yield is still + // worth reaping. + Err(e) => { + eprintln!( + "Warning: failed to read an entry under {jail_parent}: {e}. Jails there may not have been examined." + ); + if failure.is_none() { + failure = Some(JailError::ReadJailParent { + path: jail_parent.to_owned(), + source: e, + }); + } + continue; + }, + }; if !entry.file_type().is_ok_and(|file_type| file_type.is_dir()) { continue; } @@ -634,6 +666,24 @@ mod tests { assert_eq!(swept, 1); } + #[test] + fn a_jail_parent_that_cannot_be_read_is_not_an_empty_one() { + // The sweep exists to find what a previous runner left behind, so a + // read it could not perform must not reach the caller as a clean host. + // A file where the jail parent should be reads back `ENOTDIR`, the same + // way an unlistable directory reads back `EACCES`. + let (_dir, root) = temp_root(); + let not_a_dir = root.join("firecracker"); + fs::write(¬_a_dir, b"in the way").unwrap(); + + let err = sweep_jails_with(¬_a_dir, |_j| Reaped::Clear, |_v| Ok(())).unwrap_err(); + + assert!( + matches!(err, JailError::ReadJailParent { .. }), + "a read that failed is reported, not counted as zero jails: {err}" + ); + } + #[test] fn sweep_missing_parent_is_zero() { let (_dir, root) = temp_root(); From 307d933925e29c311e4f7c1faf449e63a75c29f2 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 04:41:04 +0000 Subject: [PATCH 54/91] Keep the checked socket path end to end `SocketPath` exists so that every value has been measured against the `sun_path` limit, and the API client took one and immediately downgraded it to a `String`, as did the error it raises when the address cannot be used. The guarantee was discarded at the one boundary where the kernel actually enforces the limit, and for nothing: the type is `Clone` and prints the same. It is carried through now, so the error that most often means "too long for sun_path" holds a value that has been checked against it. --- plus/bencher_runner/src/firecracker/client.rs | 12 ++++++++---- plus/bencher_runner/src/firecracker/error.rs | 6 +++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/client.rs b/plus/bencher_runner/src/firecracker/client.rs index 611ffe8f0..5e4660c59 100644 --- a/plus/bencher_runner/src/firecracker/client.rs +++ b/plus/bencher_runner/src/firecracker/client.rs @@ -15,7 +15,11 @@ use crate::jail::SocketPath; /// Client for the Firecracker REST API. pub struct FirecrackerClient { - socket_path: String, + /// Held as a [`SocketPath`], not a string. The type is the proof that this + /// path fits `sun_path`, and this is the one place the limit is actually + /// enforced by the kernel, so downgrading it here would discard the + /// guarantee exactly where it is worth having. + socket_path: SocketPath, } impl FirecrackerClient { @@ -25,7 +29,7 @@ impl FirecrackerClient { /// socket view; the jailed VMM binds the chroot view of the same file. pub fn new(socket_path: &SocketPath) -> Self { Self { - socket_path: socket_path.as_str().to_owned(), + socket_path: socket_path.clone(), } } @@ -39,7 +43,7 @@ impl FirecrackerClient { /// instead of at the cause. An over-long socket path is rejected by the /// standard library before any syscall, which is exactly that case. pub fn try_ready(&self) -> Result { - match UnixStream::connect(&self.socket_path) { + match UnixStream::connect(self.socket_path.as_str()) { Ok(mut stream) => { drop(stream.set_read_timeout(Some(Duration::from_secs(1)))); drop(stream.set_write_timeout(Some(Duration::from_secs(1)))); @@ -148,7 +152,7 @@ impl FirecrackerClient { /// /// Returns the HTTP status code and response body. fn http_put(&self, path: &str, json_body: &str) -> Result<(u16, String), FirecrackerError> { - let mut stream = UnixStream::connect(&self.socket_path)?; + let mut stream = UnixStream::connect(self.socket_path.as_str())?; stream.set_read_timeout(Some(Duration::from_secs(5)))?; stream.set_write_timeout(Some(Duration::from_secs(5)))?; diff --git a/plus/bencher_runner/src/firecracker/error.rs b/plus/bencher_runner/src/firecracker/error.rs index adf990594..12f1b6d8b 100644 --- a/plus/bencher_runner/src/firecracker/error.rs +++ b/plus/bencher_runner/src/firecracker/error.rs @@ -71,7 +71,11 @@ pub enum FirecrackerError { #[error("Firecracker API socket {path} is unusable: {source}")] SocketUnusable { /// The socket path the runner tried to reach. - path: String, + /// + /// The checked type, not a string: every value has been measured + /// against `sun_path`, which is the limit this error is most often + /// about. + path: crate::jail::SocketPath, /// Why it could not be reached. source: std::io::Error, }, From a95dd7a47472b41ed4759b34618690ae05e17356 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 04:41:14 +0000 Subject: [PATCH 55/91] Hold the jailer's exit status, not a rendering of it Error variants wrap the real type rather than a stringified one, and this one rendered an `ExitStatus` at the point it was constructed. It holds the status itself now, which prints the same and stays inspectable by anything that matches on the error. --- plus/bencher_runner/src/firecracker/error.rs | 2 +- plus/bencher_runner/src/firecracker/process.rs | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/error.rs b/plus/bencher_runner/src/firecracker/error.rs index 12f1b6d8b..d3d236003 100644 --- a/plus/bencher_runner/src/firecracker/error.rs +++ b/plus/bencher_runner/src/firecracker/error.rs @@ -61,7 +61,7 @@ pub enum FirecrackerError { )] JailerExited { /// How the jailer exited. - status: String, + status: std::process::ExitStatus, }, /// The API socket address itself cannot be used. diff --git a/plus/bencher_runner/src/firecracker/process.rs b/plus/bencher_runner/src/firecracker/process.rs index ed91b856e..e628d500e 100644 --- a/plus/bencher_runner/src/firecracker/process.rs +++ b/plus/bencher_runner/src/firecracker/process.rs @@ -181,9 +181,7 @@ impl FirecrackerProcess { return Ok(()); } if let Ok(Some(status)) = self.child.try_wait() { - return Err(FirecrackerError::JailerExited { - status: status.to_string(), - }); + return Err(FirecrackerError::JailerExited { status }); } std::thread::sleep(poll_interval); } From bd1046196868278568c3e205819cb8b72c9c306d Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 04:41:14 +0000 Subject: [PATCH 56/91] Unwind the network namespace handle through clear The two cleanup paths in `ensure` unlinked the handle directly. The second is reached only when the namespace turned out not to be distinct, which is after the bind mount is in place, so the unlink fails with `EBUSY` and leaves the handle mounted: the state that makes every later `ensure` fail on a stacked mount, which is what `clear` was written to unwind. Both paths go through it now, so the handle is only ever removed one way. --- plus/bencher_runner/src/jail/netns.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/plus/bencher_runner/src/jail/netns.rs b/plus/bencher_runner/src/jail/netns.rs index 838cab0ba..539210f1b 100644 --- a/plus/bencher_runner/src/jail/netns.rs +++ b/plus/bencher_runner/src/jail/netns.rs @@ -99,15 +99,20 @@ pub fn ensure() -> Result { source: e, })?; + // Unwound through `clear`, never a bare unlink. Reaching the second arm + // means the bind mount is definitely there, since that is what makes the + // namespace live, and unlinking a mounted path fails with `EBUSY`: the + // handle would stay mounted, which is the state that makes every later + // `ensure` fail on a stacked mount. `clear` is the only way it is removed. if let Err(e) = create(&handle) { - drop(fs::remove_file(&handle)); + drop(clear(&handle)); return Err(e); } // The namespace has to be a real one and not the runner's own, or the VMM // would keep host network reach. Cheap, and the whole point of the module. if !is_live_netns(&handle) { - drop(fs::remove_file(&handle)); + drop(clear(&handle)); return Err(JailError::NetnsNotDistinct { path: handle.clone(), }); From c24a432ae394cea8b2e679e327fb65bc66fcefe9 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 04:41:22 +0000 Subject: [PATCH 57/91] Drop the empty error channel on cgroup cleanup Every path through `cleanup` returned `Ok(())`: the only failure it has, a `rmdir` the kernel refuses, is warned and routed through the reclaim signal. The `Result` was a channel with nothing in it, which is why `Drop` had to discard it. It returns nothing now and `Drop` simply calls it. --- plus/bencher_runner/src/jail/cgroup.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index 0187e9eac..b7c5a1fc7 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -295,7 +295,11 @@ impl CgroupManager { /// logged: it means something is still in this cgroup, and the only way to /// get to it later is through the chroot of the same id, so the signal both /// holds that directory and earns the next job a sweep. - pub fn cleanup(&mut self) -> Result<(), RunnerError> { + /// + /// Returns nothing, because the signal is where a failure goes. A `Result` + /// here would be a channel with nothing in it that every caller, `Drop` + /// included, would have to discard. + pub fn cleanup(&mut self) { if self.created && self.cgroup_path.exists() { if let Err(e) = fs::remove_dir(&self.cgroup_path) { eprintln!( @@ -307,13 +311,12 @@ impl CgroupManager { self.created = false; } } - Ok(()) } } impl Drop for CgroupManager { fn drop(&mut self) { - drop(self.cleanup()); + self.cleanup(); } } @@ -672,7 +675,7 @@ mod tests { fs::create_dir_all(manager.path()).unwrap(); fs::write(manager.path().join("cgroup.procs"), "42\n").unwrap(); - manager.cleanup().unwrap(); + manager.cleanup(); assert!( reclaim_failed.is_set(), @@ -693,7 +696,7 @@ mod tests { }; fs::create_dir_all(manager.path()).unwrap(); - manager.cleanup().unwrap(); + manager.cleanup(); assert!(!manager.path().exists()); assert!( From c47927ce1f6ed2da3386797fcde96f1e5c6fe729 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 04:41:41 +0000 Subject: [PATCH 58/91] Report a probe timeout when it happens, not minutes later When no jailed VMM appears within the probe timeout, the scenario waited on a runner that is by definition still running, so the failure surfaced only once the runner's own two minute timeout expired. That delay lands entirely on whoever is debugging the path. The runner is killed first now, and only when it has not already been reaped, since signalling a reaped pid can reach whatever inherited the number. --- tasks/test_runner/src/task/scenarios.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index 8f09fe94b..7fbe9daf3 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -2829,6 +2829,15 @@ fn run_runner_after_orphan( }; let Some((vm_id, jail_root, vmm_pid)) = orphan else { + // Killed before the wait. A probe that timed out leaves the runner still + // going, so waiting on it first would sit there until the runner's own + // timeout expired and report the failure minutes late, which is exactly + // when somebody is watching. Only if it has not already been reaped: + // `try_wait` in the loop above reaps it, and signalling a reaped pid can + // reach whatever inherited the number. + if child.try_wait()?.is_none() { + kill_pid(child.id(), libc::SIGKILL); + } drop(child.wait()); let (stdout, stderr) = readers.join(); bail!( From e8e9a2a6e68de8ae9a3f4262a59a4167ebc342b6 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 04:41:41 +0000 Subject: [PATCH 59/91] Return the scenarios' state directory to its invoker The scenarios must run as root, so the runner leaves its state directory root-owned inside the repo tree. CI throws the tree away, but on a developer's machine the next unprivileged cargo invocation trips over a directory it can neither read nor remove. It is handed back to whoever invoked sudo, and when there is nobody to hand it to the path is printed with the command that clears it rather than left to be discovered. --- tasks/test_runner/src/task/scenarios.rs | 52 ++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index 7fbe9daf3..d4897ed68 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -175,19 +175,59 @@ impl Scenarios { scenarios.extend(jail_scenarios()); scenarios.extend(nosandbox_scenarios()); - if let Some(name) = &self.scenario { + let result = if let Some(name) = &self.scenario { // Run a single scenario - let scenario = scenarios + scenarios .iter() .find(|s| s.name == name) - .with_context(|| format!("Unknown scenario: {name}"))?; - - run_scenario(scenario, &runner_bin) + .with_context(|| format!("Unknown scenario: {name}")) + .and_then(|scenario| run_scenario(scenario, &runner_bin)) } else { // Run all scenarios run_all_scenarios(&scenarios, &runner_bin) - } + }; + + // Whatever the outcome. The state directory is only root-owned because + // the scenarios had to be, and it sits inside the repo tree. + return_state_dir_to_invoker(); + + result + } +} + +/// Hand the state directory back to whoever invoked `sudo`. +/// +/// The scenarios must run as root, and the runner creates its state directory at +/// 0700 owned by root under the repo's target tree. CI throws that tree away, so +/// it costs nothing there, but on a developer's machine the next unprivileged +/// `cargo` invocation trips over a directory it can neither read nor remove. +/// `SUDO_UID` names who to give it back to; with no one to give it back to, or a +/// `chown` that will not run, the path is printed with the command that clears +/// it rather than left to be discovered. +fn return_state_dir_to_invoker() { + let state_dir = scenario_state_dir(); + if !state_dir.exists() { + return; + } + + if let Some((uid, gid)) = invoking_user() + && Command::new("chown") + .args(["-R", &format!("{uid}:{gid}"), state_dir.as_str()]) + .status() + .is_ok_and(|status| status.success()) + { + println!("Returned {state_dir} to uid {uid}"); + return; } + + println!("Note: {state_dir} is left owned by root. Remove it with: sudo rm -rf {state_dir}"); +} + +/// The uid and gid that invoked `sudo`, when one did. +fn invoking_user() -> Option<(u32, u32)> { + let uid = std::env::var("SUDO_UID").ok()?.parse().ok()?; + let gid = std::env::var("SUDO_GID").ok()?.parse().ok()?; + Some((uid, gid)) } /// List all available scenarios. From eb97a64ae5225cb121b7c396d3eb66f513a862b1 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 05:38:25 +0000 Subject: [PATCH 60/91] Owe another sweep when a chroot will not go away The sweep warned about a `remove_dir_all` it could not finish and reported success, which spent the reclaim signal on a sweep that had not finished. Job 1's teardown fails and arms the signal, job 2 sweeps and fails the same way, the failure is swallowed, the signal is spent, and from job 3 on preparation returns early while a VMM binary and a full guest rootfs sit there until the daemon restarts. That is the leak the signal exists to prevent, surviving exactly one retry. The failure is disk, not fidelity: the VMM is gone and the cgroup with it, so the job may run. So the sweep reports what it did rather than failing, and preparation spends the signal only on a sweep that finished, arming it again on one that did not. That is the third instance of one mistake, a fallible teardown step whose failure never reached the mechanism built to retry it, so the sweep decomposes into a per-jail function whose three outcomes are the three things such a step is allowed to do. --- plus/bencher_runner/src/jail/mod.rs | 31 ++- plus/bencher_runner/src/jail/state.rs | 303 ++++++++++++++++++-------- 2 files changed, 237 insertions(+), 97 deletions(-) diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 2c2fa2548..3a0532338 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -290,13 +290,19 @@ impl HostPreparation { if self.prepared && !self.reclaim_failed.is_set() { return Ok(()); } - prepare_host(euid, state_dir, jail_user)?; - // Spent only now that a sweep has run to completion. Spending it before - // the sweep would disarm the mechanism precisely when it is needed: the - // signal would be gone, this process would still count as prepared, and - // every later job would return early while the jail that could not be - // reclaimed sat there holding the benchmark cores. - self.reclaim_failed.clear(); + let swept = prepare_host(euid, state_dir, jail_user)?; + // Spent only on a sweep that finished, and armed by one that did not. + // Spending it any earlier disarms the mechanism precisely when it is + // needed: the signal would be gone, this process would still count as + // prepared, and every later job would return early while the jail nobody + // could reclaim sat there. That is true of a sweep that failed outright, + // which never reaches here, and equally of one that returned `Ok` having + // left a chroot on disk. + if swept.is_complete() { + self.reclaim_failed.clear(); + } else { + self.reclaim_failed.set(); + } self.prepared = true; Ok(()) } @@ -330,7 +336,7 @@ fn prepare_host( euid: u32, state_dir: &camino::Utf8Path, jail_user: JailUser, -) -> Result<(), crate::error::JailError> { +) -> Result { // Checked first, and by name. Without it the most likely upgrade failure // surfaces as a permission error on a directory, or a bare EPERM out of // `unshare`, neither of which mentions root or the flag that avoids it. @@ -343,12 +349,15 @@ fn prepare_host( let _lock = JailLock::acquire(state.path())?; let swept = state::sweep_jails(&state.jail_parent())?; - if swept > 0 { + if swept.reclaimed() > 0 { // Each one held a copy of the VMM binary and a full guest rootfs // image, so an operator should hear about it. - println!(" Reclaimed {swept} stale jail(s) from {state_dir}"); + println!( + " Reclaimed {} stale jail(s) from {state_dir}", + swept.reclaimed() + ); } - Ok(()) + Ok(swept) } /// Refuse to build a jail without the privileges building one needs. diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index d8aa2f586..db33b6908 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -175,8 +175,36 @@ const RUNNER_ENTRIES: [&str; 2] = [CHROOT_BASE, LOCK_FILE]; /// another program put there is what the guard is for. const BENIGN_ENTRIES: [&str; 1] = ["lost+found"]; -/// Remove every jail directory under `jail_parent`, returning how many were -/// reclaimed. +/// What one sweep did. +/// +/// Separating "reclaimed" from "left behind" is what lets the caller decide +/// about the reclaim signal. A jail whose chroot would not go away is disk, not +/// a contended benchmark, so it does not fail the job; but the sweep is the +/// mechanism that reclaims it, so a sweep that left one behind has to leave the +/// signal armed rather than spend it. See [`crate::jail`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Swept { + /// Jails whose chroot is gone. + reclaimed: usize, + /// Jails still on disk that a later sweep owes another attempt. + left_behind: usize, +} + +impl Swept { + /// Jails whose chroot is gone. + #[must_use] + pub fn reclaimed(self) -> usize { + self.reclaimed + } + + /// Whether the sweep owes nothing further. + #[must_use] + pub fn is_complete(self) -> bool { + self.left_behind == 0 + } +} + +/// Remove every jail directory under `jail_parent`, reporting what it did. /// /// Jobs run serially, so anything found here is stale by construction. The /// runner disappears without unwinding in several ordinary ways, including @@ -186,7 +214,7 @@ const BENIGN_ENTRIES: [&str; 1] = ["lost+found"]; /// /// Non-directory entries are left alone: the jailer only ever creates /// directories here, so anything else was put there by someone else. -pub fn sweep_jails(jail_parent: &Utf8Path) -> Result { +pub fn sweep_jails(jail_parent: &Utf8Path) -> Result { sweep_jails_with( jail_parent, super::reap::reap_jailed_vmm, @@ -209,7 +237,7 @@ fn sweep_jails_with( jail_parent: &Utf8Path, reap: R, remove_cgroup: C, -) -> Result +) -> Result where R: Fn(&Utf8Path) -> Reaped, C: Fn(&VmId) -> Result<(), JailError>, @@ -222,7 +250,7 @@ where // left behind. The rule `check_root_is_ours` follows, one level up. let entries = match fs::read_dir(jail_parent) { Ok(entries) => entries, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(0), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Swept::default()), Err(e) => { return Err(JailError::ReadJailParent { path: jail_parent.to_owned(), @@ -231,15 +259,25 @@ where }, }; - let mut swept = 0; + let mut swept = Swept::default(); // The first failure is remembered but does not abandon the rest: one jail // whose cgroup will not go away must not leave every other stale jail // unreaped, with its chroot and cgroup still in place. let mut failure = None; for entry in entries { - let entry = match entry { - Ok(entry) => entry, + let outcome = match entry { + Ok(entry) => match jail_id(jail_parent, &entry) { + Ok(Some(vm_id)) => reclaim_one( + &jail_parent.join(vm_id.as_str()), + &vm_id, + &reap, + &remove_cgroup, + ), + // Not a jail, or not ours: nothing owed either way. + Ok(None) => continue, + Err(e) => Reclamation::Failed(e), + }, // A listing that breaks off partway leaves jails unexamined, so it // is remembered like any other failure rather than passing for a // sweep that found nothing. What the listing did yield is still @@ -248,92 +286,144 @@ where eprintln!( "Warning: failed to read an entry under {jail_parent}: {e}. Jails there may not have been examined." ); + Reclamation::Failed(JailError::ReadJailParent { + path: jail_parent.to_owned(), + source: e, + }) + }, + }; + + match outcome { + Reclamation::Reclaimed => swept.reclaimed += 1, + Reclamation::LeftBehind => swept.left_behind += 1, + Reclamation::Failed(e) => { if failure.is_none() { - failure = Some(JailError::ReadJailParent { - path: jail_parent.to_owned(), - source: e, - }); + failure = Some(e); } - continue; }, - }; - if !entry.file_type().is_ok_and(|file_type| file_type.is_dir()) { - continue; } - // Skipped rather than lossily converted. A lossy name rebuilds into a - // path naming a different file, and everything downstream then works - // on the wrong one: the reap stats a path that does not exist and - // reports the jail clear, so a live VMM is neither reaped nor - // mentioned, and the cgroup removal targets a name nobody created. - // The runner only ever creates UTF-8 names here, so anything else is - // not ours to touch. - let file_name = entry.file_name(); - let Some(name) = file_name.to_str() else { - eprintln!( - "Warning: skipping an entry with a non-UTF-8 name under {jail_parent}; the runner did not create it" - ); - continue; - }; - let vm_id = VmId::from_chroot_name(name.to_owned()); - let jail_dir = jail_parent.join(vm_id.as_str()); - - // Reap before removing, and only remove once the jail is clear. - // Deleting the tree under a live VMM would not stop it, and it would - // destroy the only handle for identifying that process later: without - // the directory the next sweep never sees this id, never removes its - // cgroup, and the cgroup leaks for good. - if let Reaped::StillRunning { pid } = reap(&jail_dir.join(JAIL_ROOT)) { - // Fatal to the job, not to the runner. A stray VMM runs untrusted - // guest code on the benchmark cores, and nothing downstream - // catches it: these cgroups claim no exclusive cpuset, so the next - // job's cpuset applies and verifies cleanly while being contended - // the whole time. Refusing to measure is the only honest answer. - // - // Every surviving jail is reported and the first becomes the - // error, so an operator sees each one on every attempt rather than - // once. + } + + match failure { + Some(e) => Err(e), + None => Ok(swept), + } +} + +/// What became of one stale jail. +/// +/// The three variants are the three columns of the table in [`crate::jail`], so +/// a step added to [`reclaim_one`] has to pick one. +enum Reclamation { + /// The cgroup and the chroot are both gone. + Reclaimed, + /// Still on disk. Costs disk rather than fidelity, so the job may run, but + /// the sweep is what reclaims it and this one did not, so another is owed. + LeftBehind, + /// The host cannot be trusted to measure until this is resolved. + Failed(JailError), +} + +/// The identity of the jail an entry names, if it is one of ours. +/// +/// `Ok(None)` is an entry that is not a jail. An entry whose kind cannot be read +/// is not one of those: it may be a jail, so skipping it silently would leave a +/// live VMM unexamined while still reporting a sweep that found nothing wrong. +fn jail_id(jail_parent: &Utf8Path, entry: &fs::DirEntry) -> Result, JailError> { + match entry.file_type() { + Ok(file_type) if file_type.is_dir() => {}, + Ok(_) => return Ok(None), + Err(e) => { eprintln!( - "Warning: leaving stale jail {jail_dir} in place because VMM pid {pid} is still running on the benchmark cores." + "Warning: cannot tell what {} under {jail_parent} is: {e}. If it is a jail, it was not examined.", + entry.file_name().display() ); - if failure.is_none() { - failure = Some(JailError::JailStillRunning { - path: jail_dir.clone(), - pid, - }); - } - continue; - } + return Err(JailError::ReadJailParent { + path: jail_parent.to_owned(), + source: e, + }); + }, + } - // The cgroup goes first, and the chroot only once the cgroup is gone. - // The two are named by the same id, and the directory is the only - // handle a later sweep has for finding the cgroup again, so removing - // the directory while the cgroup survives strands that cgroup for - // good: the next sweep never sees the id, never retries the removal, - // and something may still be running on the benchmark cores under it. - // A leftover cgroup claims nothing, since these cgroups set no - // exclusive cpuset, but a removal that fails usually means something is - // still in it, which is why it is reported rather than swallowed. - if let Err(e) = remove_cgroup(&vm_id) { + // Skipped rather than lossily converted. A lossy name rebuilds into a + // path naming a different file, and everything downstream then works + // on the wrong one: the reap stats a path that does not exist and + // reports the jail clear, so a live VMM is neither reaped nor + // mentioned, and the cgroup removal targets a name nobody created. + // The runner only ever creates UTF-8 names here, so anything else is + // not ours to touch. + let file_name = entry.file_name(); + let Some(name) = file_name.to_str() else { + eprintln!( + "Warning: skipping an entry with a non-UTF-8 name under {jail_parent}; the runner did not create it" + ); + return Ok(None); + }; + Ok(Some(VmId::from_chroot_name(name.to_owned()))) +} + +/// Reap, then unwind one stale jail: its cgroup first, then its chroot. +fn reclaim_one(jail_dir: &Utf8Path, vm_id: &VmId, reap: &R, remove_cgroup: &C) -> Reclamation +where + R: Fn(&Utf8Path) -> Reaped, + C: Fn(&VmId) -> Result<(), JailError>, +{ + // Reap before removing, and only remove once the jail is clear. Deleting the + // tree under a live VMM would not stop it, and it would destroy the only + // handle for identifying that process later: without the directory the next + // sweep never sees this id, never removes its cgroup, and the cgroup leaks + // for good. + // + // Fatal to the job, not to the runner, whether the VMM was found alive or + // could not be looked for at all. A stray VMM runs untrusted guest code on + // the benchmark cores, and nothing downstream catches it: these cgroups + // claim no exclusive cpuset, so the next job's cpuset applies and verifies + // cleanly while being contended the whole time. Refusing to measure is the + // only honest answer, and a jail that could not be examined has not earned a + // better one. + match reap(&jail_dir.join(JAIL_ROOT)) { + Reaped::Clear => {}, + Reaped::StillRunning { pid } => { eprintln!( - "Warning: leaving stale jail {jail_dir} in place because its cgroup could not be removed: {e}" + "Warning: leaving stale jail {jail_dir} in place because VMM pid {pid} is still running on the benchmark cores." ); - if failure.is_none() { - failure = Some(e); - } - continue; - } + return Reclamation::Failed(JailError::JailStillRunning { + path: jail_dir.to_owned(), + pid, + }); + }, + } - // A chroot that will not go away costs disk. Worth a warning, not - // worth refusing to run. - match fs::remove_dir_all(&jail_dir) { - Ok(()) => swept += 1, - Err(e) => eprintln!("Warning: failed to sweep stale jail {jail_dir}: {e}"), - } + // The cgroup goes first, and the chroot only once the cgroup is gone. The + // two are named by the same id, and the directory is the only handle a later + // sweep has for finding the cgroup again, so removing the directory while the + // cgroup survives strands that cgroup for good: the next sweep never sees the + // id, never retries the removal, and something may still be running on the + // benchmark cores under it. A leftover cgroup claims nothing, since these + // cgroups set no exclusive cpuset, but a removal that fails usually means + // something is still in it, which is why it is reported rather than + // swallowed. + if let Err(e) = remove_cgroup(vm_id) { + eprintln!( + "Warning: leaving stale jail {jail_dir} in place because its cgroup could not be removed: {e}" + ); + return Reclamation::Failed(e); } - match failure { - Some(e) => Err(e), - None => Ok(swept), + // A chroot that will not go away costs disk, not fidelity: the VMM is gone + // and the cgroup with it, so the job may run. But the sweep is the only thing + // that reclaims it, and this one just failed to, so it is counted as still + // owed. Warning alone would have the caller spend the reclaim signal on a + // sweep that did not finish, and the leak would then survive until the daemon + // restarted. + match fs::remove_dir_all(jail_dir) { + Ok(()) => Reclamation::Reclaimed, + Err(e) => { + eprintln!( + "Warning: failed to sweep stale jail {jail_dir}: {e}. It holds a VMM binary and a full guest rootfs; the next job will try again." + ); + Reclamation::LeftBehind + }, } } @@ -510,7 +600,9 @@ mod tests { fs::create_dir_all(state.jail_dir(&VmId::from_chroot_name("two".to_owned()))).unwrap(); assert_eq!( - sweep_jails_with(&state.jail_parent(), |_j| Reaped::Clear, |_v| Ok(())).unwrap(), + sweep_jails_with(&state.jail_parent(), |_j| Reaped::Clear, |_v| Ok(())) + .unwrap() + .reclaimed(), 2 ); assert!( @@ -537,7 +629,9 @@ mod tests { fs::create_dir_all(state.jail_dir(&VmId::from_chroot_name("stale".to_owned()))).unwrap(); assert_eq!( - sweep_jails_with(&state.jail_parent(), |_j| Reaped::Clear, |_v| Ok(())).unwrap(), + sweep_jails_with(&state.jail_parent(), |_j| Reaped::Clear, |_v| Ok(())) + .unwrap() + .reclaimed(), 1 ); assert!( @@ -649,6 +743,40 @@ mod tests { assert!(err.to_string().contains("stuck"), "names the cgroup: {err}"); } + #[test] + fn a_chroot_that_will_not_go_away_is_owed_another_sweep() { + // Disk, not fidelity: the VMM is gone and the cgroup with it, so the job + // runs. But the sweep is the only thing that reclaims the tree, and this + // sweep did not, so it reports the debt rather than passing for complete. + // Warning alone would have the caller spend the reclaim signal and the + // leak would outlive every later job. + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")); + state.create().unwrap(); + let stuck = VmId::from_chroot_name("stuck".to_owned()); + fs::create_dir_all(state.jail_root(&stuck)).unwrap(); + // A jail directory that `remove_dir_all` cannot finish: the tree is + // unsearchable, so the walk inside it fails. + fs::set_permissions(state.jail_dir(&stuck), fs::Permissions::from_mode(0o000)).unwrap(); + + let swept = sweep_jails_with( + &state.jail_parent(), + |_jail_root| Reaped::Clear, + |_vm_id| Ok(()), + ) + .unwrap(); + + // Restored before the assertions so the temp directory can be cleaned + // up whichever way they go. + fs::set_permissions(state.jail_dir(&stuck), fs::Permissions::from_mode(0o700)).unwrap(); + + assert_eq!(swept.reclaimed(), 0); + assert!( + !swept.is_complete(), + "a sweep that left a chroot behind still owes one" + ); + } + #[test] fn a_cleared_jail_is_still_swept() { let (_dir, root) = temp_root(); @@ -663,7 +791,8 @@ mod tests { ) .unwrap(); - assert_eq!(swept, 1); + assert_eq!(swept.reclaimed(), 1); + assert!(swept.is_complete()); } #[test] @@ -688,7 +817,9 @@ mod tests { fn sweep_missing_parent_is_zero() { let (_dir, root) = temp_root(); assert_eq!( - sweep_jails_with(&root.join("nope"), |_j| Reaped::Clear, |_v| Ok(())).unwrap(), + sweep_jails_with(&root.join("nope"), |_j| Reaped::Clear, |_v| Ok(())) + .unwrap() + .reclaimed(), 0 ); } From df48dc31e4b1842f20b17d9ab2aa9f38873640b5 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 05:38:46 +0000 Subject: [PATCH 61/91] Refuse to call a jail clear without having looked in it The scan behind the reap swallowed two failures and reported no VMM: a jail root it could not stat, and a `/proc` it could not list. Either one had the sweep delete the tree on the strength of a question nobody answered, which is the same defect as the ones the sweep has already been fixed for, in the one function whose answer authorizes a destructive step. Absence still means clear, because nothing can be chrooted into a directory that does not exist. Everything else is `Unexaminable`, which the sweep treats exactly as it treats a VMM known to be alive: the jail stays and the job fails. A `/proc//root` that cannot be read stays ignored, because a process vanishing under a scan of `/proc` is the answer rather than a failure to get one. --- plus/bencher_runner/src/error.rs | 6 ++ plus/bencher_runner/src/jail/reap.rs | 126 ++++++++++++++++++++++---- plus/bencher_runner/src/jail/state.rs | 36 ++++++++ 3 files changed, 149 insertions(+), 19 deletions(-) diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index 2023cbe8f..4665939c6 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -172,6 +172,12 @@ pub enum JailError { )] JailStillRunning { path: Utf8PathBuf, pid: u32 }, + #[cfg(target_os = "linux")] + #[error( + "A stale jail at {path} could not be examined, so whether a VMM is still running in it is unknown. It is left in place, and a jail that cannot be checked is not a jail that has been cleared." + )] + JailUnexaminable { path: Utf8PathBuf }, + #[cfg(target_os = "linux")] #[error( "The kernel narrowed the cgroup cpuset at {path}: asked for cpus {requested}, got {effective}. The benchmark would not have run on the cores it claims." diff --git a/plus/bencher_runner/src/jail/reap.rs b/plus/bencher_runner/src/jail/reap.rs index 201f42722..de1a70190 100644 --- a/plus/bencher_runner/src/jail/reap.rs +++ b/plus/bencher_runner/src/jail/reap.rs @@ -59,6 +59,15 @@ pub enum Reaped { /// The VMM that is still running. pid: u32, }, + /// The jail could not be examined, so what is in it is unknown. + /// + /// Distinct from [`Self::Clear`], which is the whole reason this variant + /// exists: a jail root that cannot be stat'ed, or a `/proc` that cannot be + /// listed, says nothing about the jail, and reporting nothing as "empty" is + /// what would have the caller delete a tree with a live VMM in it. The + /// caller treats it exactly like [`Self::StillRunning`], with no pid to + /// name. + Unexaminable, } /// Kill the VMM confined to `jail_root`, if one is still running. @@ -79,7 +88,7 @@ pub fn reap_jailed_vmm(jail_root: &Utf8Path) -> Reaped { /// manufacturing sixty-four jailed processes. fn reap_jailed_vmm_with(jail_root: &Utf8Path, find: F, reap: R) -> Reaped where - F: Fn(&Utf8Path) -> Option, + F: Fn(&Utf8Path) -> std::io::Result>, R: Fn(u32, &Utf8Path) -> Reaped, { // Rescan after each reap rather than assuming one process per jail. That @@ -89,11 +98,21 @@ where // load-bearing is worth enforcing rather than trusting, and a survivor // would otherwise have the tree removed out from under it. for _ in 0..MAX_JAILED_PROCESSES { - let Some(pid) = find(jail_root) else { - return Reaped::Clear; - }; - if let Reaped::StillRunning { pid } = reap(pid, jail_root) { - return Reaped::StillRunning { pid }; + match find(jail_root) { + Ok(Some(pid)) => { + if let Reaped::StillRunning { pid } = reap(pid, jail_root) { + return Reaped::StillRunning { pid }; + } + }, + Ok(None) => return Reaped::Clear, + // A scan that could not run has not found the jail empty, it has + // found out nothing. The caller deletes a tree on this answer. + Err(e) => { + eprintln!( + "Warning: cannot examine {jail_root} to see whether a VMM is still in it: {e}. It is left in place." + ); + return Reaped::Unexaminable; + }, } } @@ -111,13 +130,19 @@ where // sweeps again; measuring through a jail that is still occupied is not, and // that is the one outcome this module exists to prevent. match find(jail_root) { - Some(pid) => { + Ok(Some(pid)) => { eprintln!( "Warning: gave up scanning {jail_root} after {MAX_JAILED_PROCESSES} passes; pid {pid} still matches it while every reap reported the jail clear." ); Reaped::StillRunning { pid } }, - None => Reaped::Clear, + Ok(None) => Reaped::Clear, + Err(e) => { + eprintln!( + "Warning: cannot examine {jail_root} to see whether a VMM is still in it: {e}. It is left in place." + ); + Reaped::Unexaminable + }, } } @@ -173,25 +198,46 @@ fn reap_one(pid: u32, jail_root: &Utf8Path) -> Reaped { /// chroot. Comparing device and inode rather than the path is what makes this /// exact: the jailer pivots into a private mount namespace, so the path reads /// back as `/`, while the identity is preserved. -fn find_jailed_vmm(jail_root: &Utf8Path) -> Option { - let jail = fs::metadata(jail_root).ok()?; - for entry in fs::read_dir("/proc").ok()?.flatten() { +/// +/// `Ok(None)` is "nothing is confined here", which a jail root that is not there +/// at all also means: nothing can be chrooted into a directory that does not +/// exist. Every other failure is an error rather than an absence, because the +/// caller deletes a directory tree on the strength of this answer and a scan +/// that could not run has established nothing. +fn find_jailed_vmm(jail_root: &Utf8Path) -> std::io::Result> { + let jail = match fs::metadata(jail_root) { + Ok(jail) => jail, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e), + }; + for entry in fs::read_dir("/proc")? { + let entry = entry?; let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { continue; }; if matches_jail(pid, &jail) { - return Some(pid); + return Ok(Some(pid)); } } - None + Ok(None) } /// Whether a process's root directory is `jail_root`. +/// +/// A jail root that cannot be stat'ed reads as "not this jail's VMM", which +/// keeps the kill from landing on a process this function cannot vouch for. The +/// caller's next scan turns the same failure into [`Reaped::Unexaminable`], so +/// nothing downstream mistakes it for an empty jail. fn is_jailed_vmm(pid: u32, jail_root: &Utf8Path) -> bool { fs::metadata(jail_root).is_ok_and(|jail| matches_jail(pid, &jail)) } /// Whether a process's root directory is the same inode as `jail`. +/// +/// A `/proc//root` that cannot be read means the process is gone or is not +/// one this runner may inspect, and either way it is not the jail's VMM. That is +/// the one failure here that is genuinely nothing: processes come and go under a +/// scan of `/proc` constantly. fn matches_jail(pid: u32, jail: &fs::Metadata) -> bool { // Following this magic symlink crosses into the process's own mount // namespace, which a privileged reader is allowed to do. @@ -317,7 +363,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); - assert_eq!(find_jailed_vmm(&root), None); + assert_eq!(find_jailed_vmm(&root).unwrap(), None); } #[test] @@ -325,7 +371,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); - assert_eq!(find_jailed_vmm(&root.join("absent")), None); + assert_eq!(find_jailed_vmm(&root.join("absent")).unwrap(), None); assert!(!is_jailed_vmm(std::process::id(), &root.join("absent"))); } @@ -410,7 +456,7 @@ mod tests { &root, |_jail_root| { scans.set(scans.get() + 1); - Some(99) + Ok(Some(99)) }, |pid, _jail_root| Reaped::StillRunning { pid }, ); @@ -433,7 +479,7 @@ mod tests { let reaped = reap_jailed_vmm_with( &root, - |_jail_root| Some(7), + |_jail_root| Ok(Some(7)), |_pid, _jail_root| { reaps.set(reaps.get() + 1); Reaped::Clear @@ -460,7 +506,7 @@ mod tests { &root, |_jail_root| { scans.set(scans.get() + 1); - (scans.get() <= MAX_JAILED_PROCESSES).then_some(7) + Ok((scans.get() <= MAX_JAILED_PROCESSES).then_some(7)) }, |_pid, _jail_root| Reaped::Clear, ); @@ -473,6 +519,48 @@ mod tests { ); } + #[test] + fn a_jail_that_cannot_be_examined_is_not_reported_clear() { + // The caller deletes a directory tree on this answer, so a scan that + // could not run must not read as an empty jail. A jail root that is + // simply absent is a different thing and stays clear: nothing can be + // chrooted into a directory that is not there. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + let reaped = reap_jailed_vmm_with( + &root, + |_jail_root| Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)), + |_pid, _jail_root| Reaped::Clear, + ); + + assert_eq!(reaped, Reaped::Unexaminable); + assert_eq!(reap_jailed_vmm(&root.join("absent")), Reaped::Clear); + } + + #[test] + fn a_scan_that_fails_after_the_bound_is_not_reported_clear() { + // The same rule on the way out of the loop as on the way in. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let scans = std::cell::Cell::new(0); + + let reaped = reap_jailed_vmm_with( + &root, + |_jail_root| { + scans.set(scans.get() + 1); + if scans.get() > MAX_JAILED_PROCESSES { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + } else { + Ok(Some(7)) + } + }, + |_pid, _jail_root| Reaped::Clear, + ); + + assert_eq!(reaped, Reaped::Unexaminable); + } + #[test] fn a_still_running_vmm_carries_its_pid() { // The caller keys the decision not to delete a directory off this, so @@ -481,7 +569,7 @@ mod tests { assert_ne!(still, Reaped::Clear); match still { Reaped::StillRunning { pid } => assert_eq!(pid, 4242), - Reaped::Clear => panic!("expected StillRunning"), + Reaped::Clear | Reaped::Unexaminable => panic!("expected StillRunning"), } } } diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index db33b6908..e32f4fc47 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -392,6 +392,14 @@ where pid, }); }, + Reaped::Unexaminable => { + eprintln!( + "Warning: leaving stale jail {jail_dir} in place because whether a VMM is still running in it could not be determined." + ); + return Reclamation::Failed(JailError::JailUnexaminable { + path: jail_dir.to_owned(), + }); + }, } // The cgroup goes first, and the chroot only once the cgroup is gone. The @@ -777,6 +785,34 @@ mod tests { ); } + #[test] + fn a_jail_that_could_not_be_examined_is_left_in_place() { + // The reap could not establish whether a VMM is in there. Removing the + // tree on that would be the same destructive step as removing it under a + // VMM known to be alive, so it gets the same answer. + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")); + state.create().unwrap(); + let unknown = VmId::from_chroot_name("unknown".to_owned()); + fs::create_dir_all(state.jail_root(&unknown)).unwrap(); + + let err = sweep_jails_with( + &state.jail_parent(), + |_jail_root| Reaped::Unexaminable, + |_vm_id| Ok(()), + ) + .unwrap_err(); + + assert!( + state.jail_dir(&unknown).exists(), + "not ours to delete blind" + ); + assert!( + matches!(err, JailError::JailUnexaminable { .. }), + "a jail that could not be checked fails the job: {err}" + ); + } + #[test] fn a_cleared_jail_is_still_swept() { let (_dir, root) = temp_root(); From ec747989a0788d5a14a573e407ac2310803815d4 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 05:39:07 +0000 Subject: [PATCH 62/91] Write down what a failing teardown step does Three defects in this module were one mistake: a fallible step whose failure never reached the mechanism built to retry it. Each was found by someone reading the code rather than by the code making the choice hard to skip, and the default a step falls into when nobody decides is `eprintln!`, which is the one option that neither fails the job nor arms a retry. So the choice is written down. Every fallible step in the sweep, the reap, the chroot teardown, the cgroup teardown, and host preparation is enumerated against the three things it can do with a failure, with the reason beside it. The enumeration is the point: a step added here has to appear in it, and picking a column means saying out loud whether the host can still be trusted to measure. --- plus/bencher_runner/src/jail/cgroup.rs | 4 ++ plus/bencher_runner/src/jail/mod.rs | 51 ++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index b7c5a1fc7..74290c645 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -283,6 +283,10 @@ impl CgroupManager { /// that survive a direct-child kill, e.g. on timeout or cancellation, /// so no stray work lingers on benchmark cores and the cgroup can be /// removed. + /// + /// Best effort is sound here only because something else catches it: + /// whatever this fails to kill is exactly what makes [`Self::cleanup`]'s + /// `rmdir` fail, and that arms the reclaim signal. See [`crate::jail`]. pub fn kill_all(&self) { if let Err(e) = self.write_file("cgroup.kill", "1") { eprintln!("Warning: failed to kill cgroup subtree: {e}"); diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 3a0532338..4bb71b5dc 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -5,6 +5,57 @@ //! the persistent state directory the chroots are built under, the empty //! network namespace the VMM joins, and the cgroup that both places it on the //! benchmark cores and bounds its resources. +//! +//! # What a failing teardown step does +//! +//! Reclamation is where this module is easiest to get wrong, because every step +//! of it runs after the thing it is cleaning up already happened, so a failure +//! has no obvious caller to tell. Three separate defects here were the same +//! mistake: a fallible step whose failure never reached the mechanism built to +//! retry it. So every fallible step does exactly one of three things, and adding +//! a step means choosing which, rather than reaching for `eprintln!`: +//! +//! - **Fails the job.** The host cannot be trusted to measure. Used wherever +//! something may still be running on the benchmark cores, including wherever +//! the runner could not establish that nothing is. Recoverable by +//! construction: nothing latches, so the next job tries the whole thing again. +//! - **Arms the retry.** [`ReclaimFailed`], read by +//! [`HostPreparation::ensure`]. Used where the cost is disk rather than a +//! contended benchmark, and where the code has no caller to report to at all, +//! which is every step reached from a `Drop`. +//! - **Ignored.** Only where the failure is itself the answer, or where a later +//! step is guaranteed to catch it. Each one below says which. +//! +//! | Step | On failure | +//! |---|---| +//! | [`HostPreparation::ensure`]: the root check | fails the job | +//! | [`HostPreparation::ensure`]: creating the state directory | fails the job | +//! | [`HostPreparation::ensure`]: reading `/etc/passwd`, `/etc/group` | ignored: the check is advisory and cannot see a directory service anyway | +//! | [`HostPreparation::ensure`]: taking the jail lock | fails the job | +//! | [`HostPreparation::ensure`]: a sweep that returns an error | fails the job | +//! | [`HostPreparation::ensure`]: a sweep that leaves a chroot behind | arms the retry | +//! | [`state::sweep_jails`]: the jail parent is absent | nothing to sweep | +//! | [`state::sweep_jails`]: the jail parent cannot be read | fails the job | +//! | [`state::sweep_jails`]: an entry cannot be read | fails the job | +//! | [`state::sweep_jails`]: an entry's kind cannot be read | fails the job: it may be a jail | +//! | [`state::sweep_jails`]: a name that is not UTF-8 | ignored: every name here is a UUID this runner minted, so it is not ours | +//! | [`state::sweep_jails`]: the reap reports a live VMM | fails the job | +//! | [`state::sweep_jails`]: the reap could not examine the jail | fails the job | +//! | [`state::sweep_jails`]: removing the cgroup | fails the job, and the chroot is kept because its name is the cgroup's only handle | +//! | [`state::sweep_jails`]: removing the chroot | arms the retry | +//! | [`reap::reap_jailed_vmm`]: the jail root is absent | clear: nothing can be chrooted into a directory that is not there | +//! | [`reap::reap_jailed_vmm`]: the jail root cannot be stat'ed | reported unexaminable, which fails the job | +//! | [`reap::reap_jailed_vmm`]: `/proc` cannot be listed | reported unexaminable | +//! | [`reap::reap_jailed_vmm`]: a `/proc//root` cannot be read | ignored: that process is gone or is not this jail's | +//! | [`reap::reap_jailed_vmm`]: pinning or signalling the VMM | reported still running, which fails the job | +//! | [`reap::reap_jailed_vmm`]: a VMM that will not exit | reported still running | +//! | [`reap::reap_jailed_vmm`]: the rescan bound runs out | reported still running | +//! | [`reap::reap_jailed_vmm`]: a `/proc//status` that cannot be read | ignored: the process is gone, which is what was being asked | +//! | [`JailDir`] teardown: the chroot is already gone | ignored: that is the goal state | +//! | [`JailDir`] teardown: removing the chroot | arms the retry | +//! | [`JailDir`] teardown: the retry is already armed | keeps the chroot, since its name is the cgroup's only handle | +//! | [`CgroupManager`] teardown: `rmdir` of the cgroup | arms the retry | +//! | [`CgroupManager`] teardown: killing the cgroup's survivors | ignored: whatever survives is what makes the `rmdir` above fail, which arms the retry | #[cfg(target_os = "linux")] mod cgroup; From fa1dfc148cde3bde60f6de5d188d0a518d462a05 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 05:39:21 +0000 Subject: [PATCH 63/91] Say that the jail uid is shared across concurrent runners The rationale for one dedicated uid rested on jobs running serially, which is true per runner and not per host. Two runners with different state directories hold different jail locks, so they can have jobs in flight at once as the same uid, and that is a configuration the network namespace module is already built around. Two VMMs sharing a uid can signal each other and, where ptrace_scope permits, trace each other, which is the hazard the jail user type exists to describe. The sentence rather than a per-runner uid: allocating ids without coordination between runners that by construction do not know about each other trades a known hazard for an unknown one, and an operator running two runners on one host can already give them different --jail-uid values, which is what the flag is for. --- plus/bencher_runner/src/jail/mod.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 4bb71b5dc..3d4887618 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -97,6 +97,18 @@ pub const DEFAULT_STATE_DIR: &str = "/var/lib/bencher-runner"; /// chroot that is swept, so a per-job allocator adds a scheme without closing /// a live vector. /// +/// Serially per runner, which is what the jail lock enforces. Two runners with +/// different `--state-dir` values hold different locks and can therefore have +/// jobs in flight at the same time, as the same uid, and that is a supported +/// configuration: the network namespace module is built around it. Two VMMs +/// sharing a uid can signal each other and, where `ptrace_scope` permits, trace +/// each other, which is the hazard [`JailUser`] describes. A per-runner id would +/// close it, and is not attempted here: the ids would have to be allocated +/// without coordination between runners that by construction do not know about +/// each other, which trades a known hazard for an unknown one. An operator +/// running two runners on one host should give them different `--jail-uid` +/// values, which is what that flag is for. +/// /// The number is Bencher's historic default self-hosted API server port, /// retired in favor of the IANA-registered 6610, so it reads as a project /// convention rather than an arbitrary pick. It also lands in the unallocated From d4d793f1c00fc7f3832dd42b9e7b3770ddc53409 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 05:39:21 +0000 Subject: [PATCH 64/91] Stop the scenarios' assertions passing without looking `assert_no_chroot_remains` treated a jail parent it could not read as an empty one, so the assertion that catches a leaked chroot passed vacuously exactly when something was wrong. That is the same shape as the product defects this branch has been fixing, in the place where it costs most: a test that cannot fail is worse than no test, because it reads as coverage. The same sweep over the rest of the file. `find_jail` and the harness's `find_jailed_vmm` no longer read a failed listing as "nothing yet", which would have surfaced three minutes later as a probe timeout blaming the runner for what the harness could not see. `is_firecracker` distinguishes a process that is gone from a `comm` it could not read, since the assertion using it passes on false. The `exists` checks that pass on absence became `try_exists`, which does not report an error as absence. The sibling of last round's probe timeout gets the same treatment too: a run that never observed the VMM is killed before it is waited on, so the failure reports now rather than after the runner's own timeout, and only when it has not already been reaped. --- tasks/test_runner/src/task/scenarios.rs | 142 ++++++++++++++++++------ 1 file changed, 110 insertions(+), 32 deletions(-) diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index d4897ed68..58ace85ae 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -2658,18 +2658,33 @@ fn assert_job_succeeded(output: &ScenarioOutput, marker: &str) -> Result<()> { /// a full guest rootfs image. fn assert_no_chroot_remains(state_dir: &Utf8Path) -> Result<()> { let parent = jail_parent(state_dir); - let leftovers: Vec = match fs::read_dir(&parent) { - Ok(entries) => entries - .flatten() - .map(|entry| entry.file_name().to_string_lossy().into_owned()) - .collect(), - Err(_) => Vec::new(), + // A read that failed is not an empty directory. An assertion that could not + // look has not passed, it has not run, and a vacuous pass here would hide the + // exact leak it exists to catch. Absence is the one reading that does mean + // nothing was left behind: the runner creates this tree on demand. + let entries = match fs::read_dir(&parent) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => { + return Err(e).with_context(|| { + format!("Failed to read {parent}, so whether a chroot was left behind is unknown") + }); + }, }; - if leftovers.is_empty() { - Ok(()) - } else { - bail!("Chroots left behind under {parent}: {leftovers:?}") + + let mut leftovers = Vec::new(); + for entry in entries { + let entry = entry.with_context(|| { + format!("Failed to read an entry under {parent}, so whether a chroot was left behind is unknown") + })?; + leftovers.push(entry.file_name().to_string_lossy().into_owned()); } + + anyhow::ensure!( + leftovers.is_empty(), + "Chroots left behind under {parent}: {leftovers:?}" + ); + Ok(()) } /// Check that the jailed VMM is unprivileged and already in its cgroup. @@ -2680,10 +2695,10 @@ fn assert_no_chroot_remains(state_dir: &Utf8Path) -> Result<()> { /// not after the VM is running. fn probe_confinement(state_dir: &Utf8Path) -> Result { let parent = jail_parent(state_dir); - let Some((vm_id, jail_root)) = find_jail(&parent) else { + let Some((vm_id, jail_root)) = find_jail(&parent)? else { return Ok(false); }; - let Some(pid) = find_jailed_vmm(&jail_root) else { + let Some(pid) = find_jailed_vmm(&jail_root)? else { return Ok(false); }; @@ -2699,18 +2714,33 @@ fn probe_confinement(state_dir: &Utf8Path) -> Result { } /// Find the single chroot under the jail parent, if one exists yet. -fn find_jail(parent: &Utf8Path) -> Option<(String, Utf8PathBuf)> { - for entry in fs::read_dir(parent).ok()?.flatten() { - if !entry.file_type().is_ok_and(|file_type| file_type.is_dir()) { +/// +/// `Ok(None)` is "not yet", which the parent not existing also means: the runner +/// creates it on demand. Every other failure is an error, because this drives a +/// poll loop whose only other outcome is a timeout, and a timeout would report +/// that no VMM ever appeared when the truth is that nobody could look. +fn find_jail(parent: &Utf8Path) -> Result> { + let entries = match fs::read_dir(parent) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e).with_context(|| format!("Failed to read {parent}")), + }; + + for entry in entries { + let entry = entry.with_context(|| format!("Failed to read an entry under {parent}"))?; + let file_type = entry + .file_type() + .with_context(|| format!("Failed to read the kind of an entry under {parent}"))?; + if !file_type.is_dir() { continue; } let vm_id = entry.file_name().to_string_lossy().into_owned(); let jail_root = parent.join(&vm_id).join("root"); if jail_root.is_dir() { - return Some((vm_id, jail_root)); + return Ok(Some((vm_id, jail_root))); } } - None + Ok(None) } /// Find the pid of the VMM confined to `jail_root`, if it is running yet. @@ -2721,24 +2751,36 @@ fn find_jail(parent: &Utf8Path) -> Option<(String, Utf8PathBuf)> { /// device and inode of the chroot directory, so stat'ing through /// `/proc//root` and stat'ing the jail root agree for exactly the VMM /// confined to this jail and for no other process on the host. -fn find_jailed_vmm(jail_root: &Utf8Path) -> Option { +/// +/// `Ok(None)` is "not running yet", which a jail root that does not exist also +/// means. A jail root that cannot be stat'ed, or a `/proc` that cannot be listed, +/// is neither: it would surface as a probe timeout blaming the runner for +/// something the harness could not see. +fn find_jailed_vmm(jail_root: &Utf8Path) -> Result> { use std::os::unix::fs::MetadataExt as _; - let jail = fs::metadata(jail_root).ok()?; - for entry in fs::read_dir("/proc").ok()?.flatten() { + let jail = match fs::metadata(jail_root) { + Ok(jail) => jail, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e).with_context(|| format!("Failed to stat {jail_root}")), + }; + for entry in fs::read_dir("/proc").context("Failed to read /proc")? { + let entry = entry.context("Failed to read a /proc entry")?; let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { continue; }; // Following the magic symlink crosses into the process's own mount - // namespace, which a privileged reader is allowed to do. + // namespace, which a privileged reader is allowed to do. A read that + // fails is a process that has exited or is not this jail's, which is the + // one failure here that is genuinely an answer. let Ok(root) = fs::metadata(format!("/proc/{pid}/root")) else { continue; }; if root.dev() == jail.dev() && root.ino() == jail.ino() { - return Some(pid); + return Ok(Some(pid)); } } - None + Ok(None) } /// Check the VMM dropped root and runs as the user the jail was handed to. @@ -2857,8 +2899,8 @@ fn run_runner_after_orphan( // empty directory created microseconds before the kill. let deadline = std::time::Instant::now() + PROBE_TIMEOUT; let orphan = loop { - if let Some((vm_id, jail_root)) = find_jail(&parent) - && let Some(pid) = find_jailed_vmm(&jail_root) + if let Some((vm_id, jail_root)) = find_jail(&parent)? + && let Some(pid) = find_jailed_vmm(&jail_root)? { break Some((vm_id, jail_root, pid)); } @@ -2889,7 +2931,10 @@ fn run_runner_after_orphan( drop(child.wait()); drop(readers.join()); - if !jail_root.exists() { + if !jail_root + .try_exists() + .with_context(|| format!("Failed to check whether {jail_root} was left behind"))? + { bail!( "The chroot {jail_root} was reclaimed despite the runner being killed without unwinding, so the sweep is untested" ); @@ -2902,17 +2947,24 @@ fn run_runner_after_orphan( // that a hand-reap guards against is exactly what the sweep now exists to // prevent, so if the sweep fails this scenario has to go red. let cgroup = stale_cgroup(&vm_id); - let cgroup_existed = cgroup.exists(); + let cgroup_existed = cgroup + .try_exists() + .with_context(|| format!("Failed to check whether {cgroup} was created"))?; println!( " orphaned jail {vm_id} (VMM pid {vmm_pid}, cgroup present: {cgroup_existed}), running a second job..." ); let output = run_runner(image_path, args, runner_bin)?; - if jail_root.exists() { + // `try_exists`, not `exists`: the latter reports false for an error as well + // as for absence, which would pass this assertion for the wrong reason. + if jail_root + .try_exists() + .with_context(|| format!("Failed to check whether {jail_root} survived"))? + { bail!("The orphaned chroot {jail_root} survived the next job, so it was never swept"); } - if is_firecracker(vmm_pid) { + if is_firecracker(vmm_pid)? { bail!( "The orphaned VMM (pid {vmm_pid}) is still running after the next job, so the sweep never reaped it. It still holds the benchmark cores." ); @@ -2920,7 +2972,11 @@ fn run_runner_after_orphan( // Only meaningful where a cgroup was created at all: a host with no CPU // isolation never makes one, and asserting its absence would pass for the // wrong reason. - if cgroup_existed && cgroup.exists() { + if cgroup_existed + && cgroup + .try_exists() + .with_context(|| format!("Failed to check whether {cgroup} survived"))? + { bail!( "The orphaned cgroup {cgroup} survived the next job, so the sweep never removed it. Stale cgroups accumulate, and one that will not go away usually means its VMM is still running." ); @@ -2938,8 +2994,21 @@ fn stale_cgroup(vm_id: &str) -> Utf8PathBuf { /// /// Checking the command as well as the pid keeps a recycled pid from reading /// as a VMM that was never reaped. -fn is_firecracker(pid: u32) -> bool { - fs::read_to_string(format!("/proc/{pid}/comm")).is_ok_and(|comm| comm.trim() == "firecracker") +/// +/// A process that is gone has no `comm` to read, and that is the answer the +/// caller wants. Any other read failure is not: the assertion that uses this +/// passes when it returns false, so swallowing an error would pass it for the +/// wrong reason. +fn is_firecracker(pid: u32) -> Result { + match fs::read_to_string(format!("/proc/{pid}/comm")) { + Ok(comm) => Ok(comm.trim() == "firecracker"), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e).with_context(|| { + format!( + "Failed to read the command of pid {pid}, so whether the VMM was reaped is unknown" + ) + }), + } } /// Reader threads draining a child's piped output. @@ -3035,6 +3104,15 @@ fn run_runner_with_probe( std::thread::sleep(PROBE_INTERVAL); } + // A probe that ended without observing the VMM leaves the runner going, and + // waiting on it would sit there until the runner's own timeout expired, + // reporting the failure minutes late. Only a run that observed what it came + // for is allowed to finish, since its output is the result being collected. + // Guarded on the reap, because `try_wait` above reaps and signalling a reaped + // pid can reach whatever inherited the number. + if !matches!(observed, Some(Ok(()))) && child.try_wait()?.is_none() { + kill_pid(child.id(), libc::SIGKILL); + } let status = child.wait()?; let (stdout, stderr) = readers.join(); From 67ad1de3365f06a84f49e342e38ecca66a9605e3 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 06:54:00 +0000 Subject: [PATCH 65/91] Never report a measurement read that did not happen The rule the teardown audit produced is not about teardown: a read that fails must never be reported as a state that was observed. Measurement breaks it more easily than teardown does, because a failed read still leaves a plausible value in hand. Five places took one: `verify_cpuset` treated an effective set it could not read as an undelegated controller, which is the finding that prompted this. Reaching that read means the write to `cpuset.cpus` already succeeded, so the controller is demonstrably delegated: the degrade told the operator something the code had just disproved, and the run went ahead on a cpuset nobody read back. The read back is the whole mechanism, so a read that did not happen is a failure of it. `apply_cpuset` used `exists` on `cpuset.cpus`, which reports an error as an absent file, so a cgroup the runner could not stat became a host with no isolation to offer. `effective_mems` answered node 0 to any failure, and that value is written to `cpuset.mems`, so a failed read confined the guest's memory to one node on a host that may have several. `enable_controllers` read an unreadable `cgroup.subtree_control` as an empty one and blamed the host for a controller it may well have enabled. The cgroup metrics reported zero CPU usage for a field they could not parse, which is the plainest form of the thing measurement must never do, and the reported type already allowed absence. `CpuLayout::detect` keeps its fallback but announces it. The counted layout assumes cores are numbered 0..n, which is wrong on a host whose online set has gaps, so a layout the runner had to guess now says so rather than being pinned to silently. An absent file still means the host has nothing to tell, and that stays a declared, loud absence of isolation. The distinction throughout is between a host with nothing to offer and a host nobody could ask. --- plus/bencher_runner/src/cpu.rs | 36 +++++- plus/bencher_runner/src/jail/cgroup.rs | 133 +++++++++++++++----- plus/bencher_runner/src/metrics.rs | 70 ++++++----- plus/bencher_runner/src/tuning/partition.rs | 15 ++- 4 files changed, 186 insertions(+), 68 deletions(-) diff --git a/plus/bencher_runner/src/cpu.rs b/plus/bencher_runner/src/cpu.rs index bfab91ea7..5c4f8607e 100644 --- a/plus/bencher_runner/src/cpu.rs +++ b/plus/bencher_runner/src/cpu.rs @@ -17,6 +17,28 @@ use std::fs; #[cfg(target_os = "linux")] use std::io; +/// The kernel's list of online CPU IDs. +#[cfg(target_os = "linux")] +const ONLINE_CPUS: &str = "/sys/devices/system/cpu/online"; + +/// Say that the CPU layout is a guess rather than the host's online set. +/// +/// The count-based fallback numbers cores `0..n`. That is wrong on a host whose +/// online set has gaps, which is the ordinary result of disabling SMT on a +/// topology with interleaved sibling numbering: the online set reads `0,2,4,6` +/// and a counted layout pins the benchmark to cores that are offline. Nothing +/// downstream can tell the difference, so the one place that knows says so. +#[cfg(target_os = "linux")] +#[expect( + clippy::print_stderr, + reason = "a CPU layout the runner had to guess is announced" +)] +fn warn_guessed_layout(reason: &str) { + eprintln!( + "Warning: {reason}. The CPU layout falls back to a core count, which assumes cores are numbered 0..n; on a host whose online set has gaps the benchmark cores may be wrong." + ); +} + /// CPU layout for the runner. /// /// Partitions available cores into housekeeping (for heartbeat, networking) @@ -38,14 +60,18 @@ impl CpuLayout { /// interleaved sibling numbering (common on AMD) leaves a non-contiguous /// online set like `0,2,4,6`, and a count-based layout would pin to /// offline cores. + /// A fallback that cannot read the online set is announced rather than taken + /// quietly: it is a layout the runner guessed, and every cpuset written from + /// it claims cores nobody confirmed are online. #[must_use] pub fn detect() -> Self { #[cfg(target_os = "linux")] - if let Ok(online) = fs::read_to_string("/sys/devices/system/cpu/online") - && let Some(ids) = parse_cpu_id_list(&online) - && !ids.is_empty() - { - return Self::with_cpu_ids(ids); + match fs::read_to_string(ONLINE_CPUS) { + Ok(online) => match parse_cpu_id_list(&online).filter(|ids| !ids.is_empty()) { + Some(ids) => return Self::with_cpu_ids(ids), + None => warn_guessed_layout(&format!("{ONLINE_CPUS} reads as '{}'", online.trim())), + }, + Err(e) => warn_guessed_layout(&format!("{ONLINE_CPUS} could not be read: {e}")), } Self::with_core_count(Self::available_cores()) diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index 74290c645..20288100f 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -102,8 +102,14 @@ impl CgroupManager { .or_else(|_| fs::write(&subtree_control, "+cpu +memory +pids +io")) .or_else(|_| fs::write(&subtree_control, "+cpu +memory +pids")); - // Verify that required controllers are enabled - let enabled = fs::read_to_string(&subtree_control).unwrap_or_default(); + // Verify that required controllers are enabled. A read that failed is + // not an empty list: reporting one would blame the host for a + // controller it may well have enabled, on the strength of a question + // nobody answered. + let enabled = fs::read_to_string(&subtree_control).map_err(|e| JailError::ReadCgroup { + path: subtree_control.clone(), + source: e, + })?; if let Some(missing) = missing_required_controller(&enabled) { return Err(match write_result { Err(e) => JailError::EnableControllers { @@ -158,9 +164,16 @@ impl CgroupManager { return Ok(Cpuset::Unavailable("the benchmark core set is empty")); } + // `try_exists`, not `exists`: the latter reports an error as an absent + // file, so a cgroup directory the runner cannot stat would be declared + // an undelegated controller. That is a claim about the host made from a + // question that failed, and it is the difference between a host with no + // isolation to offer and a host nobody could ask. let path = self.cgroup_path.join("cpuset.cpus"); - if !path.exists() { - return Ok(Cpuset::Unavailable(UNDELEGATED)); + match path.try_exists() { + Ok(true) => {}, + Ok(false) => return Ok(Cpuset::Unavailable(UNDELEGATED)), + Err(e) => return Err(JailError::ReadCgroup { path, source: e }.into()), } if let Err(e) = fs::write(&path, &cpuset) { return classify_cpuset_error(path, e); @@ -169,10 +182,13 @@ impl CgroupManager { // Also need to set cpuset.mems for cpuset to work. Use the parent's // effective memory nodes so multi-node NUMA hosts are not forced onto // node 0. Applied cpus without mems is the half-applied case. - let mems = self - .cgroup_path - .parent() - .map_or_else(|| "0".to_owned(), effective_mems); + let mems = match self.cgroup_path.parent() { + Some(parent) => effective_mems(parent).map_err(|e| JailError::ReadCgroup { + path: parent.join(MEMS_EFFECTIVE), + source: e, + })?, + None => NODE_ZERO.to_owned(), + }; let mems_path = self.cgroup_path.join("cpuset.mems"); if let Err(e) = fs::write(&mems_path, &mems) { return classify_cpuset_error(mems_path, e); @@ -204,15 +220,17 @@ impl CgroupManager { ("cpuset.mems.effective", mems), ] { let path = self.cgroup_path.join(file); - let effective = match fs::read_to_string(&path) { - Ok(effective) => effective, - // Nothing to read back means nothing was delegated, the same - // conclusion the write path draws. - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - return Ok(Cpuset::Unavailable(UNDELEGATED)); - }, - Err(e) => return Err(JailError::ReadCgroup { path, source: e }.into()), - }; + // Every failure, absence included. Reaching this function means the + // write to `cpuset.cpus` succeeded, which is proof the controller is + // delegated, so a missing effective file here cannot mean it is not: + // reporting an undelegated controller would tell the operator + // something this code just observed to be false, while the run went + // ahead on a cpuset that was never verified. The read is the whole + // mechanism, so a read that did not happen is a failure of it. + let effective = fs::read_to_string(&path).map_err(|e| JailError::ReadCgroup { + path: path.clone(), + source: e, + })?; if parse_cpuset(&effective) != parse_cpuset(requested) { return Err(JailError::CpusetNarrowed { @@ -324,16 +342,31 @@ impl Drop for CgroupManager { } } +/// The `cpuset.mems.effective` file, read to mirror a parent's memory nodes. +pub(crate) const MEMS_EFFECTIVE: &str = "cpuset.mems.effective"; + +/// The single memory node a host without a readable node set is assumed to have. +const NODE_ZERO: &str = "0"; + /// Read a cgroup's effective memory nodes (`cpuset.mems.effective`). /// -/// Falls back to node `0` when the file is missing or empty (e.g., the -/// cpuset controller is not enabled). Using effective mems instead of a -/// hardcoded node keeps multi-node NUMA hosts from forcing all benchmark +/// Falls back to node `0` when the file is missing or empty, which is what the +/// cpuset controller not being enabled looks like. Using effective mems instead +/// of a hardcoded node keeps multi-node NUMA hosts from forcing all benchmark /// memory onto node 0. -pub(crate) fn effective_mems(cgroup: &Utf8Path) -> String { - match fs::read_to_string(cgroup.join("cpuset.mems.effective")) { - Ok(mems) if !mems.trim().is_empty() => mems.trim().to_owned(), - _ => "0".to_owned(), +/// +/// Which is exactly why any other failure is an error rather than the fallback. +/// This value is written to `cpuset.mems`, so answering node 0 to a read that +/// did not happen confines the guest's memory to one node on a host that may +/// have several, and the run then reports a number measured under a constraint +/// nobody chose. An absent file says the host has nothing to tell; a failed read +/// says nobody asked it. +pub(crate) fn effective_mems(cgroup: &Utf8Path) -> Result { + match fs::read_to_string(cgroup.join(MEMS_EFFECTIVE)) { + Ok(mems) if !mems.trim().is_empty() => Ok(mems.trim().to_owned()), + Ok(_) => Ok(NODE_ZERO.to_owned()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(NODE_ZERO.to_owned()), + Err(e) => Err(e), } } @@ -547,7 +580,12 @@ mod tests { } #[test] - fn an_undelegated_memory_node_set_degrades() { + fn a_memory_node_set_that_cannot_be_read_back_fails_the_job() { + // This asserted a degrade until the rule was written down. The cpus were + // applied and verified, so the controller is demonstrably delegated; + // reporting it undelegated because the mems could not be read back would + // tell the operator something this function just disproved, and the run + // would go ahead on a memory binding nobody confirmed. let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); fs::write(root.join("cpuset.cpus"), "").unwrap(); @@ -556,9 +594,11 @@ mod tests { let manager = CgroupManager::detached(root); let layout = CpuLayout::with_core_count(8); - assert_eq!( - manager.apply_cpuset(&layout).unwrap(), - Cpuset::Unavailable(UNDELEGATED) + let err = manager.apply_cpuset(&layout).unwrap_err().to_string(); + + assert!( + err.contains("cpuset.mems.effective"), + "names the read that did not happen: {err}" ); } @@ -730,13 +770,44 @@ mod tests { assert!(!manager.contains_pid(789).unwrap()); } + #[test] + fn an_unreadable_node_set_is_not_node_zero() { + // The value is written to `cpuset.mems`, so answering node 0 to a read + // that failed would confine the guest to one node on a host that may + // have several, and the run would report a number measured under a + // constraint nobody chose. A file in place of the cgroup directory reads + // back `ENOTDIR`, the same way an unlistable one reads back `EACCES`. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let not_a_dir = root.join("cgroup"); + fs::write(¬_a_dir, b"in the way").unwrap(); + + effective_mems(¬_a_dir).unwrap_err(); + } + + #[test] + fn a_cpuset_that_cannot_be_verified_is_not_a_degrade() { + // The write proves the controller is delegated, so a missing effective + // file cannot mean it is not. Degrading here would tell the operator the + // controller was never delegated while the run went ahead on a cpuset + // nobody read back. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + fs::write(root.join("cpuset.cpus"), "").unwrap(); + fs::write(root.join("cpuset.mems"), "").unwrap(); + let manager = CgroupManager::detached(root); + let layout = CpuLayout::with_core_count(8); + + manager.apply_cpuset(&layout).unwrap_err(); + } + #[test] fn effective_mems_reads_file() { let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); fs::write(root.join("cpuset.mems.effective"), "0-1\n").unwrap(); - assert_eq!(effective_mems(&root), "0-1"); + assert_eq!(effective_mems(&root).unwrap(), "0-1"); } #[test] @@ -744,7 +815,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); - assert_eq!(effective_mems(&root), "0"); + assert_eq!(effective_mems(&root).unwrap(), "0"); } #[test] @@ -753,6 +824,6 @@ mod tests { let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); fs::write(root.join("cpuset.mems.effective"), "\n").unwrap(); - assert_eq!(effective_mems(&root), "0"); + assert_eq!(effective_mems(&root).unwrap(), "0"); } } diff --git a/plus/bencher_runner/src/metrics.rs b/plus/bencher_runner/src/metrics.rs index 4bba7d2dc..461f4c9d3 100644 --- a/plus/bencher_runner/src/metrics.rs +++ b/plus/bencher_runner/src/metrics.rs @@ -48,18 +48,24 @@ pub struct CgroupMetrics { /// /// Reads `cpu.stat` and `memory.peak` from the cgroup directory. /// Returns `None` if the path doesn't exist. +/// +/// Every field is optional, and a field that could not be read stays absent +/// rather than becoming a number. These are reported to an operator as measured +/// values, so a zero standing in for a failed read is the plainest form of the +/// one thing measurement is never allowed to do. Absence is already how this +/// reports "no cgroup at all", so it costs nothing to be honest per field. pub fn read_cgroup_metrics(cgroup_path: &Utf8Path) -> Option { if !cgroup_path.exists() { return None; } - let cpu_stat = read_cpu_stat(cgroup_path); + let cpu_stat = read_cpu_stat(cgroup_path).unwrap_or_default(); let memory_peak = read_file_u64(&cgroup_path.join("memory.peak")); Some(CgroupMetrics { - cpu_usage_us: cpu_stat.as_ref().map(|s| s.usage_usec), - cpu_user_us: cpu_stat.as_ref().map(|s| s.user_usec), - cpu_system_us: cpu_stat.as_ref().map(|s| s.system_usec), + cpu_usage_us: cpu_stat.usage_usec, + cpu_user_us: cpu_stat.user_usec, + cpu_system_us: cpu_stat.system_usec, memory_peak_bytes: memory_peak, }) } @@ -72,37 +78,37 @@ pub fn format_metrics(metrics: &RunMetrics) -> Option { Some(format!("---BENCHER_METRICS:{json}---")) } +/// The three fields of `cpu.stat` this runner reports. +/// +/// Each one is what the file said, or nothing. A field the file did not carry, or +/// carried unparseably, is not zero usage: zero is a measurement, and this never +/// measured it. +#[derive(Default)] #[expect( clippy::struct_field_names, reason = "matches cgroup cpu.stat field names" )] struct CpuStat { - usage_usec: u64, - user_usec: u64, - system_usec: u64, + usage_usec: Option, + user_usec: Option, + system_usec: Option, } fn read_cpu_stat(cgroup_path: &Utf8Path) -> Option { let content = std::fs::read_to_string(cgroup_path.join("cpu.stat")).ok()?; - let mut usage = None; - let mut user = None; - let mut system = None; + let mut stat = CpuStat::default(); for line in content.lines() { let mut parts = line.split_whitespace(); match (parts.next(), parts.next()) { - (Some("usage_usec"), Some(v)) => usage = v.parse().ok(), - (Some("user_usec"), Some(v)) => user = v.parse().ok(), - (Some("system_usec"), Some(v)) => system = v.parse().ok(), + (Some("usage_usec"), Some(v)) => stat.usage_usec = v.parse().ok(), + (Some("user_usec"), Some(v)) => stat.user_usec = v.parse().ok(), + (Some("system_usec"), Some(v)) => stat.system_usec = v.parse().ok(), _ => {}, } } - Some(CpuStat { - usage_usec: usage.unwrap_or(0), - user_usec: user.unwrap_or(0), - system_usec: system.unwrap_or(0), - }) + Some(stat) } fn read_file_u64(path: &Utf8Path) -> Option { @@ -129,21 +135,23 @@ mod tests { fs::write(path.join("cpu.stat"), content).unwrap(); let stat = read_cpu_stat(path).unwrap(); - assert_eq!(stat.usage_usec, 12345); - assert_eq!(stat.user_usec, 6000); - assert_eq!(stat.system_usec, 6345); + assert_eq!(stat.usage_usec, Some(12345)); + assert_eq!(stat.user_usec, Some(6000)); + assert_eq!(stat.system_usec, Some(6345)); } #[test] - fn read_cpu_stat_missing_fields_default_to_zero() { + fn a_field_the_file_did_not_carry_is_absent_not_zero() { + // Zero is a measurement. A field that was never read has to reach the + // operator as missing, which is what the reported type already allows. let dir = tempfile::tempdir().unwrap(); let path = tempdir_utf8(&dir); fs::write(path.join("cpu.stat"), "usage_usec 100\n").unwrap(); let stat = read_cpu_stat(path).unwrap(); - assert_eq!(stat.usage_usec, 100); - assert_eq!(stat.user_usec, 0); - assert_eq!(stat.system_usec, 0); + assert_eq!(stat.usage_usec, Some(100)); + assert_eq!(stat.user_usec, None); + assert_eq!(stat.system_usec, None); } #[test] @@ -157,9 +165,9 @@ mod tests { .unwrap(); let stat = read_cpu_stat(path).unwrap(); - assert_eq!(stat.usage_usec, 0); // parse fails -> unwrap_or(0) - assert_eq!(stat.user_usec, 100); - assert_eq!(stat.system_usec, 0); // no value at all + assert_eq!(stat.usage_usec, None, "a value that would not parse"); + assert_eq!(stat.user_usec, Some(100)); + assert_eq!(stat.system_usec, None, "no value at all"); } #[test] @@ -169,9 +177,9 @@ mod tests { fs::write(path.join("cpu.stat"), "").unwrap(); let stat = read_cpu_stat(path).unwrap(); - assert_eq!(stat.usage_usec, 0); - assert_eq!(stat.user_usec, 0); - assert_eq!(stat.system_usec, 0); + assert_eq!(stat.usage_usec, None); + assert_eq!(stat.user_usec, None); + assert_eq!(stat.system_usec, None); } #[test] diff --git a/plus/bencher_runner/src/tuning/partition.rs b/plus/bencher_runner/src/tuning/partition.rs index e20e7a78a..207578ba9 100644 --- a/plus/bencher_runner/src/tuning/partition.rs +++ b/plus/bencher_runner/src/tuning/partition.rs @@ -89,10 +89,23 @@ impl BencherPartition { ) { return PartitionLevel::Member; } + // A node set that could not be read is not node 0. Degrading to member + // is the declared, loud absence of isolation; writing a guessed node set + // would confine the benchmark's memory on the strength of a read that + // did not happen. + let mems = match effective_mems(&self.root) { + Ok(mems) => mems, + Err(e) => { + eprintln!( + "Warning: failed to read the cgroup root's effective memory nodes ({e}); no cpuset partition" + ); + return PartitionLevel::Member; + }, + }; if !save_and_write( guard, &self.path.join("cpuset.mems"), - &effective_mems(&self.root), + &mems, "bencher cpuset.mems", ) { return PartitionLevel::Member; From 402628e664ff5e63cc9d31fea6b174937bd75d4d Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 06:54:34 +0000 Subject: [PATCH 66/91] Keep a chroot for its own cgroup, not for any stale jail The chroot teardown read the runner-wide reclaim signal to decide whether to hold its tree, and that signal is raised by more than the thing its comment described. A sweep that could not reclaim some unrelated stale jail raises it too, so the current job held its own chroot even when its own cgroup came down cleanly. Bounded at about two extra trees and self-healing, so not a leak, but the comment claimed the rule was narrower than it was, and a document that describes the code inaccurately is worse than no document. Narrowed rather than reworded. Two questions were sharing one flag, which is what made the earlier ordering bugs easy to write in the first place. A job now carries both signals together: the runner-wide one that earns a later job another sweep, and a job-scoped one saying this job's own cgroup outlived its teardown. A cgroup that will not go away raises both, because both are true, and the chroot reads only the second. --- plus/bencher_runner/src/firecracker/mod.rs | 6 +- plus/bencher_runner/src/jail/cgroup.rs | 34 +++---- plus/bencher_runner/src/jail/chroot.rs | 51 +++++----- plus/bencher_runner/src/jail/mod.rs | 110 ++++++++++++++++++--- plus/bencher_runner/src/local_isolation.rs | 4 +- plus/bencher_runner/src/vm.rs | 23 ++--- 6 files changed, 154 insertions(+), 74 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/mod.rs b/plus/bencher_runner/src/firecracker/mod.rs index 954d8931b..4aaec51d9 100644 --- a/plus/bencher_runner/src/firecracker/mod.rs +++ b/plus/bencher_runner/src/firecracker/mod.rs @@ -32,7 +32,7 @@ use std::time::{Duration, Instant}; use camino::Utf8PathBuf; use crate::cpu::CpuLayout; -use crate::jail::{CgroupManager, Cpuset, JailPaths, JailUser, ReclaimFailed, VmId}; +use crate::jail::{CgroupManager, Cpuset, JailPaths, JailSignals, JailUser, VmId}; use crate::metrics::{self, RunMetrics}; pub use error::FirecrackerError; @@ -76,7 +76,7 @@ pub struct FirecrackerJobConfig { /// Shared with the chroot guard of the same id: a cgroup this job cannot /// remove has to hold that chroot, which is the only handle a later sweep /// has for finding the cgroup again. - pub reclaim_failed: ReclaimFailed, + pub signals: JailSignals, /// Number of vCPUs. pub vcpus: u8, /// Memory size in MiB. @@ -135,7 +135,7 @@ pub fn run_firecracker( // Step 0: Create cgroup with cpuset if CPU layout is provided let cgroup = if let Some(layout) = &config.cpu_layout { if layout.has_isolation() { - match CgroupManager::new(vm_id, config.reclaim_failed.clone()) { + match CgroupManager::new(vm_id, config.signals.clone()) { Ok(cg) => { // A cgroup that exists but does not confine the VMM to the // benchmark cores would report a number measured somewhere diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index 20288100f..3fe358164 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -9,7 +9,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use crate::RunnerError; use crate::cpu::CpuLayout; use crate::error::JailError; -use crate::jail::{ReclaimFailed, VmId}; +use crate::jail::{JailSignals, VmId}; /// Default cgroup v2 mount point. const CGROUP_ROOT: &str = "/sys/fs/cgroup"; @@ -21,9 +21,9 @@ pub(crate) const BENCHER_CGROUP_BASE: &str = "bencher"; pub struct CgroupManager { cgroup_path: Utf8PathBuf, created: bool, - /// Raised when this cgroup could not be removed, so a later sweep retries - /// it and the chroot that names it is held until then. - reclaim_failed: ReclaimFailed, + /// Raised when this cgroup could not be removed, which holds the chroot that + /// names it and earns a later job another sweep. + signals: JailSignals, } impl CgroupManager { @@ -33,7 +33,7 @@ impl CgroupManager { /// cannot be removed has to keep that directory alive, because the /// directory name is the only handle a later sweep has for finding this /// cgroup again. - pub fn new(vm_id: &VmId, reclaim_failed: ReclaimFailed) -> Result { + pub fn new(vm_id: &VmId, signals: JailSignals) -> Result { let cgroup_path = Utf8PathBuf::from(CGROUP_ROOT) .join(BENCHER_CGROUP_BASE) .join(vm_id.as_str()); @@ -70,7 +70,7 @@ impl CgroupManager { Ok(Self { cgroup_path, created, - reclaim_failed, + signals, }) } @@ -84,7 +84,7 @@ impl CgroupManager { Self { cgroup_path, created: false, - reclaim_failed: ReclaimFailed::unwatched(), + signals: JailSignals::unwatched(), } } @@ -328,7 +328,7 @@ impl CgroupManager { "Warning: failed to remove cgroup {}: {e}. Something is still in it, so the next job sweeps it along with the jail that names it.", self.cgroup_path ); - self.reclaim_failed.set(); + self.signals.cgroup_survived(); } else { self.created = false; } @@ -674,12 +674,12 @@ mod tests { let ours = CgroupManager { cgroup_path: root.join("ours"), created: true, - reclaim_failed: ReclaimFailed::unwatched(), + signals: JailSignals::unwatched(), }; let theirs = CgroupManager { cgroup_path: root.join("theirs"), created: false, - reclaim_failed: ReclaimFailed::unwatched(), + signals: JailSignals::unwatched(), }; fs::create_dir_all(ours.path()).unwrap(); fs::create_dir_all(theirs.path()).unwrap(); @@ -710,11 +710,11 @@ mod tests { // nothing for a later sweep to find it by. let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); - let reclaim_failed = ReclaimFailed::default(); + let signals = JailSignals::unwatched(); let mut manager = CgroupManager { cgroup_path: root.join("stuck"), created: true, - reclaim_failed: reclaim_failed.clone(), + signals: signals.clone(), }; fs::create_dir_all(manager.path()).unwrap(); fs::write(manager.path().join("cgroup.procs"), "42\n").unwrap(); @@ -722,8 +722,8 @@ mod tests { manager.cleanup(); assert!( - reclaim_failed.is_set(), - "a cgroup that outlives its job has to earn another sweep" + signals.must_keep_chroot(), + "a cgroup that outlives its job holds the chroot that names it" ); assert!(manager.path().exists()); } @@ -732,11 +732,11 @@ mod tests { fn a_removed_cgroup_leaves_the_signal_alone() { let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); - let reclaim_failed = ReclaimFailed::default(); + let signals = JailSignals::unwatched(); let mut manager = CgroupManager { cgroup_path: root.join("gone"), created: true, - reclaim_failed: reclaim_failed.clone(), + signals: signals.clone(), }; fs::create_dir_all(manager.path()).unwrap(); @@ -744,7 +744,7 @@ mod tests { assert!(!manager.path().exists()); assert!( - !reclaim_failed.is_set(), + !signals.must_keep_chroot(), "a clean teardown must not hold the chroot back" ); } diff --git a/plus/bencher_runner/src/jail/chroot.rs b/plus/bencher_runner/src/jail/chroot.rs index fb4c463c5..058f6c8bc 100644 --- a/plus/bencher_runner/src/jail/chroot.rs +++ b/plus/bencher_runner/src/jail/chroot.rs @@ -14,7 +14,7 @@ use std::os::unix::fs::{PermissionsExt as _, chown}; use camino::{Utf8Path, Utf8PathBuf}; use crate::error::JailError; -use crate::jail::{JailUser, ReclaimFailed, StateDir, VmId}; +use crate::jail::{JailSignals, JailUser, StateDir, VmId}; /// A job's chroot tree, removed when this value is dropped. /// @@ -22,23 +22,20 @@ use crate::jail::{JailUser, ReclaimFailed, StateDir, VmId}; /// `Drop` covers completion, timeout, cancellation, and every error return; /// the sweep in `prepare_host` covers the exits that never unwind. /// -/// The cgroup's teardown runs first, which is what makes the reclaim signal -/// meaningful here: `run_firecracker` owns the cgroup and returns before this -/// guard is dropped. +/// The cgroup's teardown runs first, which is what makes its signal meaningful +/// here: `run_firecracker` owns this job's cgroup and returns before this guard +/// is dropped. The signal read is this job's own, not the runner-wide one a +/// stale jail elsewhere can raise. #[derive(Debug)] pub struct JailDir { dir: Utf8PathBuf, root: Utf8PathBuf, - reclaim_failed: ReclaimFailed, + signals: JailSignals, } impl JailDir { /// Create the chroot tree for `vm_id` at mode 0700. - pub fn create( - state: &StateDir, - vm_id: &VmId, - reclaim_failed: ReclaimFailed, - ) -> Result { + pub fn create(state: &StateDir, vm_id: &VmId, signals: JailSignals) -> Result { let dir = state.jail_dir(vm_id); let root = state.jail_root(vm_id); @@ -58,11 +55,7 @@ impl JailDir { })?; } - Ok(Self { - dir, - root, - reclaim_failed, - }) + Ok(Self { dir, root, signals }) } /// The chroot root, which becomes `/` inside the jail. @@ -74,13 +67,17 @@ impl JailDir { impl Drop for JailDir { fn drop(&mut self) { - // A cgroup this job could not remove keeps the chroot alive. The two + // A cgroup *this job* could not remove keeps the chroot alive. The two // are named by the same id, and this directory is the only handle a // later sweep has for finding that cgroup again, so removing it here // would strand the cgroup for good with whatever is still in it. The - // same signal already earns the next job a sweep, which reclaims both + // same failure already earned the next job a sweep, which reclaims both // in the right order. - if self.reclaim_failed.is_set() { + // + // This job's cgroup, and nothing else: the runner-wide reclaim signal is + // also raised by a stale jail some sweep could not reclaim, which is no + // reason to keep a chroot whose own cgroup came down cleanly. + if self.signals.must_keep_chroot() { eprintln!( "Warning: leaving jail {} in place because its cgroup could not be removed. The directory names that cgroup, so the next job sweeps both.", self.dir @@ -95,7 +92,7 @@ impl Drop for JailDir { "Warning: failed to remove jail {}: {e}. It holds a VMM binary and a full guest rootfs; the next job will sweep it.", self.dir ); - self.reclaim_failed.set(); + self.signals.chroot_survived(); } } } @@ -153,7 +150,7 @@ mod tests { fn create_builds_a_private_chroot_tree() { let (_dir, state) = state_in_tmpdir(); - let jail = JailDir::create(&state, &vm_id(), ReclaimFailed::default()).unwrap(); + let jail = JailDir::create(&state, &vm_id(), JailSignals::unwatched()).unwrap(); assert_eq!(jail.root(), state.jail_root(&vm_id())); assert!(jail.root().is_dir()); @@ -168,7 +165,7 @@ mod tests { let (_dir, state) = state_in_tmpdir(); fs::create_dir_all(state.jail_root(&vm_id())).unwrap(); - JailDir::create(&state, &vm_id(), ReclaimFailed::default()).unwrap(); + JailDir::create(&state, &vm_id(), JailSignals::unwatched()).unwrap(); } #[test] @@ -176,7 +173,7 @@ mod tests { let (_dir, state) = state_in_tmpdir(); { - let jail = JailDir::create(&state, &vm_id(), ReclaimFailed::default()).unwrap(); + let jail = JailDir::create(&state, &vm_id(), JailSignals::unwatched()).unwrap(); fs::write(jail.root().join("rootfs.ext4"), b"guest").unwrap(); fs::create_dir_all(jail.root().join("dev")).unwrap(); } @@ -197,7 +194,7 @@ mod tests { // impossible to create. fs::write(state.jail_dir(&vm_id()), b"in the way").unwrap(); - JailDir::create(&state, &vm_id(), ReclaimFailed::default()).unwrap_err(); + JailDir::create(&state, &vm_id(), JailSignals::unwatched()).unwrap_err(); } #[test] @@ -206,11 +203,11 @@ mod tests { // raises this. Removing the chroot anyway would leave nothing for a // later sweep to find the cgroup by. let (_dir, state) = state_in_tmpdir(); - let reclaim_failed = ReclaimFailed::default(); - let jail = JailDir::create(&state, &vm_id(), reclaim_failed.clone()).unwrap(); + let signals = JailSignals::unwatched(); + let jail = JailDir::create(&state, &vm_id(), signals.clone()).unwrap(); fs::write(jail.root().join("rootfs.ext4"), b"guest").unwrap(); - reclaim_failed.set(); + signals.cgroup_survived(); drop(jail); assert!( @@ -222,7 +219,7 @@ mod tests { #[test] fn drop_tolerates_an_already_removed_tree() { let (_dir, state) = state_in_tmpdir(); - let jail = JailDir::create(&state, &vm_id(), ReclaimFailed::default()).unwrap(); + let jail = JailDir::create(&state, &vm_id(), JailSignals::unwatched()).unwrap(); fs::remove_dir_all(state.jail_dir(&vm_id())).unwrap(); drop(jail); } diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 3d4887618..168580c8e 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -253,18 +253,6 @@ pub struct HostPreparation { pub struct ReclaimFailed(Arc); impl ReclaimFailed { - /// A signal nothing reads, for a cgroup no sweep can find again. - /// - /// A non-sandboxed run has no chroot, and its cgroup is not named by any - /// directory the sweep walks, so there is no handle for a later sweep to - /// work from and nothing a raised signal could change. Named rather than - /// defaulted so the call site says which of the two it is. - #[cfg(target_os = "linux")] - #[must_use] - pub fn unwatched() -> Self { - Self::default() - } - /// Record that a jail could not be reclaimed. pub fn set(&self) { self.0.store(true, Ordering::SeqCst); @@ -296,6 +284,104 @@ impl ReclaimFailed { } } +/// Whether this job's own cgroup outlived its teardown. +/// +/// Job-scoped, where [`ReclaimFailed`] is runner-scoped, and that distinction is +/// the point. The chroot has to be kept when *this* job's cgroup could not be +/// removed, because the directory name is the only handle a later sweep has for +/// finding that cgroup. A runner-wide signal cannot say that: it is also raised +/// by an unrelated stale jail the sweep could not reclaim, and reading it made +/// the current job hold a chroot whose own cgroup came down cleanly. Two +/// questions on one flag is also what made the earlier ordering bugs easy to +/// write. +#[derive(Debug, Clone, Default)] +pub struct CgroupSurvived(Arc); + +impl CgroupSurvived { + /// Record that this job's cgroup could not be removed. + pub fn set(&self) { + self.0.store(true, Ordering::SeqCst); + } + + /// Whether this job's cgroup is still there. + /// + /// Only the chroot teardown reads it, and the jail is Linux-only. + #[cfg(target_os = "linux")] + fn is_set(&self) -> bool { + self.0.load(Ordering::SeqCst) + } +} + +/// The signals one job's teardown raises. +/// +/// Carried together because every teardown step needs both: a cgroup that will +/// not go away has to hold this job's chroot *and* earn the next job a sweep. +/// Keeping them in one value with two names is what stops the two being confused +/// for each other again. +#[derive(Debug, Clone)] +pub struct JailSignals { + /// Runner-wide: a later job owes another sweep. + /// + /// Only the jail raises and reads these, and the jail is Linux-only. + #[cfg_attr( + not(target_os = "linux"), + expect(dead_code, reason = "the jail is Linux-only") + )] + reclaim_failed: ReclaimFailed, + /// This job only: its cgroup outlived its teardown. + #[cfg_attr( + not(target_os = "linux"), + expect(dead_code, reason = "the jail is Linux-only") + )] + cgroup_survived: CgroupSurvived, +} + +impl JailSignals { + /// The signals for one job, sharing the runner's reclaim signal. + #[must_use] + pub fn for_job(reclaim_failed: ReclaimFailed) -> Self { + Self { + reclaim_failed, + cgroup_survived: CgroupSurvived::default(), + } + } + + /// Signals nothing reads, for a cgroup no sweep can find again. + /// + /// A non-sandboxed run has no chroot to hold and no directory the sweep + /// walks, so there is no handle for a later sweep to work from and nothing a + /// raised signal could change. Named rather than defaulted so the call site + /// says which of the two it is. + #[cfg(target_os = "linux")] + #[must_use] + pub fn unwatched() -> Self { + Self::for_job(ReclaimFailed::default()) + } + + /// Record that this job's cgroup could not be removed. + /// + /// Both signals, because both are true: the chroot that names this cgroup + /// has to stay, and the next job has to sweep for it. + #[cfg(target_os = "linux")] + pub fn cgroup_survived(&self) { + self.cgroup_survived.set(); + self.reclaim_failed.set(); + } + + /// Record that this job's chroot could not be removed. + #[cfg(target_os = "linux")] + pub fn chroot_survived(&self) { + self.reclaim_failed.set(); + } + + /// Whether this job's cgroup is still there, so its chroot must stay. + #[cfg(target_os = "linux")] + #[must_use] + pub(crate) fn must_keep_chroot(&self) -> bool { + self.cgroup_survived.is_set() + } +} + impl HostPreparation { /// A runner process that has not prepared the host yet. #[must_use] diff --git a/plus/bencher_runner/src/local_isolation.rs b/plus/bencher_runner/src/local_isolation.rs index 753806a0e..b7f356a54 100644 --- a/plus/bencher_runner/src/local_isolation.rs +++ b/plus/bencher_runner/src/local_isolation.rs @@ -61,8 +61,8 @@ impl LocalIsolation { // No chroot names this cgroup, and no sweep walks it, so a teardown it // cannot finish has nothing to hand the work to. - let reclaim_failed = crate::jail::ReclaimFailed::unwatched(); - let cgroup = match crate::jail::CgroupManager::new(&run_id, reclaim_failed) { + let signals = crate::jail::JailSignals::unwatched(); + let cgroup = match crate::jail::CgroupManager::new(&run_id, signals) { Ok(cgroup) => { // Best effort here, unlike the sandboxed path: a local run // makes no confinement claim to begin with, so losing the diff --git a/plus/bencher_runner/src/vm.rs b/plus/bencher_runner/src/vm.rs index 1d6002e9f..127849e57 100644 --- a/plus/bencher_runner/src/vm.rs +++ b/plus/bencher_runner/src/vm.rs @@ -9,7 +9,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use crate::error::RunnerError; use crate::jail::{ - HostPreparation, JailDir, JailLock, JailPaths, ReclaimFailed, StateDir, VmId, chroot, netns, + HostPreparation, JailDir, JailLock, JailPaths, JailSignals, StateDir, VmId, chroot, netns, state, }; use crate::run::{RunOutput, prepare_oci_workspace}; @@ -85,7 +85,11 @@ pub fn vm_execute( // before any of them exist. Dropping this guard removes the chroot tree, // which is what the workspace temp directory used to cover. let vm_id = VmId::new(); - let jail_dir = JailDir::create(&state_dir, &vm_id, host.reclaim_signal())?; + // Minted per job, beside the id: the cgroup and the chroot of one job share + // these, and a stale jail some other sweep could not reclaim is not this + // job's business. + let signals = JailSignals::for_job(host.reclaim_signal()); + let jail_dir = JailDir::create(&state_dir, &vm_id, signals.clone())?; let jail = JailPaths::new(jail_dir.root())?; println!(" Jail: {}", jail.root()); @@ -121,15 +125,8 @@ pub fn vm_execute( chroot::grant_jail_read(kernel_dest)?; // Step 7-8: Build Firecracker config and run the microVM - let fc_config = build_firecracker_config( - config, - work_dir, - vm_id, - &state_dir, - jail, - netns, - host.reclaim_signal(), - )?; + let fc_config = + build_firecracker_config(config, work_dir, vm_id, &state_dir, jail, netns, signals)?; let run_output = run_firecracker(&fc_config, cancel_flag)?; @@ -144,7 +141,7 @@ fn build_firecracker_config( state_dir: &StateDir, jail: JailPaths, netns: Utf8PathBuf, - reclaim_failed: ReclaimFailed, + signals: JailSignals, ) -> Result { // The jailer copies `--exec-file` into the chroot itself and rejects a // multiply linked file, so Firecracker is staged outside the jail and is @@ -191,7 +188,7 @@ fn build_firecracker_config( jail_user: config.jail_user, chroot_base_dir: state_dir.chroot_base(), netns, - reclaim_failed, + signals, vcpus, memory_mib, boot_args: config.kernel_cmdline.clone(), From b3df643c294d11c5593536ad748752547350f044 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 06:54:34 +0000 Subject: [PATCH 67/91] Extend the table to what a run measures The table covered teardown because that is where the defects had been found. Then one turned up in the cpuset verification, which the table's own rule already forbade: a read that fails must never be reported as a state that was observed. The rule was never specific to teardown, so the scope was the hole, not the rule. The header now says it governs both, and the second table lists every read that decides what a run measured: the cpuset apply and verify, the effective node set, the controller list, the partition levels, the CPU layout detection, the metrics, and the preflight checks. Measurement gets the same three columns as teardown with different consequences: fail the job, declare the absence loudly, or be genuinely irrelevant. It closes with what neither half may do, which is answer a question it could not ask. The intra-doc links to `cfg`-gated items are plain code spans now. They resolved only on Linux, so the table I added last round generated unresolved-link warnings on any other host. --- plus/bencher_runner/src/jail/mod.rs | 103 ++++++++++++++++++---------- 1 file changed, 67 insertions(+), 36 deletions(-) diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 168580c8e..1325851cc 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -6,26 +6,37 @@ //! network namespace the VMM joins, and the cgroup that both places it on the //! benchmark cores and bounds its resources. //! -//! # What a failing teardown step does +//! # What a failing step does //! -//! Reclamation is where this module is easiest to get wrong, because every step -//! of it runs after the thing it is cleaning up already happened, so a failure -//! has no obvious caller to tell. Three separate defects here were the same -//! mistake: a fallible step whose failure never reached the mechanism built to -//! retry it. So every fallible step does exactly one of three things, and adding -//! a step means choosing which, rather than reaching for `eprintln!`: +//! Teardown is where this module is easiest to get wrong, because every step of +//! it runs after the thing it is cleaning up already happened, so a failure has +//! no obvious caller to tell. Measurement is easiest to get wrong for the +//! mirror-image reason: a read that fails still leaves a value in hand, and a +//! plausible one. Several separate defects here were the same mistake in those +//! two dresses, a fallible step whose failure either never reached the mechanism +//! built to retry it or was reported as a state that had been observed. So every +//! fallible step does exactly one of three things, and adding a step means +//! choosing which, rather than reaching for `eprintln!` or a fallback value: //! //! - **Fails the job.** The host cannot be trusted to measure. Used wherever -//! something may still be running on the benchmark cores, including wherever -//! the runner could not establish that nothing is. Recoverable by -//! construction: nothing latches, so the next job tries the whole thing again. -//! - **Arms the retry.** [`ReclaimFailed`], read by -//! [`HostPreparation::ensure`]. Used where the cost is disk rather than a -//! contended benchmark, and where the code has no caller to report to at all, -//! which is every step reached from a `Drop`. +//! something may still be running on the benchmark cores, wherever the runner +//! could not establish that nothing is, and wherever a confinement it applied +//! could not be read back. Recoverable by construction: nothing latches, so +//! the next job tries the whole thing again. +//! - **Arms the retry, or declares the absence.** For teardown that is +//! [`ReclaimFailed`], read by [`HostPreparation::ensure`], used where the cost +//! is disk rather than a contended benchmark and where the code has no caller +//! to report to at all, which is every step reached from a `Drop`. For +//! measurement it is `Cpuset::Unavailable` or an absent metric: isolation the +//! host cannot offer is reported as isolation this run did not have, loudly, +//! and never as a number. //! - **Ignored.** Only where the failure is itself the answer, or where a later //! step is guaranteed to catch it. Each one below says which. //! +//! What no step does is answer a question it could not ask. An unreadable file is +//! not an empty one, an unstattable path is not an absent one, and a field that +//! was never read is not zero. +//! //! | Step | On failure | //! |---|---| //! | [`HostPreparation::ensure`]: the root check | fails the job | @@ -34,28 +45,48 @@ //! | [`HostPreparation::ensure`]: taking the jail lock | fails the job | //! | [`HostPreparation::ensure`]: a sweep that returns an error | fails the job | //! | [`HostPreparation::ensure`]: a sweep that leaves a chroot behind | arms the retry | -//! | [`state::sweep_jails`]: the jail parent is absent | nothing to sweep | -//! | [`state::sweep_jails`]: the jail parent cannot be read | fails the job | -//! | [`state::sweep_jails`]: an entry cannot be read | fails the job | -//! | [`state::sweep_jails`]: an entry's kind cannot be read | fails the job: it may be a jail | -//! | [`state::sweep_jails`]: a name that is not UTF-8 | ignored: every name here is a UUID this runner minted, so it is not ours | -//! | [`state::sweep_jails`]: the reap reports a live VMM | fails the job | -//! | [`state::sweep_jails`]: the reap could not examine the jail | fails the job | -//! | [`state::sweep_jails`]: removing the cgroup | fails the job, and the chroot is kept because its name is the cgroup's only handle | -//! | [`state::sweep_jails`]: removing the chroot | arms the retry | -//! | [`reap::reap_jailed_vmm`]: the jail root is absent | clear: nothing can be chrooted into a directory that is not there | -//! | [`reap::reap_jailed_vmm`]: the jail root cannot be stat'ed | reported unexaminable, which fails the job | -//! | [`reap::reap_jailed_vmm`]: `/proc` cannot be listed | reported unexaminable | -//! | [`reap::reap_jailed_vmm`]: a `/proc//root` cannot be read | ignored: that process is gone or is not this jail's | -//! | [`reap::reap_jailed_vmm`]: pinning or signalling the VMM | reported still running, which fails the job | -//! | [`reap::reap_jailed_vmm`]: a VMM that will not exit | reported still running | -//! | [`reap::reap_jailed_vmm`]: the rescan bound runs out | reported still running | -//! | [`reap::reap_jailed_vmm`]: a `/proc//status` that cannot be read | ignored: the process is gone, which is what was being asked | -//! | [`JailDir`] teardown: the chroot is already gone | ignored: that is the goal state | -//! | [`JailDir`] teardown: removing the chroot | arms the retry | -//! | [`JailDir`] teardown: the retry is already armed | keeps the chroot, since its name is the cgroup's only handle | -//! | [`CgroupManager`] teardown: `rmdir` of the cgroup | arms the retry | -//! | [`CgroupManager`] teardown: killing the cgroup's survivors | ignored: whatever survives is what makes the `rmdir` above fail, which arms the retry | +//! | `sweep_jails`: the jail parent is absent | nothing to sweep | +//! | `sweep_jails`: the jail parent cannot be read | fails the job | +//! | `sweep_jails`: an entry cannot be read | fails the job | +//! | `sweep_jails`: an entry's kind cannot be read | fails the job: it may be a jail | +//! | `sweep_jails`: a name that is not UTF-8 | ignored: every name here is a UUID this runner minted, so it is not ours | +//! | `sweep_jails`: the reap reports a live VMM | fails the job | +//! | `sweep_jails`: the reap could not examine the jail | fails the job | +//! | `sweep_jails`: removing the cgroup | fails the job, and the chroot is kept because its name is the cgroup's only handle | +//! | `sweep_jails`: removing the chroot | arms the retry | +//! | `reap_jailed_vmm`: the jail root is absent | clear: nothing can be chrooted into a directory that is not there | +//! | `reap_jailed_vmm`: the jail root cannot be stat'ed | reported unexaminable, which fails the job | +//! | `reap_jailed_vmm`: `/proc` cannot be listed | reported unexaminable | +//! | `reap_jailed_vmm`: a `/proc//root` cannot be read | ignored: that process is gone or is not this jail's | +//! | `reap_jailed_vmm`: pinning or signalling the VMM | reported still running, which fails the job | +//! | `reap_jailed_vmm`: a VMM that will not exit | reported still running | +//! | `reap_jailed_vmm`: the rescan bound runs out | reported still running | +//! | `reap_jailed_vmm`: a `/proc//status` that cannot be read | ignored: the process is gone, which is what was being asked | +//! | `JailDir` teardown: the chroot is already gone | ignored: that is the goal state | +//! | `JailDir` teardown: removing the chroot | arms the retry | +//! | `JailDir` teardown: the retry is already armed | keeps the chroot, since its name is the cgroup's only handle | +//! | `CgroupManager` teardown: `rmdir` of the cgroup | arms the retry, this job's and the runner's both | +//! | `CgroupManager` teardown: killing the cgroup's survivors | ignored: whatever survives is what makes the `rmdir` above fail, which arms the retry | +//! +//! And the same three columns for the reads that decide what a run measured: +//! +//! | Step | On failure | +//! |---|---| +//! | `apply_cpuset`: no isolation in the layout, or an empty core set | declares the absence | +//! | `apply_cpuset`: `cpuset.cpus` is absent | declares the absence: the controller is not delegated | +//! | `apply_cpuset`: `cpuset.cpus` cannot be stat'ed | fails the job: an error is not an absence | +//! | `apply_cpuset`: the kernel rejects a cpuset write | fails the job: half-applied confinement | +//! | `apply_cpuset`: the parent's node set is absent | node 0, which is what an undelegated controller means | +//! | `apply_cpuset`: the parent's node set cannot be read | fails the job: this value is written, so a guess confines the guest's memory | +//! | `apply_cpuset`: the effective set cannot be read back | fails the job: the read back *is* the mechanism, and the write already proved the controller delegated | +//! | `apply_cpuset`: the kernel narrowed the set | fails the job | +//! | `enable_controllers`: `cgroup.subtree_control` cannot be read | fails the job: an unreadable list is not an empty one | +//! | `BencherPartition::apply`: any read or write in the partition path | declares the absence: the level achieved is reported, down to `member` | +//! | `CpuLayout::detect`: the online CPU list cannot be read or parsed | declares the absence: the counted layout is announced as a guess | +//! | `CpuLayout::detect`: the core count cannot be read | one core, which reports as a layout with no isolation | +//! | `metrics`: a cgroup that is not there | no metrics, reported as absent | +//! | `metrics`: a field that cannot be read or parsed | absent, never zero | +//! | `tuning::preflight`: any check that cannot be performed | ignored: advisory only, and nothing reads it to decide whether the host can measure. A quiet preflight is not evidence of a quiet host | #[cfg(target_os = "linux")] mod cgroup; From 1b4658604df5c218b8185da518e3816c49770f62 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 06:54:34 +0000 Subject: [PATCH 68/91] Announce the network namespace lock's wait The jail lock tries once without blocking so it can say who it is waiting for; the namespace lock blocked silently. That handle is global to the host, so two runners with unrelated state directories contend on it, and the runner that lost looked hung rather than queued. Both locks now share the non-blocking attempt and print the same line. --- plus/bencher_runner/src/jail/lock.rs | 10 +++++++++- plus/bencher_runner/src/jail/netns.rs | 18 +++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/plus/bencher_runner/src/jail/lock.rs b/plus/bencher_runner/src/jail/lock.rs index 2ab72a2ad..cdb87b525 100644 --- a/plus/bencher_runner/src/jail/lock.rs +++ b/plus/bencher_runner/src/jail/lock.rs @@ -69,7 +69,7 @@ impl JailLock { // Try once without blocking, so waiting can be announced rather than // looking like a hang. - if flock(&file, libc::LOCK_EX | libc::LOCK_NB).is_ok() { + if flock_nonblocking(&file).is_ok() { return Ok(Self { _file: file }); } println!(" Waiting for another bencher runner to release {path}..."); @@ -88,6 +88,14 @@ pub(super) fn flock_exclusive(file: &File) -> std::io::Result<()> { flock(file, libc::LOCK_EX) } +/// Try for an exclusive `flock` without waiting. +/// +/// Shared with the network namespace lock so both locks announce a wait the same +/// way: an unexplained pause is the worst thing either of them can do. +pub(super) fn flock_nonblocking(file: &File) -> std::io::Result<()> { + flock(file, libc::LOCK_EX | libc::LOCK_NB) +} + /// Apply `flock` to a file, retrying if a signal interrupts the wait. fn flock(file: &File, operation: libc::c_int) -> std::io::Result<()> { loop { diff --git a/plus/bencher_runner/src/jail/netns.rs b/plus/bencher_runner/src/jail/netns.rs index 539210f1b..f9aa27343 100644 --- a/plus/bencher_runner/src/jail/netns.rs +++ b/plus/bencher_runner/src/jail/netns.rs @@ -14,7 +14,7 @@ use nix::mount::{MntFlags, MsFlags, mount, umount2}; use nix::sched::{CloneFlags, unshare}; use crate::error::JailError; -use crate::jail::lock::flock_exclusive; +use crate::jail::lock::{flock_exclusive, flock_nonblocking}; /// Directory holding named network namespace handles. /// @@ -160,6 +160,14 @@ struct NetnsLock { impl NetnsLock { /// Take the lock, waiting for whichever runner holds it. + /// + /// Announces the wait, as the jail lock does. This handle is global to the + /// host, so two runners with unrelated state directories contend here, and a + /// runner that sat silently on it would look hung rather than queued. + #[expect( + clippy::print_stdout, + reason = "prints why the runner is waiting, as the jail lock does" + )] fn acquire() -> Result { let path = Utf8Path::new(NETNS_LOCK_PATH); let file = fs::OpenOptions::new() @@ -171,6 +179,14 @@ impl NetnsLock { path: path.to_owned(), source: e, })?; + + // Try once without blocking, so the wait can be announced rather than + // looking like a hang. + if flock_nonblocking(&file).is_ok() { + return Ok(Self { _file: file }); + } + println!(" Waiting for another bencher runner to release {path}..."); + flock_exclusive(&file).map_err(|e| JailError::NetnsLock { path: path.to_owned(), source: e, From a02a1c2fa3e186ffcac7d217308d7ca93eed50bb Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 06:54:34 +0000 Subject: [PATCH 69/91] Keep the API harness out of the real state directory The elevated daemon started with no `--state-dir`, so it prepared, chmodded, and swept `/var/lib/bencher-runner` as root, then abandoned it root-owned. On a developer's machine that is a real directory a real runner may own, holding real jails, and the harness swept it for them. The scenarios harness went to some trouble to avoid exactly this, and the two are consistent now: a state directory beside the runner binary, inside the target tree the harness already owns, and removed on the way out with the sudo that created it. What cannot be removed is named along with the command that finishes the job. --- tasks/test_api/src/task/plus/runner.rs | 52 +++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/tasks/test_api/src/task/plus/runner.rs b/tasks/test_api/src/task/plus/runner.rs index 0d9fd112a..8f0e357ca 100644 --- a/tasks/test_api/src/task/plus/runner.rs +++ b/tasks/test_api/src/task/plus/runner.rs @@ -195,6 +195,15 @@ impl RunnerTest { let runner_cmd = Command::cargo_bin("runner")?; let runner_bin = runner_cmd.get_program().to_owned(); + // Never the default state directory. This runner is root, and + // preparing `/var/lib/bencher-runner` chmods it to 0700, sweeps + // every jail in it, and leaves it root-owned on the machine of + // whoever ran this harness, which on a developer's box is a real + // directory a real runner may own. The scenarios harness keeps its + // state under the target directory for the same reason. + let state_dir = elevated_state_dir(&runner_bin); + println!(" Runner state directory: {}", state_dir.display()); + let mut runner_child = Command::new("sudo"); let runner_child = runner_child .args(["-n", "--"]) @@ -208,6 +217,8 @@ impl RunnerTest { "--runner", "test-runner", ]) + .arg("--state-dir") + .arg(&state_dir) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::inherit()); // Its own process group, so teardown can signal the runner even @@ -223,7 +234,7 @@ impl RunnerTest { "runner", std::time::Duration::from_secs(30), ); - Some((runner_child, reader_handle)) + Some((runner_child, reader_handle, state_dir)) } else { println!("Skipping Firecracker runner daemon (no KVM)"); None @@ -312,9 +323,10 @@ impl RunnerTest { }; // Always kill runner daemons, even if the test failed - if let Some((mut runner_child, reader_handle)) = runner_child_and_handle { + if let Some((mut runner_child, reader_handle, state_dir)) = runner_child_and_handle { kill_elevated_runner(&mut runner_child); let _join = reader_handle.join(); + remove_elevated_state_dir(&state_dir); } let _kill = no_sandbox_child.kill(); let _wait = no_sandbox_child.wait(); @@ -353,6 +365,42 @@ fn ensure_passwordless_sudo() -> anyhow::Result<()> { Ok(()) } +/// The state directory the elevated runner is pointed at. +/// +/// Beside the runner binary, so it lands in the target directory the harness +/// already owns and is thrown away with it. +fn elevated_state_dir(runner_bin: &std::ffi::OsStr) -> std::path::PathBuf { + std::path::Path::new(runner_bin) + .parent() + .unwrap_or(std::path::Path::new(".")) + .join("test-api-runner-state") +} + +/// Remove the state directory the elevated runner left behind. +/// +/// Root-owned, because the runner that created it was, so the unprivileged +/// harness cannot remove it itself. Passwordless sudo was already established +/// before the daemon started. A failure here is not a test failure, but it is +/// said out loud with the command that finishes the job, because what is left is +/// a root-owned directory inside the developer's own tree. +fn remove_elevated_state_dir(state_dir: &std::path::Path) { + if !state_dir.exists() { + return; + } + let removed = Command::new("sudo") + .args(["-n", "rm", "-rf"]) + .arg(state_dir) + .status() + .is_ok_and(|status| status.success()); + if !removed { + println!( + "Note: {} is left owned by root. Remove it with: sudo rm -rf {}", + state_dir.display(), + state_dir.display() + ); + } +} + /// Stop the elevated runner daemon and anything it spawned. /// /// The runner runs as root, so the unprivileged test process cannot signal it, From f599006f553cc391ae304bdb714567193867ca11 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 07:51:11 +0000 Subject: [PATCH 70/91] Put the absolute state directory rule on the type The check lived in the argument parser, so it protected the one caller that went through the CLI and nothing else. `bencher_runner` is a library, and the reason the rule exists holds for every caller: the path reaches the jailer as `--chroot-base-dir`, which the jailer resolves against its own working directory rather than the runner's, so a relative one has every path the state directory hands out naming a different file for the jailer than for the runner. `StateDir` carries the invariant now, so a handle to a relative root cannot be constructed. The parser keeps its check and calls the same function rather than restating the rule, so there is one implementation and the operator still hears about it at the command line instead of when the first sandboxed Job builds its jail. --- plus/bencher_runner/src/error.rs | 5 ++ plus/bencher_runner/src/jail/chroot.rs | 2 +- plus/bencher_runner/src/jail/mod.rs | 24 +++++++++- plus/bencher_runner/src/jail/state.rs | 65 +++++++++++++++++--------- plus/bencher_runner/src/lib.rs | 5 +- plus/bencher_runner/src/vm.rs | 2 +- services/runner/src/parser/mod.rs | 19 +++----- 7 files changed, 85 insertions(+), 37 deletions(-) diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index 4665939c6..a8cb6f418 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -64,6 +64,11 @@ pub enum JailError { )] PrivilegedJailUser { field: &'static str }, + #[error( + "The state directory {path} must be an absolute path. It reaches the jailer as --chroot-base-dir, which the jailer resolves against its own working directory rather than the runner's, so a relative path builds the chroot somewhere the runner does not look." + )] + RelativeStateDir { path: Utf8PathBuf }, + #[error( "The state directory {path} already exists, is not empty, and was not created by the runner. Point --state-dir at a directory the runner owns, or at a subdirectory of this one." )] diff --git a/plus/bencher_runner/src/jail/chroot.rs b/plus/bencher_runner/src/jail/chroot.rs index 058f6c8bc..87b15ddbb 100644 --- a/plus/bencher_runner/src/jail/chroot.rs +++ b/plus/bencher_runner/src/jail/chroot.rs @@ -141,7 +141,7 @@ mod tests { fn state_in_tmpdir() -> (tempfile::TempDir, StateDir) { let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); - let state = StateDir::new(root.join("state")); + let state = StateDir::new(root.join("state")).unwrap(); state.create().unwrap(); (dir, state) } diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 1325851cc..bbda17c95 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -209,6 +209,28 @@ impl Default for JailUser { } } +/// Require an absolute state directory. +/// +/// The invariant belongs to the library, not to one command line. The path +/// reaches the jailer as `--chroot-base-dir`, which the jailer resolves against +/// its own working directory rather than the runner's, so a relative value +/// builds the chroot somewhere the runner does not look: the sweep never reaches +/// it and the lock does not protect it. Every caller that hands this crate a +/// state directory is exposed to that, and the CLI is only one of them. +/// +/// Shared with the argument parser rather than restated there, so the rule has +/// one implementation and the operator still hears about it before the runner +/// starts. +pub fn check_absolute_state_dir(path: &camino::Utf8Path) -> Result<(), crate::error::JailError> { + if path.is_absolute() { + Ok(()) + } else { + Err(crate::error::JailError::RelativeStateDir { + path: path.to_owned(), + }) + } +} + /// The identity of one microVM. /// /// The same string is the jailer's `--id`, the name of the chroot directory, @@ -522,7 +544,7 @@ fn prepare_host( // `unshare`, neither of which mentions root or the flag that avoids it. check_root(euid)?; - let state = StateDir::new(state_dir.to_owned()); + let state = StateDir::new(state_dir.to_owned())?; state.create()?; warn_on_named_account(jail_user); diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index e32f4fc47..d15cbc460 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -40,9 +40,15 @@ pub struct StateDir { impl StateDir { /// Create a handle for the state directory rooted at `root`. - #[must_use] - pub fn new(root: Utf8PathBuf) -> Self { - Self { root } + /// + /// Absolute, checked here so the type carries the invariant: a relative root + /// reaches the jailer as a `--chroot-base-dir` it resolves against its own + /// working directory, and every path this type hands out would then name a + /// different file for the runner than for the jailer. See + /// [`crate::jail::check_absolute_state_dir`]. + pub fn new(root: Utf8PathBuf) -> Result { + crate::jail::check_absolute_state_dir(&root)?; + Ok(Self { root }) } /// The state directory itself. @@ -69,6 +75,7 @@ impl StateDir { /// /// What the filesystem itself put there is not somebody else's data. See /// [`BENIGN_ENTRIES`]. + /// fn check_root_is_ours(&self) -> Result<(), JailError> { let entries = match fs::read_dir(&self.root) { Ok(entries) => entries, @@ -447,7 +454,7 @@ mod tests { #[test] fn jail_layout_matches_jailer_template() { - let state = StateDir::new(Utf8PathBuf::from("/var/lib/bencher-runner")); + let state = StateDir::new(Utf8PathBuf::from("/var/lib/bencher-runner")).unwrap(); assert_eq!(state.chroot_base(), "/var/lib/bencher-runner/jail"); assert_eq!( state.jail_parent(), @@ -471,7 +478,7 @@ mod tests { #[test] fn create_is_idempotent_and_private() { let (_dir, root) = temp_root(); - let state = StateDir::new(root.join("state")); + let state = StateDir::new(root.join("state")).unwrap(); state.create().unwrap(); state.create().unwrap(); @@ -485,7 +492,7 @@ mod tests { #[test] fn create_tightens_a_lax_directory() { let (_dir, root) = temp_root(); - let state = StateDir::new(root.join("state")); + let state = StateDir::new(root.join("state")).unwrap(); fs::create_dir_all(state.path()).unwrap(); fs::set_permissions(state.path(), fs::Permissions::from_mode(0o755)).unwrap(); @@ -504,7 +511,10 @@ mod tests { fs::create_dir_all(foreign.join("dpkg")).unwrap(); fs::create_dir_all(foreign.join("systemd")).unwrap(); - StateDir::new(foreign.clone()).create().unwrap_err(); + StateDir::new(foreign.clone()) + .unwrap() + .create() + .unwrap_err(); let mode = fs::metadata(&foreign).unwrap().permissions().mode(); assert_ne!(mode & 0o777, 0o700, "a refused root must not be chmodded"); @@ -520,7 +530,7 @@ mod tests { let not_a_dir = root.join("state"); fs::write(¬_a_dir, b"operator note").unwrap(); - let err = StateDir::new(not_a_dir).create().unwrap_err(); + let err = StateDir::new(not_a_dir).unwrap().create().unwrap_err(); assert!( matches!(err, JailError::ReadStateDir { .. }), @@ -538,7 +548,7 @@ mod tests { let volume = root.join("volume"); fs::create_dir_all(volume.join("lost+found")).unwrap(); - StateDir::new(volume.clone()).create().unwrap(); + StateDir::new(volume.clone()).unwrap().create().unwrap(); let mode = fs::metadata(&volume).unwrap().permissions().mode(); assert_eq!(mode & 0o777, 0o700); @@ -554,7 +564,10 @@ mod tests { fs::create_dir_all(foreign.join("lost+found")).unwrap(); fs::create_dir_all(foreign.join("dpkg")).unwrap(); - StateDir::new(foreign.clone()).create().unwrap_err(); + StateDir::new(foreign.clone()) + .unwrap() + .create() + .unwrap_err(); let mode = fs::metadata(&foreign).unwrap().permissions().mode(); assert_ne!(mode & 0o777, 0o700, "a refused root must not be chmodded"); @@ -566,13 +579,13 @@ mod tests { let empty = root.join("empty"); fs::create_dir_all(&empty).unwrap(); - StateDir::new(empty).create().unwrap(); + StateDir::new(empty).unwrap().create().unwrap(); } #[test] fn a_directory_the_runner_already_used_is_ours() { let (_dir, root) = temp_root(); - let state = StateDir::new(root.join("state")); + let state = StateDir::new(root.join("state")).unwrap(); state.create().unwrap(); // Something the host put there afterwards does not disown it. fs::write(state.path().join("notes.txt"), b"operator note").unwrap(); @@ -587,13 +600,23 @@ mod tests { fs::create_dir_all(&state).unwrap(); fs::write(state.join(".lock"), b"").unwrap(); - StateDir::new(state).create().unwrap(); + StateDir::new(state).unwrap().create().unwrap(); + } + + #[test] + fn a_relative_state_directory_is_refused() { + // The invariant lives on the type: the jailer resolves the path it is + // handed against its own working directory, so a relative one names a + // different place for the jailer than for the runner. + StateDir::new(Utf8PathBuf::from("bencher-runner")).unwrap_err(); + StateDir::new(Utf8PathBuf::from("./bencher-runner")).unwrap_err(); + StateDir::new(Utf8PathBuf::from("/var/lib/bencher-runner")).unwrap(); } #[test] fn sweep_removes_stale_jails() { let (_dir, root) = temp_root(); - let state = StateDir::new(root.join("state")); + let state = StateDir::new(root.join("state")).unwrap(); state.create().unwrap(); // Two stale jails, one with a nested chroot tree. @@ -629,7 +652,7 @@ mod tests { #[test] fn sweep_leaves_unrelated_entries_alone() { let (_dir, root) = temp_root(); - let state = StateDir::new(root.join("state")); + let state = StateDir::new(root.join("state")).unwrap(); state.create().unwrap(); let note = state.jail_parent().join("NOTES.txt"); @@ -655,7 +678,7 @@ mod tests { // Removing the tree would not stop the VMM, and it would destroy the // only handle for identifying that process on a later sweep. let (_dir, root) = temp_root(); - let state = StateDir::new(root.join("state")); + let state = StateDir::new(root.join("state")).unwrap(); state.create().unwrap(); let live = VmId::from_chroot_name("live".to_owned()); let dead = VmId::from_chroot_name("dead".to_owned()); @@ -694,7 +717,7 @@ mod tests { // job, not once. Nothing latches, so the sweep is re-attempted and // reports again. let (_dir, root) = temp_root(); - let state = StateDir::new(root.join("state")); + let state = StateDir::new(root.join("state")).unwrap(); state.create().unwrap(); let live = VmId::from_chroot_name("live".to_owned()); fs::create_dir_all(state.jail_root(&live)).unwrap(); @@ -717,7 +740,7 @@ mod tests { // that cgroup for good: nothing ever sees the id again. One stuck // cgroup must still not abandon the rest of the sweep. let (_dir, root) = temp_root(); - let state = StateDir::new(root.join("state")); + let state = StateDir::new(root.join("state")).unwrap(); state.create().unwrap(); let stuck = VmId::from_chroot_name("stuck".to_owned()); let clear = VmId::from_chroot_name("clear".to_owned()); @@ -759,7 +782,7 @@ mod tests { // Warning alone would have the caller spend the reclaim signal and the // leak would outlive every later job. let (_dir, root) = temp_root(); - let state = StateDir::new(root.join("state")); + let state = StateDir::new(root.join("state")).unwrap(); state.create().unwrap(); let stuck = VmId::from_chroot_name("stuck".to_owned()); fs::create_dir_all(state.jail_root(&stuck)).unwrap(); @@ -791,7 +814,7 @@ mod tests { // tree on that would be the same destructive step as removing it under a // VMM known to be alive, so it gets the same answer. let (_dir, root) = temp_root(); - let state = StateDir::new(root.join("state")); + let state = StateDir::new(root.join("state")).unwrap(); state.create().unwrap(); let unknown = VmId::from_chroot_name("unknown".to_owned()); fs::create_dir_all(state.jail_root(&unknown)).unwrap(); @@ -816,7 +839,7 @@ mod tests { #[test] fn a_cleared_jail_is_still_swept() { let (_dir, root) = temp_root(); - let state = StateDir::new(root.join("state")); + let state = StateDir::new(root.join("state")).unwrap(); state.create().unwrap(); fs::create_dir_all(state.jail_root(&VmId::from_chroot_name("one".to_owned()))).unwrap(); diff --git a/plus/bencher_runner/src/lib.rs b/plus/bencher_runner/src/lib.rs index 5fd12296e..ad502659c 100644 --- a/plus/bencher_runner/src/lib.rs +++ b/plus/bencher_runner/src/lib.rs @@ -62,7 +62,10 @@ pub use config::Config; #[cfg(feature = "plus")] pub use error::{ConfigError, ExecutionError, JailError, RunnerError}; #[cfg(feature = "plus")] -pub use jail::{DEFAULT_JAIL_GID, DEFAULT_JAIL_UID, DEFAULT_STATE_DIR, HostPreparation, JailUser}; +pub use jail::{ + DEFAULT_JAIL_GID, DEFAULT_JAIL_UID, DEFAULT_STATE_DIR, HostPreparation, JailUser, + check_absolute_state_dir, +}; #[cfg(feature = "plus")] pub use log_level::SandboxLogLevel; #[cfg(feature = "plus")] diff --git a/plus/bencher_runner/src/vm.rs b/plus/bencher_runner/src/vm.rs index 127849e57..65179340a 100644 --- a/plus/bencher_runner/src/vm.rs +++ b/plus/bencher_runner/src/vm.rs @@ -32,7 +32,7 @@ pub fn vm_execute( println!(" Memory: {} MiB", config.memory.to_mib()); println!(" Timeout: {} seconds", config.timeout_secs); - let state_dir = StateDir::new(config.state_dir.clone()); + let state_dir = StateDir::new(config.state_dir.clone())?; // Prepare the host on demand, before the first jail this process builds. // Must come before the job lock is taken: preparation takes the same lock, diff --git a/services/runner/src/parser/mod.rs b/services/runner/src/parser/mod.rs index bdf8642e8..e2ed8af09 100644 --- a/services/runner/src/parser/mod.rs +++ b/services/runner/src/parser/mod.rs @@ -148,22 +148,17 @@ pub struct CliRun { pub sandbox_log_level: bencher_runner::SandboxLogLevel, } -/// Require an absolute state directory. +/// Require an absolute state directory, before the runner starts. /// -/// The path reaches the jailer as `--chroot-base-dir`, which the jailer resolves -/// against its own working directory rather than the runner's, so a relative -/// value builds the chroot somewhere the runner does not look and the sweep -/// never reaches. +/// The rule itself lives in the library, on the type that holds the path, since +/// every caller that hands it a state directory is exposed to the same thing. +/// This is the same check run early, so an operator hears about it at the +/// command line rather than when the first sandboxed Job builds its jail. #[cfg(feature = "plus")] fn absolute_state_dir(arg: &str) -> Result { let path = Utf8PathBuf::from(arg); - if path.is_absolute() { - Ok(path) - } else { - Err(format!( - "the state directory must be an absolute path, and `{path}` is relative" - )) - } + bencher_runner::check_absolute_state_dir(&path).map_err(|e| e.to_string())?; + Ok(path) } #[cfg(all(test, feature = "plus"))] From 4f0501b699349deed63e7efc8d8d1e1cfe9391c5 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 07:51:11 +0000 Subject: [PATCH 71/91] Prove the state directory is ours by its tree, not by a name The guard that keeps the runner from chmodding a directory it does not own returned the moment it saw an entry called `jail` or `.lock`, before looking at anything else. `/var/lib` on a host running this runner has a directory called `jail`, and so does anything anyone happened to name that way, so a populated system directory passed the guard and was tightened to 0700. That is the `--state-dir /var/lib` hazard the guard was written for, defeated by the shallowest possible check. Ownership is proven by `jail/firecracker` now, a path only this runner builds, and it builds the whole tree in one step so a shallower half never stands for the whole. The lock file name goes back to being private to the lock, since proving ownership by a name is exactly what stopped being something to do. --- plus/bencher_runner/src/jail/lock.rs | 11 ++--- plus/bencher_runner/src/jail/state.rs | 64 +++++++++++++++++++++------ 2 files changed, 56 insertions(+), 19 deletions(-) diff --git a/plus/bencher_runner/src/jail/lock.rs b/plus/bencher_runner/src/jail/lock.rs index cdb87b525..cd3b7fc82 100644 --- a/plus/bencher_runner/src/jail/lock.rs +++ b/plus/bencher_runner/src/jail/lock.rs @@ -27,11 +27,12 @@ use crate::error::JailError; /// only removes directories under `/jail/firecracker`) can never /// reach it. /// -/// Defined once and shared with the state directory, which counts it among the -/// entries that mark a directory as the runner's own. Two spellings of the same -/// name would leave a directory holding only this file unrecognized, and the -/// runner would refuse a state directory it created itself. -pub(super) const LOCK_FILE: &str = ".lock"; +/// Private again, and only spelled here. The state directory used to count this +/// name among the marks of a directory it owns, until proving ownership by a +/// name turned out to be the way a populated system directory could pass that +/// guard. Ownership is proven by the chroot tree now, so nothing outside this +/// module needs the name. +const LOCK_FILE: &str = ".lock"; /// Holds the jail lock for as long as it is alive. /// diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs index d15cbc460..9e0d34996 100644 --- a/plus/bencher_runner/src/jail/state.rs +++ b/plus/bencher_runner/src/jail/state.rs @@ -13,7 +13,6 @@ use camino::{Utf8Path, Utf8PathBuf}; use crate::error::JailError; use crate::jail::VmId; -use crate::jail::lock::LOCK_FILE; use crate::jail::reap::Reaped; /// Subdirectory of the state directory used as the jailer's chroot base. @@ -76,7 +75,26 @@ impl StateDir { /// What the filesystem itself put there is not somebody else's data. See /// [`BENIGN_ENTRIES`]. /// + /// Ownership is proven by the tree, never by a name. An entry called `jail` + /// proves nothing: `/var/lib` on a host running this runner has one, and so + /// does any directory somebody happened to name that way, and matching on + /// the name alone let a populated system directory pass this guard and be + /// chmodded 0700, which is the whole hazard it exists for. + /// `jail/firecracker` is a path only this runner builds, and it builds the + /// tree in one step, so there is no window where a shallower half stands for + /// the whole. fn check_root_is_ours(&self) -> Result<(), JailError> { + match self.jail_parent().try_exists() { + Ok(true) => return Ok(()), + Ok(false) => {}, + Err(e) => { + return Err(JailError::ReadStateDir { + path: self.jail_parent(), + source: e, + }); + }, + } + let entries = match fs::read_dir(&self.root) { Ok(entries) => entries, // Missing: creating it is the next step. @@ -95,9 +113,6 @@ impl StateDir { source: e, })?; let name = entry.file_name(); - if RUNNER_ENTRIES.iter().any(|ours| name == *ours) { - return Ok(()); - } if BENIGN_ENTRIES.iter().any(|benign| name == *benign) { continue; } @@ -162,12 +177,6 @@ impl StateDir { } } -/// Entries the runner creates directly in its state directory. -/// -/// Their presence is what distinguishes a directory the runner has used from -/// one that belongs to the host. -const RUNNER_ENTRIES: [&str; 2] = [CHROOT_BASE, LOCK_FILE]; - /// Entries that do not make a directory somebody else's. /// /// A dedicated filesystem is the natural home for the chroots, since each holds @@ -594,13 +603,40 @@ mod tests { } #[test] - fn a_directory_holding_only_the_lock_is_ours() { + fn a_directory_named_like_ours_is_not_ours() { + // The hazard this guard exists for, and what a name match let through: + // `/var/lib` on a host running this runner holds a directory called + // `jail`, and so does anything anyone happened to name that way. Only + // the tree the runner builds proves the directory is the runner's. + let (_dir, root) = temp_root(); + let foreign = root.join("var-lib"); + fs::create_dir_all(foreign.join("jail")).unwrap(); + fs::create_dir_all(foreign.join("dpkg")).unwrap(); + fs::write(foreign.join(".lock"), b"").unwrap(); + + StateDir::new(foreign.clone()) + .unwrap() + .create() + .unwrap_err(); + + let mode = fs::metadata(&foreign).unwrap().permissions().mode(); + assert_ne!(mode & 0o777, 0o700, "a refused root must not be chmodded"); + } + + #[test] + fn the_tree_is_what_proves_the_directory_is_ours() { + // A state directory the runner has used carries `jail/firecracker`, + // which it creates in one step, so a shallower half never stands for the + // whole. Anything the operator put there afterwards does not disown it. let (_dir, root) = temp_root(); let state = root.join("state"); - fs::create_dir_all(&state).unwrap(); - fs::write(state.join(".lock"), b"").unwrap(); + fs::create_dir_all(state.join("jail").join("firecracker")).unwrap(); + fs::write(state.join("notes.txt"), b"operator note").unwrap(); + + StateDir::new(state.clone()).unwrap().create().unwrap(); - StateDir::new(state).unwrap().create().unwrap(); + let mode = fs::metadata(&state).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o700); } #[test] From 201df939cffce3698af508afdc173024667cc9e3 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 07:51:11 +0000 Subject: [PATCH 72/91] Stop a failed stat from authorizing destruction The rule the audit produced covered reads that produce a value. A read that gates an action breaks it just as badly and looks nothing like a measurement: `exists` returns false for an error as well as for absence, so a stat that failed authorizes whatever the false branch does. A grep for the idiom found four in this file, three of which defeat a guard written for the exact outcome they allow. `remove_stale_cgroup` read a stat error as "already gone" and returned `Ok`, on which the sweep deletes the chroot that names the cgroup. That strands the cgroup permanently and destroys the only handle any later sweep had for finding it, which is precisely what the sweep's ordering exists to prevent. The retry loop had the same stat with the same meaning, so only a stat that succeeded and said absent counts now. `CgroupManager::new` read a stat error as "not there" and set the created flag, which has `Drop` remove a cgroup somebody else owns. `cleanup` read one as "gone already" and skipped both the removal and the signal, so nothing was armed and nothing would have come back for it. The fourth was harmless and is gone anyway: an `exists` gating an idempotent `create_dir_all` gated nothing the create does not gate itself. The unused cgroup v2 detector goes with it rather than sit there as an example of the pattern, and the cpuset write path no longer treats a missing file as an undelegated controller, which is what `verify_cpuset` concludes one step later about the same question. --- plus/bencher_runner/src/jail/cgroup.rs | 127 ++++++++++++++----------- 1 file changed, 74 insertions(+), 53 deletions(-) diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index 3fe358164..fdf8a2623 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -38,14 +38,16 @@ impl CgroupManager { .join(BENCHER_CGROUP_BASE) .join(vm_id.as_str()); - // Ensure parent bencher cgroup exists + // Unconditional, because `create_dir_all` on a directory that is + // already there is a success. The `exists` check this replaces was a + // read that could only mislead: an error from it would have been read as + // an absent parent, and it gated nothing the create does not gate + // itself. let parent = Utf8PathBuf::from(CGROUP_ROOT).join(BENCHER_CGROUP_BASE); - if !parent.exists() { - fs::create_dir_all(&parent).map_err(|e| JailError::CreateCgroup { - path: parent.clone(), - source: e, - })?; - } + fs::create_dir_all(&parent).map_err(|e| JailError::CreateCgroup { + path: parent.clone(), + source: e, + })?; // Enable controllers in the parent. Always attempted (idempotent): // the parent may have been created without controllers, e.g. by @@ -57,14 +59,26 @@ impl CgroupManager { // a cgroup that was already there would have it rmdir something // belonging to whoever did create it. Fresh ids make that unlikely, // but this branch exists precisely for when the id is not fresh. - let created = if cgroup_path.exists() { - false - } else { - fs::create_dir_all(&cgroup_path).map_err(|e| JailError::CreateCgroup { - path: cgroup_path.clone(), - source: e, - })?; - true + // `try_exists`, because this decides whether `Drop` may remove the + // directory. An error read as "not there" would have this claim a cgroup + // somebody else owns and then delete it on the way out, which is the one + // outcome the branch exists to prevent. + let created = match cgroup_path.try_exists() { + Ok(true) => false, + Ok(false) => { + fs::create_dir_all(&cgroup_path).map_err(|e| JailError::CreateCgroup { + path: cgroup_path.clone(), + source: e, + })?; + true + }, + Err(e) => { + return Err(JailError::ReadCgroup { + path: cgroup_path, + source: e, + } + .into()); + }, }; Ok(Self { @@ -175,8 +189,13 @@ impl CgroupManager { Ok(false) => return Ok(Cpuset::Unavailable(UNDELEGATED)), Err(e) => return Err(JailError::ReadCgroup { path, source: e }.into()), } + // Every failure, absence included. Delegation was settled by the check + // above, so a `cpuset.cpus` that has gone missing between that stat and + // this write is a cgroup disappearing underneath the runner, not a host + // that never had the controller. `verify_cpuset` reasons the same way + // about the same question one step later; the two used to disagree. if let Err(e) = fs::write(&path, &cpuset) { - return classify_cpuset_error(path, e); + return Err(JailError::WriteCgroup { path, source: e }.into()); } // Also need to set cpuset.mems for cpuset to work. Use the parent's @@ -191,7 +210,11 @@ impl CgroupManager { }; let mems_path = self.cgroup_path.join("cpuset.mems"); if let Err(e) = fs::write(&mems_path, &mems) { - return classify_cpuset_error(mems_path, e); + return Err(JailError::WriteCgroup { + path: mems_path, + source: e, + } + .into()); } self.verify_cpuset(&cpuset, &mems) @@ -322,16 +345,32 @@ impl CgroupManager { /// here would be a channel with nothing in it that every caller, `Drop` /// included, would have to discard. pub fn cleanup(&mut self) { - if self.created && self.cgroup_path.exists() { - if let Err(e) = fs::remove_dir(&self.cgroup_path) { + if !self.created { + return; + } + // A stat that failed is not a cgroup that is gone. Reading it as one + // would skip both the removal and the signal, so nothing would be armed + // and nothing would ever come back for it. + match self.cgroup_path.try_exists() { + Ok(false) => self.created = false, + Ok(true) => { + if let Err(e) = fs::remove_dir(&self.cgroup_path) { + eprintln!( + "Warning: failed to remove cgroup {}: {e}. Something is still in it, so the next job sweeps it along with the jail that names it.", + self.cgroup_path + ); + self.signals.cgroup_survived(); + } else { + self.created = false; + } + }, + Err(e) => { eprintln!( - "Warning: failed to remove cgroup {}: {e}. Something is still in it, so the next job sweeps it along with the jail that names it.", + "Warning: cannot tell whether cgroup {} is still there: {e}. It is treated as still there, so the next job sweeps it along with the jail that names it.", self.cgroup_path ); self.signals.cgroup_survived(); - } else { - self.created = false; - } + }, } } } @@ -386,24 +425,6 @@ pub enum Cpuset { Unavailable(&'static str), } -/// Decide whether a failed cpuset write is an absent controller or a refusal. -/// -/// A file that is not there is the controller not being delegated, which is a -/// limitation. Anything else is the kernel refusing a cpuset it does -/// understand, which would leave the cgroup claiming an isolation it does not -/// have. -fn classify_cpuset_error(path: Utf8PathBuf, error: std::io::Error) -> Result { - if error.kind() == std::io::ErrorKind::NotFound { - Ok(Cpuset::Unavailable(UNDELEGATED)) - } else { - Err(JailError::WriteCgroup { - path, - source: error, - } - .into()) - } -} - /// Parse a kernel cpu list (`0-3,5,7-9`) into the set of cpus it names. /// /// Compared as sets rather than as strings, because the kernel is free to @@ -450,8 +471,15 @@ pub(crate) fn remove_stale_cgroup(vm_id: &VmId) -> Result<(), JailError> { let path = Utf8PathBuf::from(CGROUP_ROOT) .join(BENCHER_CGROUP_BASE) .join(vm_id.as_str()); - if !path.exists() { - return Ok(()); + // The caller deletes the chroot that names this cgroup once this returns + // `Ok`, and that chroot is the only handle any later sweep has for finding + // the cgroup again. A stat error read as "already gone" would stranded the + // cgroup permanently and delete the one thing that could have found it, + // which is exactly what the caller's ordering exists to prevent. + match path.try_exists() { + Ok(false) => return Ok(()), + Ok(true) => {}, + Err(e) => return Err(JailError::StaleCgroup { path, source: e }), } let deadline = std::time::Instant::now() + REMOVE_TIMEOUT; @@ -462,7 +490,9 @@ pub(crate) fn remove_stale_cgroup(vm_id: &VmId) -> Result<(), JailError> { return Ok(()); }, // Someone else got there first, which is the outcome either way. - Err(_) if !path.exists() => return Ok(()), + // Only a stat that succeeded and said absent counts: anything else + // falls through to the retry and, in the end, to the error. + Err(_) if path.try_exists().is_ok_and(|exists| !exists) => return Ok(()), Err(e) if std::time::Instant::now() >= deadline => { // Reported rather than warned, because failing here means the // next job sweeps again rather than inheriting a host nobody @@ -493,15 +523,6 @@ fn missing_required_controller(enabled: &str) -> Option<&'static str> { .find(|required| !enabled.split_whitespace().any(|token| token == *required)) } -/// Check if cgroup v2 is available. -#[expect(dead_code, reason = "utility for future cgroup v2 feature detection")] -#[must_use] -pub fn is_cgroup_v2_available() -> bool { - Utf8Path::new(CGROUP_ROOT) - .join("cgroup.controllers") - .exists() -} - #[cfg(test)] mod tests { use camino::Utf8PathBuf; From 7eeccb2daa682d373a93bd80ef40ca3dfb639c97 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 07:51:33 +0000 Subject: [PATCH 73/91] Catch a jailed process that exits during the last sleep The readiness loop polls the child between sleeps but never checked after the loop, so a process that exited during the final fifty milliseconds was reported as a socket that never became ready. That points an operator at Firecracker taking too long when the truth is that it is gone, which is the confusion this error was added to remove. One more poll before giving up. Renamed with it. The jailer `exec`s Firecracker in place, so the pid the runner holds is the jailer up to that moment and the VMM afterwards, and from outside there is no telling which of the two died. Naming the jailer was right about half the time; the error says a jailed process exited and names both possibilities. --- plus/bencher_runner/src/firecracker/error.rs | 13 +++++++++---- plus/bencher_runner/src/firecracker/process.rs | 11 ++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/error.rs b/plus/bencher_runner/src/firecracker/error.rs index d3d236003..c9c6bf41d 100644 --- a/plus/bencher_runner/src/firecracker/error.rs +++ b/plus/bencher_runner/src/firecracker/error.rs @@ -52,15 +52,20 @@ pub enum FirecrackerError { #[error("Firecracker API socket not ready after {0:?}")] SocketNotReady(std::time::Duration), - /// The jailer exited before Firecracker started serving its API. + /// The jailed process exited before Firecracker started serving its API. + /// + /// Named for what the runner actually knows. The jailer `exec`s Firecracker + /// in place, so this one pid is the jailer up to that moment and Firecracker + /// afterwards, and the runner cannot tell from the outside which of the two + /// died. Blaming the jailer by name was right about half the time. /// /// Distinct from a timeout: the process is gone, so waiting cannot help, /// and the reason is on stderr under the `[firecracker]` prefix. #[error( - "The jailer exited ({status}) before the Firecracker API socket appeared; its diagnostics are above, prefixed [firecracker]" + "The jailed process exited ({status}) before the Firecracker API socket appeared. The jailer execs Firecracker in place, so this is the jailer or the VMM it became; its diagnostics are above, prefixed [firecracker]" )] - JailerExited { - /// How the jailer exited. + JailedProcessExited { + /// How it exited. status: std::process::ExitStatus, }, diff --git a/plus/bencher_runner/src/firecracker/process.rs b/plus/bencher_runner/src/firecracker/process.rs index e628d500e..dcf01101d 100644 --- a/plus/bencher_runner/src/firecracker/process.rs +++ b/plus/bencher_runner/src/firecracker/process.rs @@ -181,11 +181,20 @@ impl FirecrackerProcess { return Ok(()); } if let Ok(Some(status)) = self.child.try_wait() { - return Err(FirecrackerError::JailerExited { status }); + return Err(FirecrackerError::JailedProcessExited { status }); } std::thread::sleep(poll_interval); } + // Once more before giving up. The loop sleeps between polls, so a + // process that exits during the last sleep would otherwise be reported + // as a socket that never became ready, which points at Firecracker + // taking too long when the truth is that it is gone. That confusion is + // the entire reason this error exists. + if let Ok(Some(status)) = self.child.try_wait() { + return Err(FirecrackerError::JailedProcessExited { status }); + } + Err(FirecrackerError::SocketNotReady(timeout)) } From f2aec94370f089960ec817993ef3454410da41ff Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 07:51:33 +0000 Subject: [PATCH 74/91] Report a malformed HTTP response as malformed An unparseable status line became a 500, which attributes a server error to Firecracker that Firecracker never sent and sends whoever reads the log looking at the VMM. The response is malformed, the error for that already exists, and two tests asserted the invented status until the rule was written down. --- plus/bencher_runner/src/firecracker/client.rs | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/plus/bencher_runner/src/firecracker/client.rs b/plus/bencher_runner/src/firecracker/client.rs index 5e4660c59..19c214364 100644 --- a/plus/bencher_runner/src/firecracker/client.rs +++ b/plus/bencher_runner/src/firecracker/client.rs @@ -275,11 +275,16 @@ fn parse_http_response(data: &[u8]) -> Result<(u16, String), FirecrackerError> { .next() .ok_or(FirecrackerError::MalformedResponse("empty HTTP response"))?; + // No invented status. A status line the runner could not parse is a + // malformed response, and reporting 500 would attribute a server error to + // Firecracker that Firecracker never sent. let status_code: u16 = status_line .split_whitespace() .nth(1) - .and_then(|s| s.parse().ok()) - .unwrap_or(500); + .and_then(|code| code.parse().ok()) + .ok_or(FirecrackerError::MalformedResponse( + "HTTP status line carries no status code", + ))?; // Extract body (after \r\n\r\n) let body = response @@ -383,18 +388,31 @@ mod tests { } #[test] - fn parse_http_malformed_status_defaults_to_500() { - // No status code in the status line + fn a_status_line_without_a_status_is_malformed_not_a_500() { + // These two asserted a default of 500 until the rule was written down. + // A response the runner could not parse is a malformed response; calling + // it a 500 attributes a server error to Firecracker that Firecracker + // never sent, and sends whoever reads the log looking at the VMM. let data = b"HTTP/1.1\r\n\r\n"; - let (status, _) = parse_http_response(data).unwrap(); - assert_eq!(status, 500); + + let err = parse_http_response(data).unwrap_err(); + + assert!( + matches!(err, FirecrackerError::MalformedResponse(_)), + "a response that could not be parsed is not a status: {err}" + ); } #[test] - fn parse_http_non_numeric_status_defaults_to_500() { + fn a_status_that_is_not_a_number_is_malformed_not_a_500() { let data = b"HTTP/1.1 abc OK\r\n\r\n"; - let (status, _) = parse_http_response(data).unwrap(); - assert_eq!(status, 500); + + let err = parse_http_response(data).unwrap_err(); + + assert!( + matches!(err, FirecrackerError::MalformedResponse(_)), + "{err}" + ); } #[test] From 47eef34287769ae91e74f34c3d4aff9bde271573 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 07:51:34 +0000 Subject: [PATCH 75/91] Warn about the jail user once per runner, not once per job Host preparation runs again whenever a sweep is owed, and it carried this warning with it. A host that keeps failing to reclaim a jail would repeat the same advisory on every job, which is how an operator learns to skip warnings. The account it names cannot change under a running runner, so it is worth saying once. --- plus/bencher_runner/src/jail/mod.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index bbda17c95..4f83a032d 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -282,6 +282,14 @@ impl std::fmt::Display for VmId { /// by anything else. Tests get their own. #[derive(Debug, Default)] pub struct HostPreparation { + /// Whether the jail user warning has already been given. + /// + /// Advisory and unchanging, so it is worth saying once and not once a job. + #[cfg_attr( + not(target_os = "linux"), + expect(dead_code, reason = "host preparation is Linux-only") + )] + warned_jail_user: bool, /// Only the jail reads this, and the jail is Linux-only. #[cfg_attr( not(target_os = "linux"), @@ -492,7 +500,8 @@ impl HostPreparation { if self.prepared && !self.reclaim_failed.is_set() { return Ok(()); } - let swept = prepare_host(euid, state_dir, jail_user)?; + let swept = prepare_host(euid, state_dir, jail_user, !self.warned_jail_user)?; + self.warned_jail_user = true; // Spent only on a sweep that finished, and armed by one that did not. // Spending it any earlier disarms the mechanism precisely when it is // needed: the signal would be gone, this process would still count as @@ -538,6 +547,7 @@ fn prepare_host( euid: u32, state_dir: &camino::Utf8Path, jail_user: JailUser, + announce_jail_user: bool, ) -> Result { // Checked first, and by name. Without it the most likely upgrade failure // surfaces as a permission error on a directory, or a bare EPERM out of @@ -547,7 +557,13 @@ fn prepare_host( let state = StateDir::new(state_dir.to_owned())?; state.create()?; - warn_on_named_account(jail_user); + // Once per runner, not once per job. Preparation runs again whenever a + // sweep is owed, and a host that keeps failing to reclaim a jail would + // otherwise repeat this warning on every job until an operator stopped + // reading any of them. + if announce_jail_user { + warn_on_named_account(jail_user); + } let _lock = JailLock::acquire(state.path())?; let swept = state::sweep_jails(&state.jail_parent())?; From dfc253cf73a53e77024f620dc6f8c822a66f0b83 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 07:51:34 +0000 Subject: [PATCH 76/91] Say in the table that a read may gate an action The rule covered reads that produce a value, which is why a stat before a destructive step slipped past it three separate times: it does not look like a measurement at all, and its failure authorizes destruction rather than fabricating a number. The header says so now, and the rows the syntactic sweep turned up are in the tables. The row for teardown killing a cgroup's survivors is gone. Nothing on the jailed path calls `kill_all`: the VMM is killed with a grace period before its cgroup comes down, and whatever survives is caught by the `rmdir`, which arms the retry. The non-sandboxed path calls it explicitly on timeout. Describing a step that does not exist is worse than describing none, because this table is read as a specification, and adding the call to justify the row would be changing behavior to fit a document. --- plus/bencher_runner/src/jail/mod.rs | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 4f83a032d..0c0e09094 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -37,6 +37,16 @@ //! not an empty one, an unstattable path is not an absent one, and a field that //! was never read is not zero. //! +//! That covers reads which produce a value. It covers reads which gate an action +//! just as strictly, and those are the easier ones to miss: an `exists` before a +//! destructive step is a read whose failure authorizes destruction, and it does +//! not look like a measurement at all. A stat that fails before a `remove_dir` +//! is the same defect as a metric that fabricates a zero, and costs more. +//! +//! The rows below were found by grepping this crate for every idiom that can +//! turn a failure into an answer, rather than by reasoning about which functions +//! looked relevant. Reasoning missed instances three times running. +//! //! | Step | On failure | //! |---|---| //! | [`HostPreparation::ensure`]: the root check | fails the job | @@ -65,8 +75,11 @@ //! | `JailDir` teardown: the chroot is already gone | ignored: that is the goal state | //! | `JailDir` teardown: removing the chroot | arms the retry | //! | `JailDir` teardown: the retry is already armed | keeps the chroot, since its name is the cgroup's only handle | +//! | `CgroupManager` teardown: the cgroup cannot be stat'ed | arms the retry: it is treated as still there | //! | `CgroupManager` teardown: `rmdir` of the cgroup | arms the retry, this job's and the runner's both | -//! | `CgroupManager` teardown: killing the cgroup's survivors | ignored: whatever survives is what makes the `rmdir` above fail, which arms the retry | +//! | `CgroupManager` creation: the cgroup cannot be stat'ed | fails the job: this decides whether `Drop` may remove it | +//! | `remove_stale_cgroup`: the cgroup cannot be stat'ed | fails the job: the caller deletes the chroot on an `Ok` here | +//! | `StateDir::create`: the chroot tree cannot be stat'ed | fails the job: the 0700 chmod follows | //! //! And the same three columns for the reads that decide what a run measured: //! @@ -87,6 +100,18 @@ //! | `metrics`: a cgroup that is not there | no metrics, reported as absent | //! | `metrics`: a field that cannot be read or parsed | absent, never zero | //! | `tuning::preflight`: any check that cannot be performed | ignored: advisory only, and nothing reads it to decide whether the host can measure. A quiet preflight is not evidence of a quiet host | +//! | `pin_vcpu_threads`: a thread list that cannot be read | declares the absence: it reports how many of the vCPUs it pinned | +//! | `FirecrackerClient`: a status line that cannot be parsed | fails the request: no status is invented for a response Firecracker did not send | +//! | `find_binary`: a candidate path that cannot be stat'ed | ignored: the search is a list of guesses, and finding nothing is reported by name | +//! +//! `CgroupManager::kill_all` is deliberately not in either table. +//! Nothing on the jailed path calls it: the VMM is killed with a grace period +//! before its cgroup is torn down, and anything that somehow survives is caught +//! by the `rmdir` above, which arms the retry. The non-sandboxed path calls it +//! explicitly on timeout or cancellation, where a failure is warned and then +//! caught the same way. A row saying teardown kills survivors described a step +//! that does not exist, which is worse than no row at all: this table is read as +//! a specification. #[cfg(target_os = "linux")] mod cgroup; From c6807fbd10b4d7fa3eb5defa462c8af24c653ea8 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 07:51:51 +0000 Subject: [PATCH 77/91] Search for one binary in one place Three copies of the same search differed only in a name and a hint, which is three places for it to drift. One helper takes both. A candidate that cannot be stat'ed is still passed over rather than reported, which is the whole of the failure handling this needs: the list is guesses, and the one thing it can conclude, that nothing was found, is reported by name with what to do about it. --- plus/bencher_runner/src/vm.rs | 110 +++++++++++++--------------------- 1 file changed, 43 insertions(+), 67 deletions(-) diff --git a/plus/bencher_runner/src/vm.rs b/plus/bencher_runner/src/vm.rs index 65179340a..791d099ec 100644 --- a/plus/bencher_runner/src/vm.rs +++ b/plus/bencher_runner/src/vm.rs @@ -294,85 +294,61 @@ fn install_init_binary(rootfs: &Utf8Path) -> Result<(), RunnerError> { Ok(()) } -/// Find the bencher-init binary on disk (fallback when not bundled). -fn find_init_binary() -> Result { - // Look in these locations in order - let candidates = [ - // Next to the current executable +/// Where a bundled binary is looked for when it was not bundled in. +/// +/// Beside the runner first, so a self-contained install finds its own copy +/// before anything the host happens to have. +fn binary_candidates(name: &str) -> impl Iterator { + [ std::env::current_exe() .ok() - .and_then(|p| p.parent().map(|d| d.join("bencher-init"))) - .and_then(|p| Utf8PathBuf::try_from(p).ok()), - // Common installation paths - Some(Utf8PathBuf::from("/usr/local/bin/bencher-init")), - Some(Utf8PathBuf::from("/usr/bin/bencher-init")), - ]; + .and_then(|exe| exe.parent().map(|dir| dir.join(name))) + .and_then(|path| Utf8PathBuf::try_from(path).ok()), + Some(Utf8PathBuf::from(format!("/usr/local/bin/{name}"))), + Some(Utf8PathBuf::from(format!("/usr/bin/{name}"))), + ] + .into_iter() + .flatten() +} - for candidate in candidates.into_iter().flatten() { - if candidate.exists() { - return Ok(candidate); - } - } +/// Find a binary on disk, for when it was not bundled into the runner. +/// +/// One function for all three, because three copies of the same search differing +/// only in a name and a hint is three places for the search to drift. +/// +/// A candidate that cannot be stat'ed is passed over rather than reported, and +/// that is the whole of the failure handling this needs: the search is a list of +/// guesses, and the one thing it can conclude, that nothing was found, is +/// reported by name with what to do about it. +fn find_binary(name: &str, hint: &str) -> Result { + binary_candidates(name) + .find(|candidate| candidate.exists()) + .ok_or_else(|| { + crate::error::ConfigError::BinaryNotFound { + name: name.to_owned(), + hint: hint.to_owned(), + } + .into() + }) +} - Err(crate::error::ConfigError::BinaryNotFound { - name: "bencher-init".to_owned(), - hint: "Build with: cargo build -p bencher_init".to_owned(), - } - .into()) +/// Where to get Firecracker and its jailer, which ship together. +const FIRECRACKER_RELEASES: &str = + "Install from: https://github.com/firecracker-microvm/firecracker/releases"; + +/// Find the bencher-init binary on disk (fallback when not bundled). +fn find_init_binary() -> Result { + find_binary("bencher-init", "Build with: cargo build -p bencher_init") } /// Find the Firecracker binary on the system. fn find_firecracker_binary() -> Result { - let candidates = [ - // Next to the current executable - std::env::current_exe() - .ok() - .and_then(|p| p.parent().map(|d| d.join("firecracker"))) - .and_then(|p| Utf8PathBuf::try_from(p).ok()), - // Common installation paths - Some(Utf8PathBuf::from("/usr/local/bin/firecracker")), - Some(Utf8PathBuf::from("/usr/bin/firecracker")), - ]; - - for candidate in candidates.into_iter().flatten() { - if candidate.exists() { - return Ok(candidate); - } - } - - Err(crate::error::ConfigError::BinaryNotFound { - name: "firecracker".to_owned(), - hint: "Install from: https://github.com/firecracker-microvm/firecracker/releases" - .to_owned(), - } - .into()) + find_binary("firecracker", FIRECRACKER_RELEASES) } /// Find the jailer binary on the system (fallback when not bundled). fn find_jailer_binary() -> Result { - let candidates = [ - // Next to the current executable - std::env::current_exe() - .ok() - .and_then(|p| p.parent().map(|d| d.join("jailer"))) - .and_then(|p| Utf8PathBuf::try_from(p).ok()), - // Common installation paths - Some(Utf8PathBuf::from("/usr/local/bin/jailer")), - Some(Utf8PathBuf::from("/usr/bin/jailer")), - ]; - - for candidate in candidates.into_iter().flatten() { - if candidate.exists() { - return Ok(candidate); - } - } - - Err(crate::error::ConfigError::BinaryNotFound { - name: "jailer".to_owned(), - hint: "Install from: https://github.com/firecracker-microvm/firecracker/releases" - .to_owned(), - } - .into()) + find_binary("jailer", FIRECRACKER_RELEASES) } /// Find the kernel image on the system. From 042ef291054a035efc8983a0dd97d709baaf8546 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 07:51:51 +0000 Subject: [PATCH 78/91] Widen the syscall arguments explicitly `syscall` is variadic, so each argument is passed at the width it is written at rather than at any width the signature enforces. The integer literals and the descriptor were `i32` where the kernel takes a `long`. It works on the ABI this targets, which is exactly why it is worth writing down: the cast says the width is deliberate rather than inherited from a literal's default type. --- plus/bencher_runner/src/jail/reap.rs | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/plus/bencher_runner/src/jail/reap.rs b/plus/bencher_runner/src/jail/reap.rs index de1a70190..d4fe2e9f5 100644 --- a/plus/bencher_runner/src/jail/reap.rs +++ b/plus/bencher_runner/src/jail/reap.rs @@ -264,8 +264,17 @@ fn pidfd_open(pid: u32) -> std::io::Result> { )] // SAFETY: `pidfd_open` takes a pid and a flag word and touches no memory. // It returns a new descriptor or -1, and the descriptor is handed straight - // to `OwnedFd` so it is closed exactly once. - let raw = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) }; + // to `OwnedFd` so it is closed exactly once. Every argument is widened to + // `c_long` explicitly: `syscall` is variadic, so the width each value is + // passed at is the width written here rather than anything the signature + // enforces. + let raw = unsafe { + libc::syscall( + libc::SYS_pidfd_open, + libc::c_long::from(pid), + libc::c_long::from(0i32), + ) + }; if raw < 0 { let error = std::io::Error::last_os_error(); @@ -299,14 +308,16 @@ fn pidfd_kill(pidfd: &OwnedFd) -> std::io::Result<()> { )] // SAFETY: `pidfd` is an open, owned descriptor for the duration of the // call. A null `siginfo` pointer is the documented way to ask the kernel - // to synthesize one, and the final argument is a reserved flag word. + // to synthesize one, and the final argument is a reserved flag word. The + // integers are widened to `c_long` explicitly, since a variadic call passes + // each value at the width it is written at. let ret = unsafe { libc::syscall( libc::SYS_pidfd_send_signal, - pidfd.as_raw_fd(), - libc::SIGKILL, + libc::c_long::from(pidfd.as_raw_fd()), + libc::c_long::from(libc::SIGKILL), std::ptr::null::(), - 0, + libc::c_long::from(0i32), ) }; if ret == 0 { From 644b252ebdc4a506f99fdc43852783693cd27b05 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 08:31:01 +0000 Subject: [PATCH 79/91] Say what the state directory actually holds The guest root filesystem moved under `--state-dir` when the sandbox became a jail, and the documentation never said so. An operator sizing that directory had no way to learn it holds a full guest rootfs and a copy of the microVM binary for every Job running at once, which is the difference between a few megabytes and several gigabytes. It also names the mitigation that was already made to work. A dedicated filesystem or a `tmpfs` keeps that write traffic off the system disk, which is the answer to what the jail costs a measurement, and a freshly created filesystem is accepted despite the `lost+found` it arrives with, specifically so that mounting one is a supported thing to do. Nine locales, both subcommands. --- .../src/chunks/docs-reference/runner/de/runner-run.mdx | 5 +++++ .../src/chunks/docs-reference/runner/de/runner-up.mdx | 5 +++++ .../src/chunks/docs-reference/runner/en/runner-run.mdx | 5 +++++ .../src/chunks/docs-reference/runner/en/runner-up.mdx | 5 +++++ .../src/chunks/docs-reference/runner/es/runner-run.mdx | 5 +++++ .../src/chunks/docs-reference/runner/es/runner-up.mdx | 5 +++++ .../src/chunks/docs-reference/runner/fr/runner-run.mdx | 5 +++++ .../src/chunks/docs-reference/runner/fr/runner-up.mdx | 5 +++++ .../src/chunks/docs-reference/runner/ja/runner-run.mdx | 5 +++++ .../src/chunks/docs-reference/runner/ja/runner-up.mdx | 5 +++++ .../src/chunks/docs-reference/runner/ko/runner-run.mdx | 5 +++++ .../src/chunks/docs-reference/runner/ko/runner-up.mdx | 5 +++++ .../src/chunks/docs-reference/runner/pt/runner-run.mdx | 5 +++++ .../src/chunks/docs-reference/runner/pt/runner-up.mdx | 5 +++++ .../src/chunks/docs-reference/runner/ru/runner-run.mdx | 5 +++++ .../src/chunks/docs-reference/runner/ru/runner-up.mdx | 5 +++++ .../src/chunks/docs-reference/runner/zh/runner-run.mdx | 5 +++++ .../src/chunks/docs-reference/runner/zh/runner-up.mdx | 5 +++++ 18 files changed, 90 insertions(+) diff --git a/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx index fa25b0939..bfbccabee 100644 --- a/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx @@ -38,6 +38,11 @@ Das persistente Zustandsverzeichnis für den Runner. Es muss ein absoluter Pfad sein. Das Jail, das die Sandbox einschließt, wird unter diesem Verzeichnis erstellt, und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor dem ersten Sandbox-Job daraus entfernt. +Jedes Jail enthält ein vollständiges Gast-Root-Dateisystem und eine Kopie der microVM-Binärdatei, +daher braucht dieses Verzeichnis echte Kapazität für jeden gleichzeitig laufenden Job. +Ein eigenes Dateisystem oder ein `tmpfs`-Mount wird unterstützt und empfohlen: +es hält diese Schreiblast von der Systemplatte fern, +und ein neu erstelltes Dateisystem wird akzeptiert, auch wenn es ein `lost+found`-Verzeichnis mitbringt. Standardmäßig wird `/var/lib/bencher-runner` verwendet. Kann auch mit der Umgebungsvariable `BENCHER_STATE_DIR` gesetzt werden. diff --git a/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx index 0c69da179..94113edc3 100644 --- a/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx @@ -41,6 +41,11 @@ Das persistente Zustandsverzeichnis für den Runner. Es muss ein absoluter Pfad sein. Das Jail, das die Sandbox einschließt, wird unter diesem Verzeichnis erstellt, und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor dem ersten Sandbox-Job daraus entfernt. +Jedes Jail enthält ein vollständiges Gast-Root-Dateisystem und eine Kopie der microVM-Binärdatei, +daher braucht dieses Verzeichnis echte Kapazität für jeden gleichzeitig laufenden Job. +Ein eigenes Dateisystem oder ein `tmpfs`-Mount wird unterstützt und empfohlen: +es hält diese Schreiblast von der Systemplatte fern, +und ein neu erstelltes Dateisystem wird akzeptiert, auch wenn es ein `lost+found`-Verzeichnis mitbringt. Standardmäßig wird `/var/lib/bencher-runner` verwendet. Kann auch mit der Umgebungsvariable `BENCHER_STATE_DIR` gesetzt werden. diff --git a/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx index 30232dd14..4f186f7a5 100644 --- a/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx @@ -38,6 +38,11 @@ The persistent state directory for the Runner. It must be an absolute path. The jail that confines the sandbox is built under this directory, and jails left behind by an unclean exit are swept from it before the first sandboxed Job runs. +Each jail holds a full guest root filesystem and a copy of the microVM binary, +so this directory needs real capacity for every Job that runs at once. +A dedicated filesystem or a `tmpfs` mount is supported and recommended: +it keeps that write traffic off the system disk, +and a freshly created filesystem is accepted even though it arrives with a `lost+found` directory. By default, `/var/lib/bencher-runner` is used. Can also be set with the `BENCHER_STATE_DIR` environment variable. diff --git a/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx index 82759f086..603c98339 100644 --- a/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx @@ -41,6 +41,11 @@ The persistent state directory for the Runner. It must be an absolute path. The jail that confines the sandbox is built under this directory, and jails left behind by an unclean exit are swept from it before the first sandboxed Job runs. +Each jail holds a full guest root filesystem and a copy of the microVM binary, +so this directory needs real capacity for every Job that runs at once. +A dedicated filesystem or a `tmpfs` mount is supported and recommended: +it keeps that write traffic off the system disk, +and a freshly created filesystem is accepted even though it arrives with a `lost+found` directory. By default, `/var/lib/bencher-runner` is used. Can also be set with the `BENCHER_STATE_DIR` environment variable. diff --git a/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx index 36e9b2892..da8701683 100644 --- a/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx @@ -38,6 +38,11 @@ El directorio de estado persistente del Runner. Debe ser una ruta absoluta. La jaula que confina el sandbox se crea dentro de este directorio, y las jaulas que quedaron tras una salida no limpia se eliminan de él antes del primer Job con sandbox. +Cada jaula contiene un sistema de archivos raíz completo del huésped y una copia del binario de la microVM, +por lo que este directorio necesita capacidad real para cada Job que se ejecute a la vez. +Se admite y se recomienda un sistema de archivos dedicado o un montaje `tmpfs`: +mantiene esa carga de escritura fuera del disco del sistema, +y se acepta un sistema de archivos recién creado aunque llegue con un directorio `lost+found`. Por defecto, se usa `/var/lib/bencher-runner`. También se puede establecer con la variable de entorno `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx index f4de47880..3d7d7f465 100644 --- a/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx @@ -41,6 +41,11 @@ El directorio de estado persistente del Runner. Debe ser una ruta absoluta. La jaula que confina el sandbox se crea dentro de este directorio, y las jaulas que quedaron tras una salida no limpia se eliminan de él antes del primer Job con sandbox. +Cada jaula contiene un sistema de archivos raíz completo del huésped y una copia del binario de la microVM, +por lo que este directorio necesita capacidad real para cada Job que se ejecute a la vez. +Se admite y se recomienda un sistema de archivos dedicado o un montaje `tmpfs`: +mantiene esa carga de escritura fuera del disco del sistema, +y se acepta un sistema de archivos recién creado aunque llegue con un directorio `lost+found`. Por defecto, se usa `/var/lib/bencher-runner`. También se puede establecer con la variable de entorno `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx index 6e9ce85bf..2b694e520 100644 --- a/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx @@ -38,6 +38,11 @@ Le répertoire d'état persistant du Runner. Il doit s'agir d'un chemin absolu. La prison qui confine le bac à sable est créée dans ce répertoire, et les prisons laissées par un arrêt brutal en sont supprimées avant le premier Job avec bac à sable. +Chaque prison contient un système de fichiers racine invité complet et une copie du binaire de la microVM, +ce répertoire a donc besoin d'une capacité réelle pour chaque Job exécuté simultanément. +Un système de fichiers dédié ou un montage `tmpfs` est pris en charge et recommandé : +il maintient cette charge d'écriture hors du disque système, +et un système de fichiers fraîchement créé est accepté même s'il arrive avec un répertoire `lost+found`. Par défaut, `/var/lib/bencher-runner` est utilisé. Peut également être défini avec la variable d'environnement `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx index 81c17282f..f1b767018 100644 --- a/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx @@ -41,6 +41,11 @@ Le répertoire d'état persistant du Runner. Il doit s'agir d'un chemin absolu. La prison qui confine le bac à sable est créée dans ce répertoire, et les prisons laissées par un arrêt brutal en sont supprimées avant le premier Job avec bac à sable. +Chaque prison contient un système de fichiers racine invité complet et une copie du binaire de la microVM, +ce répertoire a donc besoin d'une capacité réelle pour chaque Job exécuté simultanément. +Un système de fichiers dédié ou un montage `tmpfs` est pris en charge et recommandé : +il maintient cette charge d'écriture hors du disque système, +et un système de fichiers fraîchement créé est accepté même s'il arrive avec un répertoire `lost+found`. Par défaut, `/var/lib/bencher-runner` est utilisé. Peut également être défini avec la variable d'environnement `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx index 7f132117f..13e363360 100644 --- a/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx @@ -38,6 +38,11 @@ Runner の永続的な状態ディレクトリ。 絶対パスである必要があります。 サンドボックスを閉じ込める jail はこのディレクトリの下に作成され、 異常終了で残った jail は、最初のサンドボックス Job の実行前にここから一掃されます。 +各 jail はゲストのルートファイルシステム全体と microVM バイナリのコピーを保持するため、 +このディレクトリには同時に実行される Job の分だけの実容量が必要です。 +専用のファイルシステムまたは `tmpfs` マウントがサポートされており、推奨されます。 +書き込み負荷をシステムディスクから切り離せます。 +作成したばかりのファイルシステムは `lost+found` ディレクトリを含んでいても受け付けられます。 デフォルトでは `/var/lib/bencher-runner` が使用されます。 環境変数 `BENCHER_STATE_DIR` でも設定できます。 diff --git a/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx index 1d1c666b1..66c80d616 100644 --- a/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx @@ -40,6 +40,11 @@ Runner の永続的な状態ディレクトリ。 絶対パスである必要があります。 サンドボックスを閉じ込める jail はこのディレクトリの下に作成され、 異常終了で残った jail は、最初のサンドボックス Job の実行前にここから一掃されます。 +各 jail はゲストのルートファイルシステム全体と microVM バイナリのコピーを保持するため、 +このディレクトリには同時に実行される Job の分だけの実容量が必要です。 +専用のファイルシステムまたは `tmpfs` マウントがサポートされており、推奨されます。 +書き込み負荷をシステムディスクから切り離せます。 +作成したばかりのファイルシステムは `lost+found` ディレクトリを含んでいても受け付けられます。 デフォルトでは `/var/lib/bencher-runner` が使用されます。 環境変数 `BENCHER_STATE_DIR` でも設定できます。 diff --git a/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx index 32776997c..dbe0e856d 100644 --- a/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx @@ -38,6 +38,11 @@ Runner의 영구 상태 디렉터리입니다. 절대 경로여야 합니다. 샌드박스를 가두는 jail이 이 디렉터리 아래에 생성되며, 비정상 종료로 남겨진 jail은 첫 번째 샌드박스 Job 실행 전에 이곳에서 정리됩니다. +각 jail은 게스트의 전체 루트 파일 시스템과 microVM 바이너리 사본을 보관하므로, +이 디렉터리에는 동시에 실행되는 Job 수만큼의 실제 용량이 필요합니다. +전용 파일 시스템이나 `tmpfs` 마운트가 지원되며 권장됩니다. +쓰기 부하를 시스템 디스크에서 분리할 수 있습니다. +새로 만든 파일 시스템은 `lost+found` 디렉터리가 있어도 허용됩니다. 기본적으로 `/var/lib/bencher-runner`가 사용됩니다. `BENCHER_STATE_DIR` 환경 변수로도 설정할 수 있습니다. diff --git a/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx index 839a2dc98..7a347ab40 100644 --- a/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx @@ -41,6 +41,11 @@ Runner의 영구 상태 디렉터리입니다. 절대 경로여야 합니다. 샌드박스를 가두는 jail이 이 디렉터리 아래에 생성되며, 비정상 종료로 남겨진 jail은 첫 번째 샌드박스 Job 실행 전에 이곳에서 정리됩니다. +각 jail은 게스트의 전체 루트 파일 시스템과 microVM 바이너리 사본을 보관하므로, +이 디렉터리에는 동시에 실행되는 Job 수만큼의 실제 용량이 필요합니다. +전용 파일 시스템이나 `tmpfs` 마운트가 지원되며 권장됩니다. +쓰기 부하를 시스템 디스크에서 분리할 수 있습니다. +새로 만든 파일 시스템은 `lost+found` 디렉터리가 있어도 허용됩니다. 기본적으로 `/var/lib/bencher-runner`가 사용됩니다. `BENCHER_STATE_DIR` 환경 변수로도 설정할 수 있습니다. diff --git a/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx index 9f7450f9a..4da00f0e3 100644 --- a/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx @@ -38,6 +38,11 @@ O diretório de estado persistente do Runner. Deve ser um caminho absoluto. A jail que confina o sandbox é criada sob este diretório, e as jails deixadas por um encerramento não limpo são removidas dele antes do primeiro Job com sandbox. +Cada jail contém um sistema de arquivos raiz completo do convidado e uma cópia do binário da microVM, +portanto este diretório precisa de capacidade real para cada Job executado ao mesmo tempo. +Um sistema de arquivos dedicado ou uma montagem `tmpfs` é suportado e recomendado: +mantém essa carga de escrita fora do disco do sistema, +e um sistema de arquivos recém-criado é aceito mesmo que venha com um diretório `lost+found`. Por padrão, `/var/lib/bencher-runner` é usado. Também pode ser definido com a variável de ambiente `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx index 7b99709dd..076a27b6d 100644 --- a/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx @@ -41,6 +41,11 @@ O diretório de estado persistente do Runner. Deve ser um caminho absoluto. A jail que confina o sandbox é criada sob este diretório, e as jails deixadas por um encerramento não limpo são removidas dele antes do primeiro Job com sandbox. +Cada jail contém um sistema de arquivos raiz completo do convidado e uma cópia do binário da microVM, +portanto este diretório precisa de capacidade real para cada Job executado ao mesmo tempo. +Um sistema de arquivos dedicado ou uma montagem `tmpfs` é suportado e recomendado: +mantém essa carga de escrita fora do disco do sistema, +e um sistema de arquivos recém-criado é aceito mesmo que venha com um diretório `lost+found`. Por padrão, `/var/lib/bencher-runner` é usado. Também pode ser definido com a variável de ambiente `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx index f20cf689e..3debccf53 100644 --- a/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx @@ -38,6 +38,11 @@ JWT-токен для аутентификации в реестре при за Путь должен быть абсолютным. Изолирующая среда (jail), ограничивающая песочницу, создаётся внутри этого каталога, а среды, оставшиеся после аварийного завершения, удаляются из него перед запуском первого Job с песочницей. +Каждая среда содержит полную корневую файловую систему гостя и копию двоичного файла microVM, +поэтому этому каталогу нужен реальный объём для каждого Job, работающего одновременно. +Выделенная файловая система или монтирование `tmpfs` поддерживается и рекомендуется: +это уводит нагрузку на запись с системного диска, +а только что созданная файловая система принимается, даже если в ней есть каталог `lost+found`. По умолчанию используется `/var/lib/bencher-runner`. Также может быть задан переменной окружения `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx index 8f3033422..251828ab5 100644 --- a/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx @@ -41,6 +41,11 @@ UUID или slug Runner, от имени которого работать. Путь должен быть абсолютным. Изолирующая среда (jail), ограничивающая песочницу, создаётся внутри этого каталога, а среды, оставшиеся после аварийного завершения, удаляются из него перед запуском первого Job с песочницей. +Каждая среда содержит полную корневую файловую систему гостя и копию двоичного файла microVM, +поэтому этому каталогу нужен реальный объём для каждого Job, работающего одновременно. +Выделенная файловая система или монтирование `tmpfs` поддерживается и рекомендуется: +это уводит нагрузку на запись с системного диска, +а только что созданная файловая система принимается, даже если в ней есть каталог `lost+found`. По умолчанию используется `/var/lib/bencher-runner`. Также может быть задан переменной окружения `BENCHER_STATE_DIR`. diff --git a/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx index fbd1058d2..9273af172 100644 --- a/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx @@ -38,6 +38,11 @@ Runner 的持久状态目录。 必须是绝对路径。 限制沙箱的 jail 在该目录下创建， 非正常退出遗留的 jail 会在第一个沙箱 Job 运行前从中清除。 +每个 jail 都保存着完整的客户机根文件系统和一份 microVM 二进制文件副本， +因此该目录需要为每个同时运行的 Job 准备真实的容量。 +支持并推荐使用专用文件系统或 `tmpfs` 挂载： +这能让写入负载远离系统磁盘， +并且刚创建的文件系统即使带有 `lost+found` 目录也会被接受。 默认使用 `/var/lib/bencher-runner`。 也可以通过 `BENCHER_STATE_DIR` 环境变量设置。 diff --git a/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx index 8e750c17f..08c232a68 100644 --- a/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx @@ -40,6 +40,11 @@ Runner 的持久状态目录。 必须是绝对路径。 限制沙箱的 jail 在该目录下创建， 非正常退出遗留的 jail 会在第一个沙箱 Job 运行前从中清除。 +每个 jail 都保存着完整的客户机根文件系统和一份 microVM 二进制文件副本， +因此该目录需要为每个同时运行的 Job 准备真实的容量。 +支持并推荐使用专用文件系统或 `tmpfs` 挂载： +这能让写入负载远离系统磁盘， +并且刚创建的文件系统即使带有 `lost+found` 目录也会被接受。 默认使用 `/var/lib/bencher-runner`。 也可以通过 `BENCHER_STATE_DIR` 环境变量设置。 From a37809589d709eb741431c5f16b3133395309a70 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 08:31:19 +0000 Subject: [PATCH 80/91] Write down why the metrics stat may stay The classification stands: this gating stat can only withhold a reading, never invent one. Every field is an option that is absent unless it was read, the cgroup block is omitted entirely when there is nothing to read, and the zero that used to stand in for an unparsed field is gone. There is no path here to a number that was not measured, which is what the rule is about. The reason is in the function now, so the next reader does not have to derive it from the type signatures. One thing was worth tightening, for a different reason than the rule. A stat that failed suppressed the reads entirely, throwing away metrics the files might still have given. Only a stat that succeeded and said absent means there is nothing to read now; anything else goes on to ask the files, which report what they can and nothing more. A test pins the property that makes the stat harmless. --- plus/bencher_runner/src/metrics.rs | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/plus/bencher_runner/src/metrics.rs b/plus/bencher_runner/src/metrics.rs index 461f4c9d3..3ff8a97d4 100644 --- a/plus/bencher_runner/src/metrics.rs +++ b/plus/bencher_runner/src/metrics.rs @@ -47,15 +47,23 @@ pub struct CgroupMetrics { /// Read cgroup metrics from the given cgroup path. /// /// Reads `cpu.stat` and `memory.peak` from the cgroup directory. -/// Returns `None` if the path doesn't exist. +/// Returns `None` when there is no cgroup to read. /// /// Every field is optional, and a field that could not be read stays absent /// rather than becoming a number. These are reported to an operator as measured /// values, so a zero standing in for a failed read is the plainest form of the /// one thing measurement is never allowed to do. Absence is already how this /// reports "no cgroup at all", so it costs nothing to be honest per field. +/// +/// That is also why the stat below is the one gating read in this crate that may +/// stay: it can only withhold a reading, never invent one. Nothing here can +/// answer with a number it did not read, so a failure reaches the operator as a +/// field that is not there. A stat that fails does not suppress the reads either, +/// which is the one thing worth tightening: only a stat that succeeded and said +/// absent means there is nothing to read, and anything else goes on to ask the +/// files themselves, since they report what they can and nothing more. pub fn read_cgroup_metrics(cgroup_path: &Utf8Path) -> Option { - if !cgroup_path.exists() { + if cgroup_path.try_exists().is_ok_and(|exists| !exists) { return None; } @@ -268,6 +276,22 @@ mod tests { assert!(read_cgroup_metrics(Utf8Path::new("/nonexistent")).is_none()); } + #[test] + fn a_cgroup_whose_files_cannot_be_read_reports_no_numbers() { + // The property that makes this module's gating stat harmless: every + // failure here withholds a field, and none of them invents one. An empty + // reading is honest; a zero would not be. + let dir = tempfile::tempdir().unwrap(); + let path = tempdir_utf8(&dir); + + let metrics = read_cgroup_metrics(path).unwrap(); + + assert_eq!(metrics.cpu_usage_us, None); + assert_eq!(metrics.cpu_user_us, None); + assert_eq!(metrics.cpu_system_us, None); + assert_eq!(metrics.memory_peak_bytes, None); + } + // --- format_metrics --- #[test] From ea75511fbc702aa8d5f8170fc370c61aad536ed2 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 08:31:19 +0000 Subject: [PATCH 81/91] Record the jail user warning when it is given The flag was set after preparation returned, so a preparation that failed at any step past the warning left it unset and the advice came back on the next job. On a host whose sweep keeps failing that is every job, which is how an operator learns to skip warnings. It is recorded where it is printed. The table also says it is unenforced on purpose. Nothing checks that the code matches it, a checker would be worth having, and building one here is not the trade to make: the table works by being read, and a reader adding a step has to pick a column. Saying so keeps the gap from looking like an oversight. --- plus/bencher_runner/src/jail/mod.rs | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 0c0e09094..a55a2c36a 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -47,6 +47,12 @@ //! turn a failure into an answer, rather than by reasoning about which functions //! looked relevant. Reasoning missed instances three times running. //! +//! Nothing enforces this table, deliberately. It is a specification a person +//! reads, and its value is that adding a step means picking a column and writing +//! the reason down next to the others. A checker that proved the code matched it +//! would be worth having, and is not worth building here: the gap is a decision, +//! not an oversight. +//! //! | Step | On failure | //! |---|---| //! | [`HostPreparation::ensure`]: the root check | fails the job | @@ -310,6 +316,8 @@ pub struct HostPreparation { /// Whether the jail user warning has already been given. /// /// Advisory and unchanging, so it is worth saying once and not once a job. + /// Set where the warning is printed, so a preparation that fails afterwards + /// does not earn the operator the same advice again. #[cfg_attr( not(target_os = "linux"), expect(dead_code, reason = "host preparation is Linux-only") @@ -525,8 +533,11 @@ impl HostPreparation { if self.prepared && !self.reclaim_failed.is_set() { return Ok(()); } - let swept = prepare_host(euid, state_dir, jail_user, !self.warned_jail_user)?; - self.warned_jail_user = true; + // The flag is passed in rather than set after, because preparation can + // fail at a step past the warning: recording it here would repeat the + // advice on every job of a host whose sweep keeps failing, and recording + // it only on success would do the same. + let swept = prepare_host(euid, state_dir, jail_user, &mut self.warned_jail_user)?; // Spent only on a sweep that finished, and armed by one that did not. // Spending it any earlier disarms the mechanism precisely when it is // needed: the signal would be gone, this process would still count as @@ -572,7 +583,7 @@ fn prepare_host( euid: u32, state_dir: &camino::Utf8Path, jail_user: JailUser, - announce_jail_user: bool, + jail_user_announced: &mut bool, ) -> Result { // Checked first, and by name. Without it the most likely upgrade failure // surfaces as a permission error on a directory, or a bare EPERM out of @@ -582,12 +593,14 @@ fn prepare_host( let state = StateDir::new(state_dir.to_owned())?; state.create()?; - // Once per runner, not once per job. Preparation runs again whenever a - // sweep is owed, and a host that keeps failing to reclaim a jail would - // otherwise repeat this warning on every job until an operator stopped - // reading any of them. - if announce_jail_user { + // Once per runner, not once per job, and recorded at the moment it is given + // rather than at the end of a preparation that may not reach one. This runs + // again whenever a sweep is owed, and a host that keeps failing to reclaim a + // jail would otherwise repeat the advice until an operator stopped reading + // any of it. + if !*jail_user_announced { warn_on_named_account(jail_user); + *jail_user_announced = true; } let _lock = JailLock::acquire(state.path())?; From b3e1d7d2238fe472a6139b06518ea1ca531ddec1 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 08:31:19 +0000 Subject: [PATCH 82/91] Fix a typo in the stale cgroup reasoning "would stranded" to "would strand". --- plus/bencher_runner/src/jail/cgroup.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index fdf8a2623..db5edeabc 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -473,7 +473,7 @@ pub(crate) fn remove_stale_cgroup(vm_id: &VmId) -> Result<(), JailError> { .join(vm_id.as_str()); // The caller deletes the chroot that names this cgroup once this returns // `Ok`, and that chroot is the only handle any later sweep has for finding - // the cgroup again. A stat error read as "already gone" would stranded the + // the cgroup again. A stat error read as "already gone" would strand the // cgroup permanently and delete the one thing that could have found it, // which is exactly what the caller's ordering exists to prevent. match path.try_exists() { From f379837307c1e8009280f1a0cbb97c98cf513b28 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 09:19:41 +0000 Subject: [PATCH 83/91] Look once more before calling a VMM still running The wait polled between sleeps and then returned a bare `false`, so a VMM that exited during the last interval of its five second budget was never looked at again. The sweep turns that into `JailError::JailStillRunning` and fails the job, confidently naming a pid that is already gone, on a host that is clean. Narrow, and the next sweep heals it, but a spurious failure with a definite error message is worse than most of what this branch has fixed. The same shape as the readiness wait in the VMM process, where the cost was a misleading message rather than a failed job. The deadline and the liveness check become parameters so the interesting case can be tested at all: reproducing a process that exits in the final interval against a real process means racing a sleep, while a deadline already spent proves the check after the loop is the whole verdict, with no sleeps and nothing timing-dependent. --- plus/bencher_runner/src/jail/reap.rs | 45 ++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/plus/bencher_runner/src/jail/reap.rs b/plus/bencher_runner/src/jail/reap.rs index d4fe2e9f5..5124bd8a0 100644 --- a/plus/bencher_runner/src/jail/reap.rs +++ b/plus/bencher_runner/src/jail/reap.rs @@ -335,14 +335,32 @@ fn pidfd_kill(pidfd: &OwnedFd) -> std::io::Result<()> { /// then stall the full timeout on every sweep and warn about a process that is /// already dead. fn wait_for_exit(pid: u32) -> bool { - let deadline = Instant::now() + REAP_TIMEOUT; + wait_for_exit_until(Instant::now() + REAP_TIMEOUT, || is_running(pid)) +} + +/// The wait, with the deadline and the liveness check supplied. +/// +/// Parameters for the same reason the scan and the kill are: the case worth +/// testing is a process that exits in the last interval of the budget, and +/// reproducing that against a real process means racing a sleep. +fn wait_for_exit_until(deadline: Instant, running: R) -> bool +where + R: Fn() -> bool, +{ while Instant::now() < deadline { - if !is_running(pid) { + if !running() { return true; } std::thread::sleep(REAP_INTERVAL); } - false + + // Once more before the verdict. The loop sleeps between checks, so a process + // that exits during the last of them was never looked at again, and a bare + // `false` here becomes `JailError::JailStillRunning`: a job failed on a host + // that is clean, with an error confidently naming a pid that is already + // gone. The same shape as the readiness wait in the VMM process, where it + // cost a misleading message rather than a spurious failure. + !running() } /// Whether a process still exists and is not a zombie. @@ -429,6 +447,27 @@ mod tests { } } + #[test] + fn a_process_that_exits_in_the_last_interval_counts_as_exited() { + // A deadline already spent, so the loop never runs and the check after + // it is the whole verdict. Without that check the caller fails the job + // on a host that is clean, naming a pid that no longer exists. + let checks = std::cell::Cell::new(0); + + let exited = wait_for_exit_until(Instant::now(), || { + checks.set(checks.get() + 1); + false + }); + + assert!(exited, "the process is gone, whatever the budget did"); + assert_eq!(checks.get(), 1, "the check after the loop is the verdict"); + } + + #[test] + fn a_process_that_outlives_the_budget_is_still_running() { + assert!(!wait_for_exit_until(Instant::now(), || true)); + } + #[test] fn a_zombie_counts_as_exited() { // Without this the reap stalls its full timeout whenever the runner is From 1dbe0f5aa8eb63b3fe4531c33b745ff167fd5168 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 09:19:41 +0000 Subject: [PATCH 84/91] Say why the grace period's fall-through is not the same bug Third instance of one loop shape, and this one is sound, so the reasoning is written down rather than left to be re-derived by whoever greps for the pattern next. A child that exits during the final sleep falls through to the kill, and nothing was reaped, so the pid is still reserved by the child and cannot have been recycled: the signal is one a zombie ignores, and the kill then reaps it and joins the reader. That is exactly what the loop would have done, so there is no verdict here to get wrong. --- plus/bencher_runner/src/firecracker/process.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plus/bencher_runner/src/firecracker/process.rs b/plus/bencher_runner/src/firecracker/process.rs index dcf01101d..51c6b2365 100644 --- a/plus/bencher_runner/src/firecracker/process.rs +++ b/plus/bencher_runner/src/firecracker/process.rs @@ -227,7 +227,12 @@ impl FirecrackerProcess { std::thread::sleep(poll_interval); } - // Force kill if still running + // Force kill if still running. Unlike the readiness wait above, a child + // that exits during the final sleep is not a missed case here: nothing + // was reaped, so the pid is still reserved by the child and cannot have + // been recycled, `kill` sends a signal that a zombie simply ignores, and + // `kill` then reaps it and joins the reader. That is precisely what the + // loop would have done, so there is no verdict to get wrong. self.kill(); } From 9ba83d56b3f8b41e77f591ae92ea2561b5656c30 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 10:06:08 +0000 Subject: [PATCH 85/91] Match the reap's outcomes exhaustively `if let Reaped::StillRunning` sent both other outcomes down the same path, so a third one arriving later would fall through into another pass of the loop rather than being reported. `reap_one` cannot return `Unexaminable` today, so this is not a live defect; it is the construct that would let one become invisible. In the one module whose entire argument is that a state nobody could examine must not read as a cleared one, that belongs to the compiler. --- plus/bencher_runner/src/jail/reap.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/plus/bencher_runner/src/jail/reap.rs b/plus/bencher_runner/src/jail/reap.rs index 5124bd8a0..0b2ef4cc9 100644 --- a/plus/bencher_runner/src/jail/reap.rs +++ b/plus/bencher_runner/src/jail/reap.rs @@ -99,10 +99,18 @@ where // would otherwise have the tree removed out from under it. for _ in 0..MAX_JAILED_PROCESSES { match find(jail_root) { - Ok(Some(pid)) => { - if let Reaped::StillRunning { pid } = reap(pid, jail_root) { - return Reaped::StillRunning { pid }; - } + Ok(Some(pid)) => match reap(pid, jail_root) { + // Killed, or already gone: look again, because a jail is not + // clear until a scan finds nothing in it. + Reaped::Clear => {}, + Reaped::StillRunning { pid } => return Reaped::StillRunning { pid }, + // `reap_one` cannot return this today, and it is matched + // exhaustively so that a third outcome arriving here is a + // compile error rather than a silent fall-through into another + // pass of the loop. This module's whole argument is that a state + // nobody could examine must not read as a cleared one, and an + // `if let` is the construct that lets exactly that happen. + Reaped::Unexaminable => return Reaped::Unexaminable, }, Ok(None) => return Reaped::Clear, // A scan that could not run has not found the jail empty, it has From 82e2273742dce4973a7cf8198930e75c22bcbe7b Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Thu, 30 Jul 2026 10:06:08 +0000 Subject: [PATCH 86/91] Document the invocation the scenarios actually accept `TEST.md` told the reader twice to run `cargo test-runner scenarios`, which has always failed since the scenarios started requiring root: the sandbox is built by dropping privilege, so they refuse to start without it. The working two-step form existed only in the CI workflow and in the bail message. A contributor following the documented instructions was guaranteed a failure, which is the cost the rule about keeping these current exists to prevent. Both invocations are the two-step form now, with the reason the build stays unprivileged: it keeps `cargo` from leaving root-owned artifacts in the target directory. The failure-patterns section replaces the note about expected tuning permission errors, which cannot happen now that the scenarios pass `--no-tuning`, with the two things a reader will actually hit: the root refusal, and no tuning output at all. The root `CLAUDE.md` entry says root alongside Linux and KVM. --- CLAUDE.md | 5 ++++- services/runner/TEST.md | 26 +++++++++++++++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8011903ff..434193b6f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -168,7 +168,10 @@ Defined in `.cargo/config.toml`: - `cargo test-api seed` needs the API server already running (`cargo run` in `services/api`) with a fresh database (`services/api/data` holds only a tracked `.gitignore`; delete `services/api/data/bencher.db`, not the whole directory) - `cargo test-api seed` also needs the `bencher` CLI binary already built (`cargo build --bin bencher`); it shells out via `assert_cmd`, which panics with `` `CARGO_BIN_EXE_bencher` is unset `` if the binary is missing - Pass `--no-git` when running the seed test in this repo: there is no colocated `.git`, so `bencher run` cannot derive a git context and the on-the-fly project naming assertions (`bencher` vs `Project`) will fail without it -- `cargo test-runner` - Runner integration tests (requires Linux + KVM) +- `cargo test-runner` - Runner integration tests (requires Linux + KVM + root) + - `cargo test-runner scenarios` always fails unelevated: the sandbox is built by dropping privilege, so the scenarios refuse to start without it. Build unprivileged, then run elevated, which also keeps `cargo` from leaving root-owned artifacts in `target`: + - `cargo test-runner scenarios --build-only` + - `sudo BENCHER_RUNNER_BIN=./target/debug/runner ./target/debug/test_runner scenarios` ## Git Flow diff --git a/services/runner/TEST.md b/services/runner/TEST.md index ec34ec993..ffb7094c8 100644 --- a/services/runner/TEST.md +++ b/services/runner/TEST.md @@ -1,6 +1,15 @@ # Runner Integration Tests -The runner integration tests (`cargo test-runner scenarios`) require Linux with KVM. They cannot run on macOS directly. A GCP VM is available for running these tests remotely. +The runner integration tests require Linux with KVM **and root**. They cannot run on macOS directly. A GCP VM is available for running these tests remotely. + +`cargo test-runner scenarios` on its own always fails now: the sandbox is built by dropping privilege rather than by starting without it, so the scenarios refuse to run unelevated. Build unprivileged, then run elevated: + +```bash +cargo test-runner scenarios --build-only +sudo BENCHER_RUNNER_BIN=./target/debug/runner ./target/debug/test_runner scenarios +``` + +Keeping the build out of the elevated half is what stops `cargo` leaving root-owned artifacts in the target directory, which is why `BENCHER_RUNNER_BIN` points the elevated binary at the one already built. ## Prerequisites @@ -55,18 +64,24 @@ All `cargo test-runner` commands must be run on the VM (they require Linux + KVM ### Run all scenarios +Two commands, because only the second may be elevated: + ```bash gcloud compute ssh bencher-vmm-test --zone=us-central1-a --project=bencher-411313 \ - --command="export PATH=\$HOME/.cargo/bin:\$PATH && cd bencher && cargo test-runner scenarios 2>&1" + --command="export PATH=\$HOME/.cargo/bin:\$PATH && cd bencher && cargo test-runner scenarios --build-only 2>&1" +gcloud compute ssh bencher-vmm-test --zone=us-central1-a --project=bencher-411313 \ + --command="cd bencher && sudo BENCHER_RUNNER_BIN=./target/debug/runner ./target/debug/test_runner scenarios 2>&1" ``` -This builds `bencher-init` (musl, statically linked) and the runner CLI, then runs all ~58 integration scenarios. Each scenario builds a Docker image, converts it to OCI format, and runs it inside a Firecracker microVM. Expect this to take several minutes. +The first builds `bencher-init` (musl, statically linked) and the runner CLI; the second runs all integration scenarios as root. Each scenario builds a Docker image, converts it to OCI format, and runs it inside a Firecracker microVM. Expect this to take several minutes. ### Run a single scenario ```bash gcloud compute ssh bencher-vmm-test --zone=us-central1-a --project=bencher-411313 \ - --command="export PATH=\$HOME/.cargo/bin:\$PATH && cd bencher && cargo test-runner scenarios --scenario basic_execution 2>&1" + --command="export PATH=\$HOME/.cargo/bin:\$PATH && cd bencher && cargo test-runner scenarios --build-only 2>&1" +gcloud compute ssh bencher-vmm-test --zone=us-central1-a --project=bencher-411313 \ + --command="cd bencher && sudo BENCHER_RUNNER_BIN=./target/debug/runner ./target/debug/test_runner scenarios --scenario basic_execution 2>&1" ``` ### List all scenarios @@ -111,4 +126,5 @@ gcloud compute ssh bencher-vmm-test --zone=us-central1-a --project=bencher-41131 - **Compilation errors about private imports**: The `plus` feature gates most code. Check `pub use` re-exports if a type is accessible within a crate but not from outside its module. - **"KVM is not available"**: The scenarios require `/dev/kvm` on the host. This is why they must run on the GCP VM, not macOS. - **Timeout scenarios take wall-clock time**: Scenarios like `timeout_handling`, `timeout_enforced`, and `minimum_timeout` intentionally wait for the VM to time out (1-10 seconds each). This is expected. -- **Tuning permission errors in stderr**: Messages like `Tuning: ASLR — skipped (write failed: Permission denied)` are expected when not running as root. They do not cause test failures. +- **"The scenarios must run as root"**: `cargo test-runner scenarios` was run directly. Use the two-step invocation at the top of this file; the message itself repeats it. +- **No tuning output at all**: expected. The scenarios pass `--no-tuning`, because they run as root now and would otherwise apply real host tuning to the machine running them, including offlining SMT siblings mid-suite. They exercise job execution, not tuning. From c6e92e2871e96bfc2693bf87e55a72e4b77daf7c Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Fri, 31 Jul 2026 02:59:20 +0000 Subject: [PATCH 87/91] Test that host tuning applies and unwinds Elevating the scenarios turned real tuning on for all twenty-five of them, and `--no-tuning` went in to stop the suite detuning the machine running it. That bought safety by giving up every scrap of coverage for eleven knobs and the cpuset partition, and the half that matters most had never executed anywhere: `TuningGuard` restores on `Drop`, and nothing in CI had ever watched it do so. One scenario gets tuning back, and it asserts the pair. While the Job runs, each setting the host will let the runner change must show its tuned value; once the runner exits, every one must be back to what it was. Applying is what the runner is for, restoring is what keeps a benchmark host from drifting a knob at a time across every Job it ever runs. What is exercised is decided by asking the host rather than by assuming. A setting that is absent, already at the target, or present but not writable is recorded and skipped, and writability is established by writing back the value already there, which changes nothing: on a kernel with no hardware watchdog `/proc/sys/kernel/nmi_watchdog` exists, reads `0`, and refuses writes, and waiting for it to change would fail for the host's reasons rather than the runner's. A host that offers nothing to change fails the scenario rather than passing it, because a tuning test that tunes nothing is the vacuous pass this suite has spent the most effort removing. The blast radius is bounded four ways. It is one scenario, so only it can contaminate anything. It runs last, so what it leaves cannot reach the others. The harness snapshots every setting first and restores them from its own copy through a guard that runs on panic and on early return, so the scenario is safe even when the mechanism it tests is broken, which is the property a test of a restore path needs. And two knobs are excluded by argument: `--smt` keeps hyper-threading on, since offlining a sibling changes the core count for everything after it and no harness can put a CPU back, and `--no-irq-steering` because an unmovable IRQ rejects the restoring write with EIO, so the harness could not promise to undo it. --- Cargo.lock | 1 + tasks/test_runner/Cargo.toml | 3 + tasks/test_runner/src/task/scenarios.rs | 553 +++++++++++++++++++++++- 3 files changed, 550 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 27ac9cb57..ba445fddd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8264,6 +8264,7 @@ dependencies = [ "clap", "libc", "serde_json", + "tempfile", ] [[package]] diff --git a/tasks/test_runner/Cargo.toml b/tasks/test_runner/Cargo.toml index 59899b24b..7a0a9d47b 100644 --- a/tasks/test_runner/Cargo.toml +++ b/tasks/test_runner/Cargo.toml @@ -19,5 +19,8 @@ camino.workspace = true clap.workspace = true serde_json.workspace = true +[dev-dependencies] +tempfile.workspace = true + [lints] workspace = true diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index 58ace85ae..79ff40ea6 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -64,6 +64,11 @@ struct Scenario { setup: Option Result<()>>, /// If set, a host-side check run while the runner is executing. probe: Option, + /// Run with host tuning enabled and assert it applies and is restored. + /// + /// Every other scenario passes `--no-tuning`, so this is the only one that + /// can leave the machine changed, and the harness undoes it itself. + tuning: bool, /// Kill the runner once its VMM is up so nothing unwinds, then run the /// image again and report the second run. /// @@ -83,6 +88,7 @@ impl Default for Scenario { cancel_after_secs: None, setup: None, probe: None, + tuning: false, orphan_then_rerun: false, // Sandboxed is the interesting case and the overwhelming majority, // so the handful of non-sandboxed scenarios opt out rather than @@ -174,6 +180,10 @@ impl Scenarios { let mut scenarios = all_scenarios(); scenarios.extend(jail_scenarios()); scenarios.extend(nosandbox_scenarios()); + // Last, always. It is the only scenario that tunes the machine, so + // nothing it leaves behind can reach the others, and if the suite is + // killed part way through it is the least likely to have started. + scenarios.extend(tuning_scenarios()); let result = if let Some(name) = &self.scenario { // Run a single scenario @@ -235,6 +245,7 @@ fn list_scenarios() { let mut scenarios = all_scenarios(); scenarios.extend(jail_scenarios()); scenarios.extend(nosandbox_scenarios()); + scenarios.extend(tuning_scenarios()); println!("Available scenarios:"); println!(); for scenario in &scenarios { @@ -298,13 +309,27 @@ fn run_scenario(scenario: &Scenario, runner_bin: &Utf8Path) -> Result<()> { let state_dir = scenario_state_dir(); drop(fs::remove_dir_all(&state_dir)); - // Prepend --sandbox firecracker for sandboxed scenarios - // --no-tuning matters now that the scenarios run as root. Unprivileged, - // every tuning knob failed with EPERM and warned; elevated they actually - // apply, and offlining SMT siblings on a two-vCPU hosted runner would - // change the core count mid-suite. The scenarios exercise job execution, - // not tuning, so this costs no coverage. - let mut args: Vec<&str> = vec!["--state-dir", state_dir.as_str(), "--no-tuning"]; + // Prepend --sandbox firecracker for sandboxed scenarios. + // + // `--no-tuning` everywhere except the one scenario that exists to test + // tuning. Elevated, the knobs really apply, and a suite that tuned the + // machine twenty-five times would offline SMT siblings on a two-vCPU hosted + // runner, changing the core count under itself. The tuning scenario turns + // them back on for one job, keeps SMT and IRQ steering out of it, and the + // harness undoes everything itself afterwards. + let mut args: Vec<&str> = vec!["--state-dir", state_dir.as_str()]; + if scenario.tuning { + // The two knobs this scenario deliberately does not exercise. `--smt` + // keeps hyper-threading on: offlining a sibling changes `nproc` for + // everything that follows in the CI job, and the harness cannot put a + // CPU back if the runner is killed before its guard runs. IRQ steering + // is skipped because a hand-restore of it is unavoidably partial, since + // an unmovable IRQ rejects the write with EIO, and the rule for this + // scenario is that the harness can undo anything it turned on. + args.extend(["--smt", "--no-irq-steering"]); + } else { + args.push("--no-tuning"); + } if scenario.sandboxed { args.extend(["--sandbox", "firecracker"]); } @@ -317,6 +342,8 @@ fn run_scenario(scenario: &Scenario, runner_bin: &Utf8Path) -> Result<()> { run_runner_after_orphan(&image_path, &args, &state_dir, runner_bin) } else if let Some(probe) = scenario.probe { run_runner_with_probe(&image_path, &args, probe, &state_dir, runner_bin) + } else if scenario.tuning { + run_runner_with_tuning(&image_path, &args, runner_bin) } else { run_runner(&image_path, &args, runner_bin) } @@ -3060,6 +3087,437 @@ fn kill_pid(pid: u32, signal: libc::c_int) { } } +/// The host settings the tuning scenario watches, and what the runner sets them +/// to. +/// +/// Only settings whose whole value the runner rewrites. The bracketed sysfs +/// files (`transparent_hugepage/*`) and the cpuset partition are handled +/// separately below, and the knobs this scenario deliberately leaves alone are +/// listed with the arguments that switch them off. +const TUNED_SETTINGS: &[(&str, &str)] = &[ + ("/proc/sys/kernel/randomize_va_space", "0"), + ("/proc/sys/kernel/nmi_watchdog", "0"), + ("/proc/sys/vm/swappiness", "10"), + ("/proc/sys/kernel/perf_event_paranoid", "-1"), + ("/proc/sys/kernel/numa_balancing", "0"), + ("/proc/sys/kernel/timer_migration", "0"), + ("/proc/sys/kernel/soft_watchdog", "0"), + ("/sys/kernel/mm/ksm/run", "0"), +]; + +/// The transparent hugepage settings, whose files list every mode and bracket +/// the selected one. +const TUNED_THP: &[&str] = &[ + "/sys/kernel/mm/transparent_hugepage/enabled", + "/sys/kernel/mm/transparent_hugepage/defrag", +]; + +/// What the runner sets the transparent hugepage mode to. +const THP_TARGET: &str = "never"; + +/// The cpuset partition files, which the tuning writes and the guard restores. +const TUNED_PARTITION: &[&str] = &[ + "/sys/fs/cgroup/bencher/cpuset.cpus", + "/sys/fs/cgroup/bencher/cpuset.mems", + "/sys/fs/cgroup/bencher/cpuset.cpus.partition", +]; + +/// One host setting the scenario expects the runner to change. +#[derive(Debug, Clone)] +struct TunedSetting { + path: Utf8PathBuf, + /// What it held before the runner started, and what it must hold after. + original: String, + /// What the runner should set it to while the Job runs, when this host + /// lets it. `None` for a setting that is present and writable but already + /// holds the target, which the runner leaves alone and reports as such. + expected: Option, + /// Whether the value is the bracketed kind (`always [madvise] never`). + bracketed: bool, +} + +/// Every host setting the tuning scenario touches, as it stood before it ran. +/// +/// The harness restores from this itself rather than trusting the mechanism it +/// is testing. A test of a restore path has to be safe when the restore path is +/// broken, which is the whole reason this scenario can be allowed to run in CI +/// at all. +#[derive(Debug)] +struct TuningSnapshot { + settings: Vec, +} + +impl TuningSnapshot { + /// Read every setting, and work out which of them this host will let the + /// runner change. + /// + /// Writability is established by writing the current value back, which + /// changes nothing and is the only honest way to know: a file that exists + /// may still be read-only, and a scenario that waited for a change the + /// kernel was never going to make would fail for the host's reasons rather + /// than the runner's. + fn take() -> Self { + let mut settings = Vec::new(); + + for (path, target) in TUNED_SETTINGS { + let path = Utf8PathBuf::from(*path); + let Some(original) = readable_setting(&path) else { + println!(" tuning: {path} is not present on this host"); + continue; + }; + let expected = if !writable_setting(&path, &original) { + println!(" tuning: {path} is present but not writable"); + None + } else if original == *target { + println!(" tuning: {path} already holds {target}"); + None + } else { + Some((*target).to_owned()) + }; + settings.push(TunedSetting { + path, + original, + expected, + bracketed: false, + }); + } + + for path in TUNED_THP { + let path = Utf8PathBuf::from(*path); + let Some(original) = readable_setting(&path) else { + println!(" tuning: {path} is not present on this host"); + continue; + }; + // Probed with the mode the file already selects, never with a + // fallback: writing `never` to a file whose selection could not be + // parsed would change the very setting this is only supposed to + // measure. + let Some(selected) = bracketed_value(&original) else { + println!(" tuning: {path} does not read as a mode listing: '{original}'"); + continue; + }; + let expected = if !writable_setting(&path, selected) { + println!(" tuning: {path} is present but not writable"); + None + } else if selected == THP_TARGET { + println!(" tuning: {path} already selects {THP_TARGET}"); + None + } else { + Some(THP_TARGET.to_owned()) + }; + settings.push(TunedSetting { + path, + original, + expected, + bracketed: true, + }); + } + + Self { settings } + } + + /// The settings this host should show changed while the Job runs. + fn expected(&self) -> impl Iterator { + self.settings + .iter() + .filter(|setting| setting.expected.is_some()) + } + + /// Whether every expected setting currently holds its tuned value. + fn all_applied(&self) -> bool { + self.expected().all(|setting| { + let Some(current) = readable_setting(&setting.path) else { + return false; + }; + let Some(target) = setting.expected.as_deref() else { + return true; + }; + if setting.bracketed { + bracketed_value(¤t) == Some(target) + } else { + current == target + } + }) + } + + /// Which expected settings are not showing their tuned value. + fn missing(&self) -> Vec { + self.expected() + .filter(|setting| { + let Some(current) = readable_setting(&setting.path) else { + return true; + }; + let target = setting.expected.as_deref().unwrap_or_default(); + if setting.bracketed { + bracketed_value(¤t) != Some(target) + } else { + current != target + } + }) + .map(|setting| { + let current = readable_setting(&setting.path).unwrap_or_else(|| "?".to_owned()); + format!( + "{} is '{current}', expected '{}'", + setting.path, + setting.expected.as_deref().unwrap_or_default() + ) + }) + .collect() + } + + /// Which settings are not back to what they were. + fn unrestored(&self) -> Vec { + self.settings + .iter() + .filter_map(|setting| { + let current = readable_setting(&setting.path)?; + (current != setting.original).then(|| { + format!( + "{} is '{current}', was '{}'", + setting.path, setting.original + ) + }) + }) + .collect() + } + + /// Put everything back, whatever the runner did or failed to do. + /// + /// Reports what it had to undo: anything here means the guard under test did + /// not do its job, and the scenario has already failed for that reason, but + /// the machine still has to be left as it was found. + fn restore(&self) { + for setting in &self.settings { + let Some(current) = readable_setting(&setting.path) else { + continue; + }; + if current == setting.original { + continue; + } + // The bracketed files take the mode alone, never the whole listing. + let value = if setting.bracketed { + bracketed_value(&setting.original) + .unwrap_or(THP_TARGET) + .to_owned() + } else { + setting.original.clone() + }; + match fs::write(&setting.path, &value) { + Ok(()) => println!(" tuning: harness restored {} to '{value}'", setting.path), + Err(e) => println!( + " tuning: harness could NOT restore {} to '{value}': {e}", + setting.path + ), + } + } + } +} + +/// Restores the host tuning when it goes out of scope. +/// +/// A guard rather than a call at the end, so a panic or an early return in the +/// scenario cannot leave the machine tuned. Nothing survives the harness itself +/// being killed, which is why the scenario runs last. +struct RestoreTuning(TuningSnapshot); + +impl Drop for RestoreTuning { + fn drop(&mut self) { + self.0.restore(); + } +} + +/// Read a host setting, trimmed, if it is there at all. +fn readable_setting(path: &Utf8Path) -> Option { + fs::read_to_string(path).ok().map(|v| v.trim().to_owned()) +} + +/// Whether a setting can be written, established by writing back what it holds. +fn writable_setting(path: &Utf8Path, current: &str) -> bool { + fs::write(path, current).is_ok() +} + +/// The selected mode in a bracketed sysfs listing (`always [madvise] never`). +fn bracketed_value(listing: &str) -> Option<&str> { + let (_, selected) = listing.split_once('[')?; + let (selected, _) = selected.split_once(']')?; + Some(selected) +} + +/// The cpuset partition files that exist, with what they hold. +/// +/// Read separately from the rest because the partition is created by the tuning +/// itself: the files do not exist before the first tuned run on a fresh host, so +/// there is nothing to snapshot and their absence afterwards is the restored +/// state. +fn partition_state() -> Vec<(Utf8PathBuf, String)> { + TUNED_PARTITION + .iter() + .map(Utf8PathBuf::from) + .filter_map(|path| readable_setting(&path).map(|value| (path, value))) + .collect() +} + +/// Run the runner with host tuning on, and assert it both applies and unwinds. +/// +/// The assertion that matters is the pair. Applying is what the runner is for; +/// restoring is what keeps a benchmark host from drifting a knob at a time +/// across every Job it ever runs, and `TuningGuard` restoring on `Drop` had +/// never once executed in CI before this scenario existed. +fn run_runner_with_tuning( + image_path: &Utf8Path, + args: &[&str], + runner_bin: &Utf8Path, +) -> Result { + let snapshot = TuningSnapshot::take(); + let expected: Vec = snapshot + .expected() + .map(|setting| { + format!( + "{} -> {}", + setting.path, + setting.expected.as_deref().unwrap_or_default() + ) + }) + .collect(); + + // A scenario that finds nothing to change would pass without testing + // anything, which is the failure this suite has spent the most effort + // removing. If a host really offers none of these, that is a fact worth a + // red build rather than a green one. + anyhow::ensure!( + !expected.is_empty(), + "No tuning knob on this host can be exercised, so the scenario would pass vacuously. Settings considered: {:?}", + snapshot + .settings + .iter() + .map(|s| s.path.as_str()) + .collect::>() + ); + println!( + " tuning: expecting {} setting(s) to change: {}", + expected.len(), + expected.join(", ") + ); + + let partition_before = partition_state(); + + // Taken before the runner starts, so the machine is put back even if the + // scenario panics, the assertions fail, or the runner dies without + // unwinding. The point of the scenario is that the guard under test might + // not work. + let restore = RestoreTuning(snapshot); + + let mut child = Command::new(runner_bin.as_str()) + .arg("run") + .arg("--image") + .arg(image_path.as_str()) + .args(args) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn()?; + let readers = drain_output(&mut child); + + // Watch for the tuning to land while the Job runs. The runner applies it + // before it pulls the image, so this is looking at a window that lasts the + // whole run. + let deadline = std::time::Instant::now() + PROBE_TIMEOUT; + let mut applied = false; + loop { + if restore.0.all_applied() { + applied = true; + break; + } + if child.try_wait()?.is_some() || std::time::Instant::now() >= deadline { + break; + } + std::thread::sleep(PROBE_INTERVAL); + } + let partition_during = partition_state(); + + if !applied && child.try_wait()?.is_none() { + kill_pid(child.id(), libc::SIGKILL); + } + let status = child.wait()?; + let (stdout, stderr) = readers.join(); + + if !applied { + bail!( + "Host tuning never applied within {PROBE_TIMEOUT:?}: {:?}.\nstdout: {stdout}\nstderr: {stderr}", + restore.0.missing() + ); + } + + // The Job has to have succeeded as well. A scenario that only watched the + // knobs would pass on a runner that tuned the host and then failed to run + // anything, which is the vacuous half of a confinement assertion in another + // dress. + if status.code() != Some(0) { + bail!( + "Tuning applied but the Job failed with exit code {:?}.\nstdout: {stdout}\nstderr: {stderr}", + status.code() + ); + } + + // And it has to be gone now that the runner has exited. + let unrestored = restore.0.unrestored(); + if !unrestored.is_empty() { + bail!( + "Host tuning was not restored when the runner exited: {unrestored:?}.\nstdout: {stdout}\nstderr: {stderr}" + ); + } + + // Only the files that were there to change. The partition creates its own + // cgroup, so a file that did not exist before the run has no previous value + // to be restored to, and asserting on its appearance would fail the scenario + // for the tuning having worked. + let partition_after = partition_state(); + let partition_unrestored: Vec = partition_before + .iter() + .filter_map(|(path, before)| { + let after = partition_after + .iter() + .find_map(|(p, v)| (p == path).then_some(v.as_str()))?; + (after != before).then(|| format!("{path} is '{after}', was '{before}'")) + }) + .collect(); + if !partition_unrestored.is_empty() { + bail!( + "The cpuset partition was not restored: {partition_unrestored:?}.\nstdout: {stdout}\nstderr: {stderr}" + ); + } + if partition_during.is_empty() { + println!(" tuning: no cpuset partition files on this host, so none were asserted"); + } + + println!( + " tuning: {} setting(s) applied and restored, {} partition file(s) checked", + expected.len(), + partition_before.len() + ); + + Ok(ScenarioOutput { + stdout, + stderr, + exit_code: status.code().unwrap_or(-1), + }) +} + +/// Scenarios covering host tuning, which every other scenario switches off. +fn tuning_scenarios() -> Vec { + vec![Scenario { + name: "host_tuning", + description: "Host tuning applies while a Job runs and is restored after", + dockerfile: r#"FROM busybox +CMD ["echo", "tuned run complete"]"#, + extra_args: &["--timeout", "60"], + tuning: true, + // The Job's own output as well as the knobs. A run that tuned the host + // and then never booted a VM would otherwise satisfy this scenario. + validate: |output| assert_job_succeeded(output, "tuned run complete"), + ..Scenario::default() + }] +} + /// Run the runner while checking a host-side invariant. fn run_runner_with_probe( image_path: &Utf8Path, @@ -3148,3 +3606,84 @@ fn run_runner( exit_code: output.status.code().unwrap_or(-1), }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_selected_mode_is_the_bracketed_one() { + // What the kernel prints for a transparent hugepage setting: every mode + // it offers, with the live one in brackets. Comparing the whole line + // against "never" would never match, and asserting on a substring would + // match a mode that is merely offered. + assert_eq!( + bracketed_value("always [madvise] never"), + Some("madvise"), + "the enabled listing" + ); + assert_eq!( + bracketed_value("always defer defer+madvise [madvise] never"), + Some("madvise"), + "the defrag listing, which offers more modes" + ); + assert_eq!(bracketed_value("[always] madvise never"), Some("always")); + assert_eq!(bracketed_value("always madvise [never]"), Some("never")); + } + + #[test] + fn a_listing_with_no_selection_has_no_value() { + // A plain sysctl is not a listing, and a truncated read is not a mode. + assert_eq!(bracketed_value("never"), None); + assert_eq!(bracketed_value(""), None); + assert_eq!(bracketed_value("always [madvise"), None); + } + + #[test] + fn a_setting_that_is_not_there_reads_as_absent() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8Path::from_path(dir.path()).unwrap(); + + assert_eq!(readable_setting(&root.join("absent")), None); + } + + #[test] + fn a_setting_reads_back_trimmed() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8Path::from_path(dir.path()).unwrap(); + let path = root.join("swappiness"); + fs::write(&path, "60\n").unwrap(); + + assert_eq!(readable_setting(&path).as_deref(), Some("60")); + } + + #[test] + fn writability_is_established_by_writing_what_is_already_there() { + // The probe that decides whether a knob can be exercised on this host. + // A file that exists may still refuse writes, which is not something a + // stat can answer: `/proc/sys/kernel/nmi_watchdog` is exactly that on a + // kernel without a hardware watchdog, and waiting for it to change would + // fail the scenario for the host's reasons rather than the runner's. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8Path::from_path(dir.path()).unwrap(); + let path = root.join("knob"); + fs::write(&path, "1\n").unwrap(); + + assert!(writable_setting(&path, "1")); + assert_eq!( + readable_setting(&path).as_deref(), + Some("1"), + "the probe writes back what was there, so it changes nothing" + ); + + // Root ignores the permission bits, and the scenarios run as root, so + // this half only means anything unprivileged. + if !is_root() { + let mut perms = fs::metadata(&path).unwrap().permissions(); + perms.set_readonly(true); + fs::set_permissions(&path, perms).unwrap(); + + assert!(!writable_setting(&path, "1")); + } + } +} From 6d545bd8f543dd4086a3d51b7ebd5f36a1066533 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Fri, 31 Jul 2026 02:59:37 +0000 Subject: [PATCH 88/91] Document the tuning scenario What `host_tuning` covers, what it deliberately does not, and the two failures a reader will actually hit: a host with nothing to change, which is refused rather than passed, and a machine left tuned, which can only mean the guard under test failed since the harness restores from its own snapshot regardless. --- services/runner/TEST.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/services/runner/TEST.md b/services/runner/TEST.md index ffb7094c8..b1610f609 100644 --- a/services/runner/TEST.md +++ b/services/runner/TEST.md @@ -75,6 +75,10 @@ gcloud compute ssh bencher-vmm-test --zone=us-central1-a --project=bencher-41131 The first builds `bencher-init` (musl, statically linked) and the runner CLI; the second runs all integration scenarios as root. Each scenario builds a Docker image, converts it to OCI format, and runs it inside a Firecracker microVM. Expect this to take several minutes. +### The tuning scenario + +`host_tuning` is the only scenario that runs with host tuning enabled, and it runs last. It asserts that each setting the host lets the runner change is changed while the Job runs and restored when it exits. It excludes two knobs deliberately: SMT stays on, because offlining a sibling changes the core count for everything that follows and no harness can put a CPU back, and IRQ steering is skipped, because an unmovable IRQ rejects the restoring write with `EIO` so the harness cannot promise to undo it. Governor, turbo, and deep C-states are not asserted either: the first two have no sysfs to write inside a VM, and the third is a held file descriptor rather than a value anything can read back. + ### Run a single scenario ```bash @@ -127,4 +131,6 @@ gcloud compute ssh bencher-vmm-test --zone=us-central1-a --project=bencher-41131 - **"KVM is not available"**: The scenarios require `/dev/kvm` on the host. This is why they must run on the GCP VM, not macOS. - **Timeout scenarios take wall-clock time**: Scenarios like `timeout_handling`, `timeout_enforced`, and `minimum_timeout` intentionally wait for the VM to time out (1-10 seconds each). This is expected. - **"The scenarios must run as root"**: `cargo test-runner scenarios` was run directly. Use the two-step invocation at the top of this file; the message itself repeats it. -- **No tuning output at all**: expected. The scenarios pass `--no-tuning`, because they run as root now and would otherwise apply real host tuning to the machine running them, including offlining SMT siblings mid-suite. They exercise job execution, not tuning. +- **No tuning output at all**: expected for every scenario except `host_tuning`. The rest pass `--no-tuning`, because they run as root and would otherwise apply real host tuning to the machine running them, including offlining SMT siblings mid-suite. +- **`host_tuning` fails with "No tuning knob on this host can be exercised"**: the host offers nothing the runner can change, so the scenario refuses to pass vacuously. The line above it lists what it considered and why each was skipped: absent, already at the target, or not writable. +- **`host_tuning` left the machine tuned**: it should not be possible. The harness snapshots every setting before the runner starts and restores from its own copy afterwards, independently of the runner's guard, and prints `harness restored ...` for anything it had to put back. Anything it reports there is a real failure of `TuningGuard`, not of the scenario. From 8350d9bb2c5ee4833ca6851516a27dc874e64b6e Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Fri, 31 Jul 2026 03:43:04 +0000 Subject: [PATCH 89/91] Remove the cgroup the tuning partition leaves behind CI found the runner leaving `/sys/fs/cgroup/bencher` with `cpuset.cpus=1-3` and `cpuset.mems=0` after a run that reported restoring the partition. Neither obvious explanation was right, and both were checked against a real cgroupfs rather than reasoned about. The restore is not broken: replaying the exact sequence as root on a real cgroup v2 host, including a delegated `cpuset` subtree and a child cgroup, clears both files back to empty and prints all three restore lines. The newline write the unit test asserts works on the real filesystem too. Only one restore line looked present in CI because the other two carry a newline as their value, so they print as a line ending in a space followed by a blank one. What does defeat it is a task still in a descendant cgroup: the kernel refuses to clear a parent's cpuset with `EIO` while anything below it would be left without CPUs, which reproduces the CI symptom exactly, values and all. The partition mode restores, the two cpuset files do not, and the runner reports both outcomes honestly. So the residue is real and the runner should not leave it. The `bencher` cgroup is the runner's own, nothing else reads it, and the next job recreates it on demand, so the guard now removes it once the settings are restored. `rmdir` is self-guarding: it succeeds only when the cgroup is empty, so a concurrent runner's job or a task this one could not reclaim leaves it standing. That also breaks a quieter cycle, since a stale `cpuset.cpus` becomes the value the next tuned run saves as the original and faithfully restores forever. Verified on a real cgroupfs: the cgroup is gone after the guard drops, where before it stood with this run's cpuset in it. The fake tree the unit tests use cannot show this, because ordinary files in a directory make `rmdir` fail where a kernel cgroup's own files do not. --- plus/bencher_runner/src/tuning/dma_latency.rs | 1 + plus/bencher_runner/src/tuning/kernel_work.rs | 1 + plus/bencher_runner/src/tuning/mod.rs | 43 +++++++++++++++++++ plus/bencher_runner/src/tuning/partition.rs | 14 ++++-- 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/plus/bencher_runner/src/tuning/dma_latency.rs b/plus/bencher_runner/src/tuning/dma_latency.rs index 516c8854a..5ae1b4374 100644 --- a/plus/bencher_runner/src/tuning/dma_latency.rs +++ b/plus/bencher_runner/src/tuning/dma_latency.rs @@ -59,6 +59,7 @@ mod tests { TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), } } diff --git a/plus/bencher_runner/src/tuning/kernel_work.rs b/plus/bencher_runner/src/tuning/kernel_work.rs index 2ef2d5381..d613442a8 100644 --- a/plus/bencher_runner/src/tuning/kernel_work.rs +++ b/plus/bencher_runner/src/tuning/kernel_work.rs @@ -101,6 +101,7 @@ mod tests { TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), } } diff --git a/plus/bencher_runner/src/tuning/mod.rs b/plus/bencher_runner/src/tuning/mod.rs index 92a7b655b..dab6e03cb 100644 --- a/plus/bencher_runner/src/tuning/mod.rs +++ b/plus/bencher_runner/src/tuning/mod.rs @@ -158,6 +158,13 @@ struct SavedSetting { #[cfg(target_os = "linux")] pub struct TuningGuard { saved: Vec, + /// Cgroups to remove once every setting has been restored. + /// + /// Removed rather than reset, because an empty cgroup carrying a stale + /// cpuset is residue the runner owns and nothing else reads. `rmdir` is + /// self-guarding: it succeeds only when the cgroup is empty, so a concurrent + /// runner's job or a task this one could not reclaim leaves it in place. + remove_if_empty: Vec, /// File descriptors held open for the lifetime of the guard /// (e.g., the PM `QoS` constraint on `/dev/cpu_dma_latency`). /// Dropped after the saved settings are restored. @@ -174,6 +181,11 @@ impl TuningGuard { pub(crate) fn save_restore(&mut self, path: Utf8PathBuf, value: String, label: String) { self.saved.push(SavedSetting { path, value, label }); } + + /// Record a cgroup to remove once the settings have been restored. + pub(crate) fn remove_when_empty(&mut self, path: Utf8PathBuf) { + self.remove_if_empty.push(path); + } } #[cfg(target_os = "linux")] @@ -182,6 +194,28 @@ impl Drop for TuningGuard { for setting in self.saved.iter().rev() { restore(&setting.path, &setting.value, &setting.label); } + // After the settings, because a cgroup that is about to be removed still + // has to have its values put back for the case where the removal cannot + // happen. + for path in &self.remove_if_empty { + remove_empty_cgroup(path); + } + } +} + +/// Remove a cgroup the runner is done with, if nothing is left in it. +/// +/// The whole point is that this can fail and that failing is correct. A cgroup +/// still holding a task, or a concurrent runner's job, refuses `rmdir` with +/// `EBUSY`, and the alternative to leaving it is tearing a cgroup out from under +/// something that is using it. +#[cfg(target_os = "linux")] +#[expect(clippy::print_stdout, reason = "tuning reports what it unwound")] +fn remove_empty_cgroup(path: &Utf8Path) { + match std::fs::remove_dir(path) { + Ok(()) => println!(" Tuning: removed the cgroup {path}"), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}, + Err(e) => println!(" Tuning: left the cgroup {path} in place: {e}"), } } @@ -191,6 +225,7 @@ pub fn apply(config: &TuningConfig) -> TuningGuard { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; if config.disable_aslr { @@ -638,6 +673,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; guard.saved.push(SavedSetting { path: file_path.clone(), @@ -668,6 +704,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; guard.saved.push(SavedSetting { path: path1.clone(), @@ -691,6 +728,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; write_sysctl(&mut guard, "/nonexistent/path/value", "0", "test"); assert!( @@ -711,6 +749,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; write_sysctl(&mut guard, path.to_str().unwrap(), "0", "test"); assert!( @@ -750,6 +789,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; write_bracketed_sysctl(&mut guard, path.to_str().unwrap(), "never", "test"); @@ -770,6 +810,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; write_bracketed_sysctl(&mut guard, path.to_str().unwrap(), "never", "test"); @@ -793,6 +834,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; write_bracketed_sysctl(&mut guard, path.to_str().unwrap(), "never", "test"); @@ -812,6 +854,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; write_sysctl(&mut guard, path.to_str().unwrap(), "10", "test"); diff --git a/plus/bencher_runner/src/tuning/partition.rs b/plus/bencher_runner/src/tuning/partition.rs index 207578ba9..0ba0b8a97 100644 --- a/plus/bencher_runner/src/tuning/partition.rs +++ b/plus/bencher_runner/src/tuning/partition.rs @@ -71,13 +71,21 @@ impl BencherPartition { return PartitionLevel::Member; } - if !self.path.exists() - && let Err(e) = fs::create_dir_all(&self.path) - { + if let Err(e) = fs::create_dir_all(&self.path) { eprintln!("Warning: failed to create cgroup {}: {e}", self.path); return PartitionLevel::Member; } + // Removed on the way out, once the values have been restored. An empty + // `bencher` cgroup left carrying this run's cpuset is residue: nothing + // else reads that cgroup, the next job recreates it on demand, and + // clearing `cpuset.cpus` back to inherit-everything is refused with + // `EIO` while any task remains in a descendant, so the restore alone + // cannot always undo what this does. Registered whoever created the + // directory, because the residue is the same either way and `rmdir` + // refuses while anything is still inside. + guard.remove_when_empty(self.path.clone()); + // A partition needs explicit cpus and mems. Mems mirror the // root's effective nodes so multi-node NUMA hosts are not forced // onto node 0. From 53035624e6c1bd1c5bfa541364af96db87eae5b0 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Fri, 31 Jul 2026 03:43:14 +0000 Subject: [PATCH 90/91] Have the tuning scenario diagnose its own partition failure The first CI run of this scenario reported which files were not restored and nothing about why, and the answer, that clearing a parent cpuset is refused while a task remains in a descendant, cost an afternoon of experiments on a real cgroupfs to establish. The bail now says whether the cgroup is still there, what tasks it holds, and which children hold what, so the next failure of this kind arrives with its cause attached rather than costing another round. --- tasks/test_runner/src/task/scenarios.rs | 58 ++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index 79ff40ea6..fcc191911 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -3357,17 +3357,54 @@ fn partition_state() -> Vec<(Utf8PathBuf, String)> { .collect() } +/// What the `bencher` cgroup looks like, for when the partition assertion fails. +/// +/// Clearing a parent's `cpuset.cpus` is refused with `EIO` while any task remains +/// in a descendant, so the useful question after a failed restore is what is +/// still in there. Without this the answer costs a CI round. +fn partition_diagnosis() -> String { + let root = Utf8Path::new("/sys/fs/cgroup/bencher"); + if !root.exists() { + return "the bencher cgroup is gone".to_owned(); + } + let procs = fs::read_to_string(root.join("cgroup.procs")).unwrap_or_default(); + let children: Vec = fs::read_dir(root) + .map(|entries| { + entries + .flatten() + .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_dir())) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect() + }) + .unwrap_or_default(); + let child_procs: Vec = children + .iter() + .map(|child| { + let tasks = + fs::read_to_string(root.join(child).join("cgroup.procs")).unwrap_or_default(); + format!( + "{child} holds [{}]", + tasks.split_whitespace().collect::>().join(" ") + ) + }) + .collect(); + format!( + "the bencher cgroup is still there, holding tasks [{}] and children {child_procs:?}", + procs.split_whitespace().collect::>().join(" ") + ) +} + /// Run the runner with host tuning on, and assert it both applies and unwinds. /// /// The assertion that matters is the pair. Applying is what the runner is for; /// restoring is what keeps a benchmark host from drifting a knob at a time /// across every Job it ever runs, and `TuningGuard` restoring on `Drop` had /// never once executed in CI before this scenario existed. -fn run_runner_with_tuning( - image_path: &Utf8Path, - args: &[&str], - runner_bin: &Utf8Path, -) -> Result { +/// Read the host, and work out what this run should change. +/// +/// Separated so the scenario itself stays readable: everything here happens +/// before the runner starts and decides whether there is anything to test. +fn plan_tuning() -> Result<(TuningSnapshot, Vec)> { let snapshot = TuningSnapshot::take(); let expected: Vec = snapshot .expected() @@ -3398,7 +3435,15 @@ fn run_runner_with_tuning( expected.len(), expected.join(", ") ); + Ok((snapshot, expected)) +} +fn run_runner_with_tuning( + image_path: &Utf8Path, + args: &[&str], + runner_bin: &Utf8Path, +) -> Result { + let (snapshot, expected) = plan_tuning()?; let partition_before = partition_state(); // Taken before the runner starts, so the machine is put back even if the @@ -3482,7 +3527,8 @@ fn run_runner_with_tuning( .collect(); if !partition_unrestored.is_empty() { bail!( - "The cpuset partition was not restored: {partition_unrestored:?}.\nstdout: {stdout}\nstderr: {stderr}" + "The cpuset partition was not restored: {partition_unrestored:?}. Now {}.\nstdout: {stdout}\nstderr: {stderr}", + partition_diagnosis() ); } if partition_during.is_empty() { From cc94a36b1458b6ac11fa604b03c997c31c2741dd Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Fri, 31 Jul 2026 04:16:22 +0000 Subject: [PATCH 91/91] Reap what the state directory wipe would strand Pid 7580 was a live process, and the evidence says whose. A killed and reaped process leaves no rmdir race at all, zero failures in forty trials, and a zombie is not even listed in `cgroup.procs`, so a pid listed there is running. The runner is never in a job cgroup: the placement writes `0` from inside the forked child, which is the jailer that becomes the VMM. So a live VMM outlived its job. How it got there is the harness. One scenario cancels the runner with SIGTERM, and `runner run` installs no handler for it, unlike `runner up`: the process dies without unwinding, leaving its VMM alive in its cgroup and its chroot on disk. That is precisely the case the sweep exists for, and the sweep identifies the VMM by the chroot, comparing device and inode against `/proc//root`. Then the harness wipes the state directory before the next scenario and destroys that handle, so every later sweep reads a clean host and the orphan runs on through the rest of the suite. It reached the tuning scenario last of all, where a live task in a descendant made the kernel refuse the parent's cpuset clear with EIO, which is the failure CI reported. The product refuses to remove a chroot whose VMM is alive for exactly this reason. The harness has been doing it once per scenario. It now reaps what the wipe would strand first, killing the VMM and removing the cgroup that shares its name, which is what the sweep would have done had the handle survived. Killing here is the harness's own business; the constraint against it applies to the runner's teardown, which is untouched. The partition diagnosis names each stranded process now rather than printing a bare pid, because what the process is decides whose bug it is, and that cost a round to establish this time. --- tasks/test_runner/src/task/scenarios.rs | 87 +++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 5 deletions(-) diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index fcc191911..74dba09d7 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -305,8 +305,11 @@ fn run_scenario(scenario: &Scenario, runner_bin: &Utf8Path) -> Result<()> { // One state directory for the suite, wiped before each scenario, so jail // assertions see only this scenario's jails and never touch a real runner's - // state. + // state. Reclaimed before the wipe, because the wipe is the one thing the + // runner's own sweep refuses to do. let state_dir = scenario_state_dir(); + reclaim_stranded_jails(&state_dir) + .with_context(|| format!("Failed to reclaim jails stranded before {}", scenario.name))?; drop(fs::remove_dir_all(&state_dir)); // Prepend --sandbox firecracker for sandboxed scenarios. @@ -3357,6 +3360,73 @@ fn partition_state() -> Vec<(Utf8PathBuf, String)> { .collect() } +/// Reap anything a previous scenario left running, before the wipe strands it. +/// +/// A cancelled scenario SIGTERMs `runner run`, which installs no handler for it, +/// so the process dies without unwinding: its VMM stays alive in its cgroup and +/// its chroot stays on disk. That is the case the runner's sweep exists for, and +/// the sweep finds the VMM by the chroot, comparing device and inode against +/// `/proc//root`. Wiping the state directory destroys that handle, so the +/// next runner sweeps a directory that no longer names anything, reports the +/// host clean, and the orphan runs on through every scenario that follows. +/// +/// The product refuses to remove a chroot whose VMM is alive for exactly this +/// reason. The harness has been doing it once per scenario, so it does the +/// reclaiming the sweep would have done rather than leaving a live VMM with +/// nothing pointing at it. +fn reclaim_stranded_jails(state_dir: &Utf8Path) -> Result<()> { + let parent = jail_parent(state_dir); + let entries = match fs::read_dir(&parent) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(e).with_context(|| format!("Failed to read {parent}")), + }; + + for entry in entries { + let entry = entry.with_context(|| format!("Failed to read an entry under {parent}"))?; + if !entry + .file_type() + .with_context(|| format!("Failed to read the kind of an entry under {parent}"))? + .is_dir() + { + continue; + } + let vm_id = entry.file_name().to_string_lossy().into_owned(); + let jail_root = parent.join(&vm_id).join("root"); + + if let Some(pid) = find_jailed_vmm(&jail_root)? { + println!(" reclaiming VMM (pid {pid}) stranded in {vm_id}"); + kill_pid(pid, libc::SIGKILL); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while is_firecracker(pid)? && std::time::Instant::now() < deadline { + std::thread::sleep(PROBE_INTERVAL); + } + anyhow::ensure!( + !is_firecracker(pid)?, + "A VMM stranded in {vm_id} (pid {pid}) would not die, so it would run on through every scenario that follows" + ); + } + + // The cgroup shares the jail's name, and nothing else will come looking + // for it once the directory below is gone. + let cgroup = stale_cgroup(&vm_id); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while cgroup.exists() { + if fs::remove_dir(&cgroup).is_ok() { + println!(" reclaimed the cgroup {cgroup}"); + break; + } + anyhow::ensure!( + std::time::Instant::now() < deadline, + "The cgroup {cgroup} could not be removed, so it would block the cpuset restore of every run that follows" + ); + std::thread::sleep(PROBE_INTERVAL); + } + } + + Ok(()) +} + /// What the `bencher` cgroup looks like, for when the partition assertion fails. /// /// Clearing a parent's `cpuset.cpus` is refused with `EIO` while any task remains @@ -3382,10 +3452,17 @@ fn partition_diagnosis() -> String { .map(|child| { let tasks = fs::read_to_string(root.join(child).join("cgroup.procs")).unwrap_or_default(); - format!( - "{child} holds [{}]", - tasks.split_whitespace().collect::>().join(" ") - ) + // Named, not numbered. A bare pid costs a round trip to identify, + // and what the process is decides whose bug it is. + let named: Vec = tasks + .split_whitespace() + .map(|pid| { + let comm = fs::read_to_string(format!("/proc/{pid}/comm")) + .map_or_else(|_| "gone".to_owned(), |comm| comm.trim().to_owned()); + format!("{pid} ({comm})") + }) + .collect(); + format!("{child} holds [{}]", named.join(" ")) }) .collect(); format!(