diff --git a/.github/workflows/runner.yml b/.github/workflows/runner.yml index bccb999e1..c6bfe04ce 100644 --- a/.github/workflows/runner.yml +++ b/.github/workflows/runner.yml @@ -54,8 +54,20 @@ jobs: run: | echo '{"features": {"containerd-snapshotter": true}}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker + - name: Build scenario binaries + run: cargo test-runner scenarios --build-only - name: Run scenarios - run: cargo test-runner scenarios + # The scenarios run as root. The sandbox is built by dropping + # privilege rather than by starting without it: the jailer creates the + # chroot's device nodes with mknod, chowns the tree to the jail user, + # pivot_roots, and joins a network namespace. The udev rule above makes + # /dev/kvm usable by an unprivileged process, which is necessary but + # not sufficient. + # + # Only this step is elevated. The build above stays as the CI user, so + # cargo's cache and target directory are not left root-owned, and + # BENCHER_RUNNER_BIN keeps cargo out of the elevated step entirely. + run: sudo BENCHER_RUNNER_BIN=./target/debug/runner ./target/debug/test_runner scenarios build_runner: name: Build Runner (${{ matrix.build }}) diff --git a/CLAUDE.md b/CLAUDE.md index 8011903ff..434193b6f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -168,7 +168,10 @@ Defined in `.cargo/config.toml`: - `cargo test-api seed` needs the API server already running (`cargo run` in `services/api`) with a fresh database (`services/api/data` holds only a tracked `.gitignore`; delete `services/api/data/bencher.db`, not the whole directory) - `cargo test-api seed` also needs the `bencher` CLI binary already built (`cargo build --bin bencher`); it shells out via `assert_cmd`, which panics with `` `CARGO_BIN_EXE_bencher` is unset `` if the binary is missing - Pass `--no-git` when running the seed test in this repo: there is no colocated `.git`, so `bencher run` cannot derive a git context and the on-the-fly project naming assertions (`bencher` vs `Project`) will fail without it -- `cargo test-runner` - Runner integration tests (requires Linux + KVM) +- `cargo test-runner` - Runner integration tests (requires Linux + KVM + root) + - `cargo test-runner scenarios` always fails unelevated: the sandbox is built by dropping privilege, so the scenarios refuse to start without it. Build unprivileged, then run elevated, which also keeps `cargo` from leaving root-owned artifacts in `target`: + - `cargo test-runner scenarios --build-only` + - `sudo BENCHER_RUNNER_BIN=./target/debug/runner ./target/debug/test_runner scenarios` ## Git Flow diff --git a/Cargo.lock b/Cargo.lock index 27ac9cb57..ba445fddd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8264,6 +8264,7 @@ dependencies = [ "clap", "libc", "serde_json", + "tempfile", ] [[package]] diff --git a/plus/bencher_runner/Cargo.toml b/plus/bencher_runner/Cargo.toml index 271fe86f5..7be174554 100644 --- a/plus/bencher_runner/Cargo.toml +++ b/plus/bencher_runner/Cargo.toml @@ -60,7 +60,13 @@ libc = { workspace = true } # Linux-only dependencies [target.'cfg(target_os = "linux")'.dependencies] -nix = { workspace = true, features = ["feature", "poll", "sched", "signal"] } +nix = { workspace = true, features = [ + "feature", + "mount", + "poll", + "sched", + "signal", +] } [build-dependencies] flate2 = { workspace = true } diff --git a/plus/bencher_runner/build.rs b/plus/bencher_runner/build.rs index b56816002..bcbb8c4b1 100644 --- a/plus/bencher_runner/build.rs +++ b/plus/bencher_runner/build.rs @@ -1,7 +1,7 @@ //! Build script for `bencher_runner`. //! -//! Bundles the `bencher-init`, `firecracker`, and `vmlinux` binaries -//! for distribution as a single binary. +//! Bundles the `bencher-init`, `firecracker`, `jailer`, and `vmlinux` +//! binaries for distribution as a single binary. //! //! In release builds, binaries are embedded via `include_bytes!`. //! In debug builds, they are downloaded/cached locally and loaded from disk at runtime. @@ -20,9 +20,10 @@ //! //! # Environment Variable Overrides //! -//! - `BENCHER_INIT_PATH` — path to a pre-built bencher-init binary -//! - `BENCHER_FIRECRACKER_PATH` — path to a pre-built firecracker binary -//! - `BENCHER_KERNEL_PATH` — path to a pre-built vmlinux kernel +//! - `BENCHER_INIT_PATH`: path to a pre-built bencher-init binary +//! - `BENCHER_FIRECRACKER_PATH`: path to a pre-built firecracker binary +//! - `BENCHER_JAILER_PATH`: path to a pre-built jailer binary +//! - `BENCHER_KERNEL_PATH`: path to a pre-built vmlinux kernel #![expect( clippy::expect_used, @@ -103,16 +104,26 @@ fn main() { generate_stub_module("init", &out_dir); } - // --- firecracker --- - let firecracker_path = find_or_download_firecracker(&out_dir); - if is_release { - let firecracker_path = firecracker_path.unwrap_or_else(|| panic!("firecracker binary not found. Set BENCHER_FIRECRACKER_PATH or ensure download succeeds.")); - generate_binary_module("firecracker", &firecracker_path, is_release, &out_dir); - } else if let Some(firecracker_path) = firecracker_path { - generate_binary_module("firecracker", &firecracker_path, is_release, &out_dir); - } else { - eprintln!("WARNING: firecracker not found, generating stub module for debug build"); - generate_stub_module("firecracker", &out_dir); + // --- firecracker and jailer --- + // Both ship in the same release archive, so a single download under a + // single hash check yields both. Bundling them together also keeps the + // VMM and its jailer at the same version across a runner self-update. + let (firecracker_path, jailer_path) = find_or_download_firecracker_release(&out_dir); + for (name, path) in [("firecracker", firecracker_path), ("jailer", jailer_path)] { + if is_release { + let path = path.unwrap_or_else(|| { + panic!( + "{name} binary not found. Set BENCHER_{}_PATH or ensure download succeeds.", + name.to_uppercase() + ) + }); + generate_binary_module(name, &path, is_release, &out_dir); + } else if let Some(path) = path { + generate_binary_module(name, &path, is_release, &out_dir); + } else { + eprintln!("WARNING: {name} not found, generating stub module for debug build"); + generate_stub_module(name, &out_dir); + } } // --- kernel (vmlinux) --- @@ -132,6 +143,7 @@ fn main() { println!("cargo:rerun-if-changed=build.rs"); println!("cargo:rerun-if-env-changed=BENCHER_INIT_PATH"); println!("cargo:rerun-if-env-changed=BENCHER_FIRECRACKER_PATH"); + println!("cargo:rerun-if-env-changed=BENCHER_JAILER_PATH"); println!("cargo:rerun-if-env-changed=BENCHER_KERNEL_PATH"); println!("cargo:rerun-if-env-changed=PROFILE"); } @@ -200,70 +212,88 @@ fn find_init_binary() -> Option { None } -/// Find or download the Firecracker binary. +/// Find or download the `firecracker` and `jailer` binaries. /// -/// Checks `BENCHER_FIRECRACKER_PATH` env var first, then tries to download -/// the `.tgz` release archive from GitHub and extract the binary to `OUT_DIR`. -fn find_or_download_firecracker(out_dir: &Path) -> Option { - // 1. Check explicit env var - if let Ok(path) = env::var("BENCHER_FIRECRACKER_PATH") { - let path = PathBuf::from(path); - if path.exists() { - eprintln!( - "Using firecracker from BENCHER_FIRECRACKER_PATH: {}", - path.display() - ); - return Some(path); - } - eprintln!( - "WARNING: BENCHER_FIRECRACKER_PATH set but file not found: {}", - path.display() - ); - } +/// Checks the `BENCHER_FIRECRACKER_PATH` and `BENCHER_JAILER_PATH` env vars +/// first, then downloads the `.tgz` release archive from GitHub once and +/// extracts whichever binaries are still missing into `OUT_DIR`. +/// +/// Returns `(firecracker, jailer)`. +fn find_or_download_firecracker_release(out_dir: &Path) -> (Option, Option) { + let firecracker_override = binary_path_override("firecracker", "BENCHER_FIRECRACKER_PATH"); + let jailer_override = binary_path_override("jailer", "BENCHER_JAILER_PATH"); - // 2. Download from GitHub releases let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); let arch = match target_arch.as_str() { "x86_64" => "x86_64", "aarch64" => "aarch64", _ => { - eprintln!("Unsupported architecture for firecracker: {target_arch}"); - return None; + eprintln!("Unsupported architecture for the Firecracker release: {target_arch}"); + return (firecracker_override, jailer_override); }, }; - let dest = out_dir.join("firecracker"); - if dest.exists() { - eprintln!("Using cached firecracker at: {}", dest.display()); - return Some(dest); - } - - let url = format!( - "https://github.com/firecracker-microvm/firecracker/releases/download/{DEFAULT_FIRECRACKER_VERSION}/firecracker-{DEFAULT_FIRECRACKER_VERSION}-{arch}.tgz", - ); + // The binaries inside the tgz are at: + // release-{version}-{arch}/{name}-{version}-{arch} + let wanted: Vec<(String, PathBuf)> = ["firecracker", "jailer"] + .into_iter() + .zip([&firecracker_override, &jailer_override]) + .filter(|&(_, overridden)| overridden.is_none()) + .map(|(name, _)| { + ( + format!( + "release-{DEFAULT_FIRECRACKER_VERSION}-{arch}/{name}-{DEFAULT_FIRECRACKER_VERSION}-{arch}", + ), + out_dir.join(name), + ) + }) + .filter(|(_, dest)| { + let cached = dest.exists(); + if cached { + eprintln!("Using cached binary at: {}", dest.display()); + } + !cached + }) + .collect(); - // The binary inside the tgz is at: - // release-{version}-{arch}/firecracker-{version}-{arch} - let entry_name = format!( - "release-{DEFAULT_FIRECRACKER_VERSION}-{arch}/firecracker-{DEFAULT_FIRECRACKER_VERSION}-{arch}", - ); + if !wanted.is_empty() { + let url = format!( + "https://github.com/firecracker-microvm/firecracker/releases/download/{DEFAULT_FIRECRACKER_VERSION}/firecracker-{DEFAULT_FIRECRACKER_VERSION}-{arch}.tgz", + ); + let expected_hash = match arch { + "x86_64" => FIRECRACKER_TGZ_SHA256_X86_64, + "aarch64" => FIRECRACKER_TGZ_SHA256_AARCH64, + _ => unreachable!(), + }; + + eprintln!("Downloading the Firecracker release from: {url}"); + if let Err(e) = download_and_extract_tgz(&url, &wanted, Some(expected_hash)) { + eprintln!("WARNING: Failed to download/extract the Firecracker release: {e}"); + } + } - let expected_hash = match arch { - "x86_64" => FIRECRACKER_TGZ_SHA256_X86_64, - "aarch64" => FIRECRACKER_TGZ_SHA256_AARCH64, - _ => unreachable!(), + let resolved = |overridden: Option, name: &str| { + overridden.or_else(|| { + let dest = out_dir.join(name); + dest.exists().then_some(dest) + }) }; - eprintln!("Downloading firecracker from: {url}"); - match download_and_extract_tgz(&url, &entry_name, &dest, Some(expected_hash)) { - Ok(()) => { - eprintln!("Extracted firecracker to: {}", dest.display()); - Some(dest) - }, - Err(e) => { - eprintln!("WARNING: Failed to download/extract firecracker: {e}"); - None - }, + ( + resolved(firecracker_override, "firecracker"), + resolved(jailer_override, "jailer"), + ) +} + +/// Resolve a build-time binary path override from an env var. +fn binary_path_override(name: &str, var: &str) -> Option { + let path = PathBuf::from(env::var(var).ok()?); + if path.exists() { + eprintln!("Using {name} from {var}: {}", path.display()); + Some(path) + } else { + eprintln!("WARNING: {var} set but file not found: {}", path.display()); + None } } @@ -351,18 +381,17 @@ fn download_file(url: &str, dest: &Path, expected_sha256: Option<&str>) -> Resul Ok(()) } -/// Download a `.tgz` archive and extract a single file from it. +/// Download a `.tgz` archive and extract the requested files from it. /// /// # Arguments /// /// * `url` - URL of the `.tgz` archive -/// * `entry_name` - Path of the entry to extract (e.g., `release-v1.15.1-x86_64/firecracker-v1.15.1-x86_64`) -/// * `dest` - Destination path for the extracted file +/// * `wanted` - `(entry_name, dest)` pairs, where `entry_name` is the path of +/// the entry inside the archive (e.g., `release-v1.15.1-x86_64/firecracker-v1.15.1-x86_64`) /// * `expected_sha256` - If `Some`, verify the archive's SHA256 before extracting fn download_and_extract_tgz( url: &str, - entry_name: &str, - dest: &Path, + wanted: &[(String, PathBuf)], expected_sha256: Option<&str>, ) -> Result<(), String> { let response = ureq::get(url) @@ -392,6 +421,13 @@ fn download_and_extract_tgz( let gz = flate2::read::GzDecoder::new(archive_bytes.as_slice()); let mut archive = tar::Archive::new(gz); + // The names still to be found, not a count of them. A tar archive may + // legally carry the same path more than once, and counting extractions down + // gets that wrong in both directions: a third match against two wanted + // entries underflows and panics with a message that says nothing about the + // archive, and two matches on the same entry reach zero and report success + // while the other is still missing. + let mut outstanding: Vec<&str> = wanted.iter().map(|(name, _)| name.as_str()).collect(); for entry in archive .entries() .map_err(|e| format!("Failed to read tar entries: {e}"))? @@ -400,19 +436,32 @@ fn download_and_extract_tgz( let path = entry .path() .map_err(|e| format!("Failed to read entry path: {e}"))?; - - if path.to_string_lossy() == entry_name { - let mut bytes = Vec::new(); - entry - .read_to_end(&mut bytes) - .map_err(|e| format!("Failed to read entry data: {e}"))?; - fs::write(dest, &bytes) - .map_err(|e| format!("Failed to write to {}: {e}", dest.display()))?; + let path = path.to_string_lossy().into_owned(); + + let Some((_, dest)) = wanted.iter().find(|(name, _)| *name == path) else { + continue; + }; + + let mut bytes = Vec::new(); + entry + .read_to_end(&mut bytes) + .map_err(|e| format!("Failed to read entry data: {e}"))?; + fs::write(dest, &bytes) + .map_err(|e| format!("Failed to write to {}: {e}", dest.display()))?; + eprintln!("Extracted '{path}' to: {}", dest.display()); + outstanding.retain(|name| *name != path); + if outstanding.is_empty() { return Ok(()); } } - Err(format!("Entry '{entry_name}' not found in archive")) + let missing = wanted + .iter() + .filter(|(_, dest)| !dest.exists()) + .map(|(name, _)| name.as_str()) + .collect::>() + .join(", "); + Err(format!("Entries not found in archive: {missing}")) } // --------------------------------------------------------------------------- @@ -489,7 +538,7 @@ pub const {name_upper}_BUNDLED: bool = true; fn generate_stub_modules() { let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set")); - for name in &["init", "firecracker", "kernel"] { + for name in &["init", "firecracker", "jailer", "kernel"] { generate_stub_module(name, &out_dir); } } diff --git a/plus/bencher_runner/src/config.rs b/plus/bencher_runner/src/config.rs index 9a445747a..1e2614755 100644 --- a/plus/bencher_runner/src/config.rs +++ b/plus/bencher_runner/src/config.rs @@ -151,6 +151,19 @@ pub struct Config { /// directly on the host (non-sandboxed mode, any platform). #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox: Option, + + /// The runner's persistent state directory. + /// + /// The jail chroot for each sandboxed run is built under this directory. + /// This field is not serialized. + #[serde(skip, default = "default_state_dir")] + pub state_dir: Utf8PathBuf, + + /// The unprivileged uid and gid the jailed VMM drops to. + /// + /// This field is not serialized. + #[serde(skip)] + pub jail_user: crate::jail::JailUser, } fn default_vcpus() -> Cpu { @@ -222,6 +235,10 @@ fn default_grace_period() -> GracePeriod { GracePeriod::MIN } +fn default_state_dir() -> Utf8PathBuf { + Utf8PathBuf::from(crate::jail::DEFAULT_STATE_DIR) +} + impl Config { /// Create a new configuration with the bundled kernel. /// @@ -255,6 +272,8 @@ impl Config { cpu_layout: None, sandbox_log_level: SandboxLogLevel::default(), sandbox: None, + state_dir: default_state_dir(), + jail_user: crate::jail::JailUser::default(), } } @@ -445,6 +464,20 @@ impl Config { self } + /// Set the runner's persistent state directory. + #[must_use] + pub fn with_state_dir(mut self, state_dir: Utf8PathBuf) -> Self { + self.state_dir = state_dir; + self + } + + /// Set the unprivileged uid and gid the jailed VMM drops to. + #[must_use] + pub fn with_jail_user(mut self, jail_user: crate::jail::JailUser) -> Self { + self.jail_user = jail_user; + self + } + /// Set the CPU layout for core isolation. /// /// When set, the Firecracker process will be pinned to benchmark cores diff --git a/plus/bencher_runner/src/cpu.rs b/plus/bencher_runner/src/cpu.rs index d8dae6a7f..5c4f8607e 100644 --- a/plus/bencher_runner/src/cpu.rs +++ b/plus/bencher_runner/src/cpu.rs @@ -17,6 +17,28 @@ use std::fs; #[cfg(target_os = "linux")] use std::io; +/// The kernel's list of online CPU IDs. +#[cfg(target_os = "linux")] +const ONLINE_CPUS: &str = "/sys/devices/system/cpu/online"; + +/// Say that the CPU layout is a guess rather than the host's online set. +/// +/// The count-based fallback numbers cores `0..n`. That is wrong on a host whose +/// online set has gaps, which is the ordinary result of disabling SMT on a +/// topology with interleaved sibling numbering: the online set reads `0,2,4,6` +/// and a counted layout pins the benchmark to cores that are offline. Nothing +/// downstream can tell the difference, so the one place that knows says so. +#[cfg(target_os = "linux")] +#[expect( + clippy::print_stderr, + reason = "a CPU layout the runner had to guess is announced" +)] +fn warn_guessed_layout(reason: &str) { + eprintln!( + "Warning: {reason}. The CPU layout falls back to a core count, which assumes cores are numbered 0..n; on a host whose online set has gaps the benchmark cores may be wrong." + ); +} + /// CPU layout for the runner. /// /// Partitions available cores into housekeeping (for heartbeat, networking) @@ -38,14 +60,18 @@ impl CpuLayout { /// interleaved sibling numbering (common on AMD) leaves a non-contiguous /// online set like `0,2,4,6`, and a count-based layout would pin to /// offline cores. + /// A fallback that cannot read the online set is announced rather than taken + /// quietly: it is a layout the runner guessed, and every cpuset written from + /// it claims cores nobody confirmed are online. #[must_use] pub fn detect() -> Self { #[cfg(target_os = "linux")] - if let Ok(online) = fs::read_to_string("/sys/devices/system/cpu/online") - && let Some(ids) = parse_cpu_id_list(&online) - && !ids.is_empty() - { - return Self::with_cpu_ids(ids); + match fs::read_to_string(ONLINE_CPUS) { + Ok(online) => match parse_cpu_id_list(&online).filter(|ids| !ids.is_empty()) { + Some(ids) => return Self::with_cpu_ids(ids), + None => warn_guessed_layout(&format!("{ONLINE_CPUS} reads as '{}'", online.trim())), + }, + Err(e) => warn_guessed_layout(&format!("{ONLINE_CPUS} could not be read: {e}")), } Self::with_core_count(Self::available_cores()) @@ -489,6 +515,26 @@ mod tests { assert_eq!(format_cpumask(&[64, 1]), "1,00000000,00000002"); } + #[test] + fn two_cores_is_enough_for_isolation() { + // The scenario suite runs on two-vCPU hosted runners, and the runner + // only builds a cgroup when the layout offers isolation. If this ever + // stopped holding, cgroup placement would silently go unexercised in + // CI, which is the whole reason the scenario asserts it. + let layout = CpuLayout::with_cpu_ids(vec![0, 1]); + + assert!(layout.has_isolation()); + assert_eq!(layout.housekeeping, vec![0]); + assert_eq!(layout.benchmark, vec![1]); + } + + #[test] + fn one_core_offers_no_isolation() { + let layout = CpuLayout::with_cpu_ids(vec![0]); + + assert!(!layout.has_isolation()); + } + #[test] fn benchmark_cpuset_string() { let layout = CpuLayout::with_core_count(8); diff --git a/plus/bencher_runner/src/error.rs b/plus/bencher_runner/src/error.rs index 94072dc32..a8cb6f418 100644 --- a/plus/bencher_runner/src/error.rs +++ b/plus/bencher_runner/src/error.rs @@ -58,6 +58,201 @@ pub enum JailError { #[error("Cpuset partition mode '{mode}' rejected by the kernel: {state}")] PartitionInvalid { mode: String, state: String }, + + #[error( + "The jail {field} must not be 0: the sandbox is built by dropping privilege, so a jail user of root is no jail at all" + )] + PrivilegedJailUser { field: &'static str }, + + #[error( + "The state directory {path} must be an absolute path. It reaches the jailer as --chroot-base-dir, which the jailer resolves against its own working directory rather than the runner's, so a relative path builds the chroot somewhere the runner does not look." + )] + RelativeStateDir { path: Utf8PathBuf }, + + #[error( + "The state directory {path} already exists, is not empty, and was not created by the runner. Point --state-dir at a directory the runner owns, or at a subdirectory of this one." + )] + ForeignStateDir { path: Utf8PathBuf }, + + #[error("Failed to create runner state directory {path}: {source}")] + CreateStateDir { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[error( + "Failed to read the runner state directory {path}: {source}. The runner cannot tell whether this directory is its own, and it will not tighten the permissions of one that might not be." + )] + ReadStateDir { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[error( + "Failed to read the jail directory {path}: {source}. The sweep cannot tell what a previous runner left behind, and a directory it could not read is not an empty one." + )] + ReadJailParent { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to create network namespace directory {path}: {source}")] + NetnsDir { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to create network namespace handle {path}: {source}")] + NetnsHandle { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to unshare the network namespace: {0}")] + Unshare(#[source] nix::Error), + + #[cfg(target_os = "linux")] + #[error("Failed to bind the network namespace handle {path}: {source}")] + BindNetns { + path: Utf8PathBuf, + source: nix::Error, + }, + + #[cfg(target_os = "linux")] + #[error("The network namespace thread panicked")] + NetnsThread, + + #[cfg(target_os = "linux")] + #[error( + "A Runner executing sandboxed Jobs must run as root, but this one is running as uid {euid}. \ + Building the sandbox needs privileges that enabling KVM does not grant: mknod for the chroot's /dev/kvm, \ + chown to hand the guest images to the sandbox user, pivot_root, and setns to join a network namespace. \ + A world-readable /dev/kvm is enough to use KVM unprivileged but not to build the sandbox around it. \ + Run as root, or give up the sandbox: start `runner up` with --danger-allow-no-sandbox and assign it \ + only Specs with no Sandbox, or invoke `runner run` without --sandbox. Either way what is given up is \ + the microVM itself and not just its confinement, so the Job executes directly on the host." + )] + NotRoot { euid: u32 }, + + #[cfg(target_os = "linux")] + #[error("Failed to open the jail lock {path}: {source}")] + OpenJailLock { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to take the jail lock {path}: {source}")] + JailLock { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to open the network namespace lock {path}: {source}")] + OpenNetnsLock { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to take the network namespace lock {path}: {source}")] + NetnsLock { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error( + "The network namespace handle {path} is not a namespace distinct from the runner's own" + )] + NetnsNotDistinct { path: Utf8PathBuf }, + + #[cfg(target_os = "linux")] + #[error( + "A stale jail at {path} still has VMM pid {pid} running in it. It is executing untrusted guest code on the benchmark cores, so any measurement taken now is contended. The jail is left in place until it can be reaped." + )] + JailStillRunning { path: Utf8PathBuf, pid: u32 }, + + #[cfg(target_os = "linux")] + #[error( + "A stale jail at {path} could not be examined, so whether a VMM is still running in it is unknown. It is left in place, and a jail that cannot be checked is not a jail that has been cleared." + )] + JailUnexaminable { path: Utf8PathBuf }, + + #[cfg(target_os = "linux")] + #[error( + "The kernel narrowed the cgroup cpuset at {path}: asked for cpus {requested}, got {effective}. The benchmark would not have run on the cores it claims." + )] + CpusetNarrowed { + path: Utf8PathBuf, + requested: String, + effective: String, + }, + + #[cfg(target_os = "linux")] + #[error( + "Failed to remove the stale cgroup {path}: {source}. Stale cgroups accumulate under the parent, and one that cannot be removed usually means something is still running in it." + )] + StaleCgroup { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to open the jail chroot {path}: {source}")] + OpenJailRoot { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error( + "The socket path {path} is {length} bytes, over the {limit} byte sun_path limit for a Unix domain socket" + )] + SocketPathTooLong { + path: String, + length: usize, + limit: usize, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to create jail chroot {path}: {source}")] + CreateJail { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to make {path} readable by the jailed VMM: {source}")] + ChmodJail { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to hand {path} to the jail uid and gid: {source}")] + ChownJail { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to open {path} for cgroup placement: {source}")] + OpenCgroupProcs { + path: Utf8PathBuf, + source: std::io::Error, + }, + + #[cfg(target_os = "linux")] + #[error("Failed to read cgroup file {path}: {source}")] + ReadCgroup { + path: Utf8PathBuf, + source: std::io::Error, + }, } #[derive(Debug, Error)] @@ -77,6 +272,13 @@ pub enum ConfigError { #[error("Binary not found: {name}. {hint}")] BinaryNotFound { name: String, hint: String }, + #[error("Failed to copy {src} to {dest}: {source}")] + CopyFile { + src: Utf8PathBuf, + dest: Utf8PathBuf, + source: std::io::Error, + }, + #[error("Failed to copy init binary from {src} to {dest}: {source}")] CopyInit { src: Utf8PathBuf, diff --git a/plus/bencher_runner/src/firecracker/client.rs b/plus/bencher_runner/src/firecracker/client.rs index e98590e83..19c214364 100644 --- a/plus/bencher_runner/src/firecracker/client.rs +++ b/plus/bencher_runner/src/firecracker/client.rs @@ -9,57 +9,69 @@ use std::io::{Read as _, Write as _}; use std::os::unix::net::UnixStream; use std::time::Duration; -use camino::Utf8Path; - use crate::firecracker::config::{Action, BootSource, Drive, MachineConfig, VsockConfig}; use crate::firecracker::error::FirecrackerError; +use crate::jail::SocketPath; /// Client for the Firecracker REST API. pub struct FirecrackerClient { - socket_path: String, + /// Held as a [`SocketPath`], not a string. The type is the proof that this + /// path fits `sun_path`, and this is the one place the limit is actually + /// enforced by the kernel, so downgrading it here would discard the + /// guarantee exactly where it is worth having. + socket_path: SocketPath, } impl FirecrackerClient { - /// Create a new client for the given API socket path. - pub fn new(socket_path: &str) -> Self { + /// Create a new client for the API socket. + /// + /// The runner reaches the socket from outside the chroot, so this is the + /// socket view; the jailed VMM binds the chroot view of the same file. + pub fn new(socket_path: &SocketPath) -> Self { Self { - socket_path: socket_path.to_owned(), + socket_path: socket_path.clone(), } } - /// Wait for the Firecracker API socket to become ready. - pub fn wait_for_ready(&self, timeout: Duration) -> Result<(), FirecrackerError> { - let start = std::time::Instant::now(); - let poll_interval = Duration::from_millis(50); - - while start.elapsed() < timeout { - if Utf8Path::new(&self.socket_path).exists() { - // Try to connect - if let Ok(mut stream) = UnixStream::connect(&self.socket_path) { - drop(stream.set_read_timeout(Some(Duration::from_secs(1)))); - drop(stream.set_write_timeout(Some(Duration::from_secs(1)))); - - let request = "GET / HTTP/1.1\r\nHost: localhost\r\nAccept: */*\r\n\r\n"; - if stream.write_all(request.as_bytes()).is_ok() { - let mut buf = [0u8; 256]; - if let Ok(n) = stream.read(&mut buf) - && n > 0 - { - return Ok(()); - } + /// Try the API socket once. + /// + /// `Ok(true)` once Firecracker is answering, `Ok(false)` while it is not + /// listening yet, and an error when the address itself cannot be used. + /// Only failures a not-yet-listening VMM actually produces are worth + /// retrying: an unusable path never becomes usable, and retrying it for a + /// whole timeout turns a precise error into one that points at Firecracker + /// instead of at the cause. An over-long socket path is rejected by the + /// standard library before any syscall, which is exactly that case. + pub fn try_ready(&self) -> Result { + match UnixStream::connect(self.socket_path.as_str()) { + Ok(mut stream) => { + drop(stream.set_read_timeout(Some(Duration::from_secs(1)))); + drop(stream.set_write_timeout(Some(Duration::from_secs(1)))); + + let request = "GET / HTTP/1.1\r\nHost: localhost\r\nAccept: */*\r\n\r\n"; + if stream.write_all(request.as_bytes()).is_ok() { + let mut buf = [0u8; 256]; + if let Ok(n) = stream.read(&mut buf) + && n > 0 + { + return Ok(true); } } - } - std::thread::sleep(poll_interval); + Ok(false) + }, + Err(e) if is_not_listening_yet(&e) => Ok(false), + Err(e) => Err(FirecrackerError::SocketUnusable { + path: self.socket_path.clone(), + source: e, + }), } - - Err(FirecrackerError::SocketNotReady(timeout)) } /// Configure the machine (vCPUs, memory). pub fn put_machine_config(&self, config: &MachineConfig) -> Result<(), FirecrackerError> { - let body = serde_json::to_string(config).map_err(|e| { - FirecrackerError::ProcessStart(format!("serialize machine config: {e}")) + let body = serde_json::to_string(config).map_err(|e| FirecrackerError::ApiEncoding { + context: "serialize machine config", + source: e, })?; let (status, response_body) = self.http_put("/machine-config", &body)?; if status >= 300 { @@ -73,8 +85,10 @@ impl FirecrackerClient { /// Configure the boot source (kernel and boot args). pub fn put_boot_source(&self, config: &BootSource) -> Result<(), FirecrackerError> { - let body = serde_json::to_string(config) - .map_err(|e| FirecrackerError::ProcessStart(format!("serialize boot source: {e}")))?; + let body = serde_json::to_string(config).map_err(|e| FirecrackerError::ApiEncoding { + context: "serialize boot source", + source: e, + })?; let (status, response_body) = self.http_put("/boot-source", &body)?; if status >= 300 { return Err(FirecrackerError::Api { @@ -87,8 +101,10 @@ impl FirecrackerClient { /// Configure a block device (drive). pub fn put_drive(&self, config: &Drive) -> Result<(), FirecrackerError> { - let body = serde_json::to_string(config) - .map_err(|e| FirecrackerError::ProcessStart(format!("serialize drive: {e}")))?; + let body = serde_json::to_string(config).map_err(|e| FirecrackerError::ApiEncoding { + context: "serialize drive", + source: e, + })?; let path = format!("/drives/{}", config.drive_id); let (status, response_body) = self.http_put(&path, &body)?; if status >= 300 { @@ -102,8 +118,10 @@ impl FirecrackerClient { /// Configure the vsock device. pub fn put_vsock(&self, config: &VsockConfig) -> Result<(), FirecrackerError> { - let body = serde_json::to_string(config) - .map_err(|e| FirecrackerError::ProcessStart(format!("serialize vsock: {e}")))?; + let body = serde_json::to_string(config).map_err(|e| FirecrackerError::ApiEncoding { + context: "serialize vsock", + source: e, + })?; let (status, response_body) = self.http_put("/vsock", &body)?; if status >= 300 { return Err(FirecrackerError::Api { @@ -116,8 +134,10 @@ impl FirecrackerClient { /// Perform a VM action (start, shutdown, etc.). pub fn put_action(&self, action: &Action) -> Result<(), FirecrackerError> { - let body = serde_json::to_string(action) - .map_err(|e| FirecrackerError::ProcessStart(format!("serialize action: {e}")))?; + let body = serde_json::to_string(action).map_err(|e| FirecrackerError::ApiEncoding { + context: "serialize action", + source: e, + })?; let (status, response_body) = self.http_put("/actions", &body)?; if status >= 300 { return Err(FirecrackerError::Api { @@ -132,7 +152,7 @@ impl FirecrackerClient { /// /// Returns the HTTP status code and response body. fn http_put(&self, path: &str, json_body: &str) -> Result<(u16, String), FirecrackerError> { - let mut stream = UnixStream::connect(&self.socket_path)?; + let mut stream = UnixStream::connect(self.socket_path.as_str())?; stream.set_read_timeout(Some(Duration::from_secs(5)))?; stream.set_write_timeout(Some(Duration::from_secs(5)))?; @@ -200,6 +220,21 @@ impl FirecrackerClient { } } +/// Whether an error means the VMM has simply not started listening yet. +/// +/// The socket file not existing, or existing with nothing accepting on it, is +/// the normal state during boot. Every other error describes the address +/// itself and will not change by waiting. +fn is_not_listening_yet(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::NotFound + | std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::WouldBlock + | std::io::ErrorKind::Interrupted + ) +} + /// Check if we have received a complete HTTP response. fn response_complete(data: &[u8]) -> bool { let header_end = find_header_end(data); @@ -238,13 +273,18 @@ fn parse_http_response(data: &[u8]) -> Result<(u16, String), FirecrackerError> { let status_line = response .lines() .next() - .ok_or_else(|| FirecrackerError::ProcessStart("empty HTTP response".to_owned()))?; + .ok_or(FirecrackerError::MalformedResponse("empty HTTP response"))?; + // No invented status. A status line the runner could not parse is a + // malformed response, and reporting 500 would attribute a server error to + // Firecracker that Firecracker never sent. let status_code: u16 = status_line .split_whitespace() .nth(1) - .and_then(|s| s.parse().ok()) - .unwrap_or(500); + .and_then(|code| code.parse().ok()) + .ok_or(FirecrackerError::MalformedResponse( + "HTTP status line carries no status code", + ))?; // Extract body (after \r\n\r\n) let body = response @@ -348,18 +388,31 @@ mod tests { } #[test] - fn parse_http_malformed_status_defaults_to_500() { - // No status code in the status line + fn a_status_line_without_a_status_is_malformed_not_a_500() { + // These two asserted a default of 500 until the rule was written down. + // A response the runner could not parse is a malformed response; calling + // it a 500 attributes a server error to Firecracker that Firecracker + // never sent, and sends whoever reads the log looking at the VMM. let data = b"HTTP/1.1\r\n\r\n"; - let (status, _) = parse_http_response(data).unwrap(); - assert_eq!(status, 500); + + let err = parse_http_response(data).unwrap_err(); + + assert!( + matches!(err, FirecrackerError::MalformedResponse(_)), + "a response that could not be parsed is not a status: {err}" + ); } #[test] - fn parse_http_non_numeric_status_defaults_to_500() { + fn a_status_that_is_not_a_number_is_malformed_not_a_500() { let data = b"HTTP/1.1 abc OK\r\n\r\n"; - let (status, _) = parse_http_response(data).unwrap(); - assert_eq!(status, 500); + + let err = parse_http_response(data).unwrap_err(); + + assert!( + matches!(err, FirecrackerError::MalformedResponse(_)), + "{err}" + ); } #[test] diff --git a/plus/bencher_runner/src/firecracker/config.rs b/plus/bencher_runner/src/firecracker/config.rs index 8e5a92f00..cfa6c75e9 100644 --- a/plus/bencher_runner/src/firecracker/config.rs +++ b/plus/bencher_runner/src/firecracker/config.rs @@ -2,6 +2,8 @@ use serde::Serialize; +use crate::jail::ChrootPath; + /// Machine configuration for Firecracker. #[derive(Debug, Serialize)] pub struct MachineConfig { @@ -16,8 +18,8 @@ pub struct MachineConfig { /// Boot source configuration. #[derive(Debug, Serialize)] pub struct BootSource { - /// Path to the kernel image on the host. - pub kernel_image_path: String, + /// Path to the kernel image, as the jailed VMM sees it. + pub kernel_image_path: ChrootPath, /// Kernel boot arguments. pub boot_args: String, } @@ -27,8 +29,8 @@ pub struct BootSource { pub struct Drive { /// Unique drive identifier. pub drive_id: String, - /// Path to the disk image on the host. - pub path_on_host: String, + /// Path to the disk image, as the jailed VMM sees it. + pub path_on_host: ChrootPath, /// Whether this is the root device. pub is_root_device: bool, /// Whether the drive is read-only. @@ -40,8 +42,8 @@ pub struct Drive { pub struct VsockConfig { /// Guest CID (must be >= 3 for Firecracker). pub guest_cid: u32, - /// Path to the Unix domain socket on the host. - pub uds_path: String, + /// Path to the Unix domain socket, as the jailed VMM sees it. + pub uds_path: ChrootPath, } /// VM action request. diff --git a/plus/bencher_runner/src/firecracker/error.rs b/plus/bencher_runner/src/firecracker/error.rs index 8801884a5..c9c6bf41d 100644 --- a/plus/bencher_runner/src/firecracker/error.rs +++ b/plus/bencher_runner/src/firecracker/error.rs @@ -5,9 +5,31 @@ use thiserror::Error; /// Errors from the Firecracker integration. #[derive(Debug, Error)] pub enum FirecrackerError { - /// Failed to start the Firecracker process. - #[error("Failed to start Firecracker process: {0}")] - ProcessStart(String), + /// Failed to spawn the jailer that starts the Firecracker process. + #[error("Failed to spawn {path}: {source}")] + Spawn { + /// The binary that could not be spawned. + path: camino::Utf8PathBuf, + /// Why it could not be spawned. + source: std::io::Error, + }, + + /// The spawned process did not provide the stdio the runner asked for. + #[error("Firecracker process stdio unavailable: {0}")] + Stdio(&'static str), + + /// A Firecracker API request or response could not be handled. + #[error("Firecracker API {context}: {source}")] + ApiEncoding { + /// What was being encoded or decoded. + context: &'static str, + /// The underlying serialization failure. + source: serde_json::Error, + }, + + /// The Firecracker API returned a response that could not be parsed. + #[error("Firecracker API response malformed: {0}")] + MalformedResponse(&'static str), /// Firecracker API returned an error. #[error("Firecracker API error: {status} {body}")] @@ -30,6 +52,39 @@ pub enum FirecrackerError { #[error("Firecracker API socket not ready after {0:?}")] SocketNotReady(std::time::Duration), + /// The jailed process exited before Firecracker started serving its API. + /// + /// Named for what the runner actually knows. The jailer `exec`s Firecracker + /// in place, so this one pid is the jailer up to that moment and Firecracker + /// afterwards, and the runner cannot tell from the outside which of the two + /// died. Blaming the jailer by name was right about half the time. + /// + /// Distinct from a timeout: the process is gone, so waiting cannot help, + /// and the reason is on stderr under the `[firecracker]` prefix. + #[error( + "The jailed process exited ({status}) before the Firecracker API socket appeared. The jailer execs Firecracker in place, so this is the jailer or the VMM it became; its diagnostics are above, prefixed [firecracker]" + )] + JailedProcessExited { + /// How it exited. + status: std::process::ExitStatus, + }, + + /// The API socket address itself cannot be used. + /// + /// Distinct from [`Self::SocketNotReady`]: waiting will not help, so the + /// error names the path and the cause instead of a timeout. + #[error("Firecracker API socket {path} is unusable: {source}")] + SocketUnusable { + /// The socket path the runner tried to reach. + /// + /// The checked type, not a string: every value has been measured + /// against `sun_path`, which is the limit this error is most often + /// about. + path: crate::jail::SocketPath, + /// Why it could not be reached. + source: std::io::Error, + }, + /// Failed to collect results via vsock. #[error("Vsock result collection failed: {0}")] VsockCollection(String), @@ -37,4 +92,31 @@ pub enum FirecrackerError { /// Job was cancelled. #[error("Job cancelled")] Cancelled, + + /// The VMM could not be placed in, or verified against, its cgroup. + #[error("Cgroup placement failed: {0}")] + CgroupPlacement(#[source] crate::error::JailError), + + /// The cgroup exists but the VMM is not in it. + /// + /// Fatal: a cgroup that does not contain the VMM is a silent lie about + /// which cores the benchmark ran on. + #[error("Firecracker (pid {pid}) is not in its cgroup {cgroup}")] + CgroupMissingPid { + /// PID of the Firecracker process. + pid: u32, + /// The cgroup it should be in. + cgroup: camino::Utf8PathBuf, + }, + + /// The cgroup exists but its cpuset could not be applied. + /// + /// Boxed because [`crate::error::RunnerError`] contains this type, and it + /// in turn contains a `RunnerError`. + #[error("Failed to confine Firecracker to the benchmark cores: {0}")] + CpusetFailed(#[source] Box), + + /// A jail artifact could not be handed to the jail uid and gid. + #[error("Jail ownership failed: {0}")] + Chown(#[source] crate::error::JailError), } diff --git a/plus/bencher_runner/src/firecracker/mod.rs b/plus/bencher_runner/src/firecracker/mod.rs index fdbdee111..4aaec51d9 100644 --- a/plus/bencher_runner/src/firecracker/mod.rs +++ b/plus/bencher_runner/src/firecracker/mod.rs @@ -3,6 +3,10 @@ //! This module manages Firecracker microVMs for running benchmarks in isolation. //! Instead of a custom VMM, we use Firecracker as an external process controlled //! via its REST API over a Unix domain socket. +//! +//! The VMM is never a plain child of the runner. It runs under the Firecracker +//! jailer, in a chroot as an unprivileged user with no host network, and is +//! placed in its cgroup before it execs. See [`crate::jail`]. #![expect( clippy::print_stdout, @@ -28,6 +32,7 @@ use std::time::{Duration, Instant}; use camino::Utf8PathBuf; use crate::cpu::CpuLayout; +use crate::jail::{CgroupManager, Cpuset, JailPaths, JailSignals, JailUser, VmId}; use crate::metrics::{self, RunMetrics}; pub use error::FirecrackerError; @@ -46,18 +51,32 @@ const GUEST_CID: u32 = 3; use crate::run::RunOutput; use config::{Action, ActionType, BootSource, Drive, MachineConfig, VsockConfig}; -use process::FirecrackerProcess; +use process::{FirecrackerProcess, JailedSpawn}; use vsock::VsockListener; /// Configuration for a Firecracker-based benchmark run. #[derive(Debug)] pub struct FirecrackerJobConfig { - /// Path to the Firecracker binary. + /// Path to the staged Firecracker binary, outside the jail. pub firecracker_bin: Utf8PathBuf, - /// Path to the kernel image. - pub kernel_path: Utf8PathBuf, - /// Path to the ext4 rootfs image. - pub rootfs_path: Utf8PathBuf, + /// Path to the jailer binary. + pub jailer_bin: Utf8PathBuf, + /// Identity of this microVM: the jailer id, the chroot name, and the + /// cgroup name. Minted before the job's artifacts, because the jail root + /// they are built in is a function of it. + pub vm_id: VmId, + /// Both views of every file inside the jail chroot. + pub jail: JailPaths, + /// The unprivileged uid and gid the VMM drops to. + pub jail_user: JailUser, + /// The jailer's `--chroot-base-dir`. + pub chroot_base_dir: Utf8PathBuf, + /// Handle of the empty network namespace the VMM joins. + pub netns: Utf8PathBuf, + /// Shared with the chroot guard of the same id: a cgroup this job cannot + /// remove has to hold that chroot, which is the only handle a later sweep + /// has for finding the cgroup again. + pub signals: JailSignals, /// Number of vCPUs. pub vcpus: u8, /// Memory size in MiB. @@ -66,8 +85,6 @@ pub struct FirecrackerJobConfig { pub boot_args: String, /// Execution timeout in seconds. pub timeout_secs: u64, - /// Working directory for temporary files (API socket, vsock UDS). - pub work_dir: Utf8PathBuf, /// Optional CPU layout for core isolation via cpuset. pub cpu_layout: Option, /// Firecracker process log level. @@ -82,16 +99,24 @@ pub struct FirecrackerJobConfig { pub grace_period: bencher_json::GracePeriod, } -/// Run a benchmark inside a Firecracker microVM. +/// Run a benchmark inside a jailed Firecracker microVM. /// /// This function: /// 1. Optionally creates a cgroup with cpuset for CPU isolation -/// 2. Starts a Firecracker process (and moves it into the cgroup) -/// 3. Configures the VM via REST API -/// 4. Creates vsock listeners for result collection -/// 5. Boots the VM -/// 6. Collects results via vsock -/// 7. Cleans up (including cgroup) +/// 2. Starts Firecracker under the jailer, placed in the cgroup before exec +/// 3. Verifies the placement landed +/// 4. Configures the VM via REST API +/// 5. Creates vsock listeners for result collection and hands them to the jail +/// 6. Boots the VM +/// 7. Collects results via vsock +/// 8. Cleans up (including cgroup) +/// +/// The cgroup is a fidelity mechanism, so its absence degrades: no CPU layout, +/// no isolation, or a cgroup that cannot be created means the job proceeds with +/// a warning. Placement and verification are conditional on the cgroup +/// existing, but when it does they are hard requirements: a host that cannot +/// isolate is a declared limitation, while a cgroup that exists but does not +/// contain the VMM is a silent lie about where the benchmark ran. /// /// Returns the benchmark output including exit code and stdout. #[expect( @@ -102,31 +127,53 @@ pub fn run_firecracker( config: &FirecrackerJobConfig, cancel_flag: Option<&Arc>, ) -> Result { - let vm_id = uuid::Uuid::new_v4().to_string(); - let api_socket_path = format!("{}/firecracker-{vm_id}.sock", config.work_dir); - let vsock_uds_path = format!("{}/vsock-{vm_id}.sock", config.work_dir); + let vm_id = &config.vm_id; + let jail = &config.jail; let start_time = Instant::now(); // Step 0: Create cgroup with cpuset if CPU layout is provided let cgroup = if let Some(layout) = &config.cpu_layout { if layout.has_isolation() { - match crate::jail::CgroupManager::new(&vm_id) { + match CgroupManager::new(vm_id, config.signals.clone()) { Ok(cg) => { - // Apply cpuset to pin Firecracker to benchmark cores - if let Err(e) = cg.apply_cpuset(layout) { - eprintln!("Warning: failed to apply cpuset: {e}"); - } else { - println!( - "CPU isolation: Firecracker pinned to cores {}", - layout.benchmark_cpuset() - ); - } - // Keep VM memory resident: swap adds run-to-run variance - if let Err(e) = cg.disable_swap() { - eprintln!("Warning: failed to disable swap for VM cgroup: {e}"); + // A cgroup that exists but does not confine the VMM to the + // benchmark cores would report a number measured somewhere + // other than where it claims, so a rejected cpuset is + // fatal. A controller the host does not delegate is a + // different thing: there is no isolation to be had, which + // is a declared limitation, so the cgroup is dropped and + // the job runs without one exactly as on a host that could + // not create it at all. + match cg + .apply_cpuset(layout) + .map_err(|e| FirecrackerError::CpusetFailed(Box::new(e)))? + { + Cpuset::Applied => { + println!( + "CPU isolation: Firecracker pinned to cores {}", + layout.benchmark_cpuset() + ); + // Keep VM memory resident: swap adds run-to-run variance + if let Err(e) = cg.disable_swap() { + eprintln!("Warning: failed to disable swap for VM cgroup: {e}"); + } + Some(cg) + }, + Cpuset::Unavailable(reason) => { + // Precise about what is lost. The vCPU threads are + // still pinned to the benchmark cores further + // down, which is gated on the layout and not on + // the cgroup, so what goes is the cgroup's hard + // confinement (nothing stops other work being + // scheduled onto those cores) along with its + // metrics and swap control. + eprintln!( + "Warning: this run has no cgroup cpuset ({reason}), so nothing keeps other work off the benchmark cores and its numbers carry more variance; vCPU threads are still pinned to them" + ); + None + }, } - Some(cg) }, Err(e) => { eprintln!("Warning: failed to create cgroup for CPU isolation: {e}"); @@ -140,27 +187,38 @@ pub fn run_firecracker( None }; - // Step 1: Start Firecracker process - println!("Starting Firecracker process..."); + // Step 1: Start the jailed Firecracker process. + // + // The cgroup descriptor is opened before the fork so the placement inside + // `pre_exec` is a bare write on an existing descriptor. Placing the VMM + // before it execs, rather than after it is already running, keeps it from + // booting its API and touching memory on the wrong cores first. + println!("Starting jailed Firecracker process..."); let housekeeping_cores = config .cpu_layout .as_ref() .map(|l| l.housekeeping.clone()) .unwrap_or_default(); - let mut fc_process = FirecrackerProcess::start( - config.firecracker_bin.as_str(), - &api_socket_path, - &vm_id, - config.log_level.as_str(), + let cgroup_procs = placement_target(cgroup.as_ref())?; + let mut fc_process = FirecrackerProcess::start(JailedSpawn { + jailer_bin: &config.jailer_bin, + exec_file: &config.firecracker_bin, + vm_id, + jail_user: config.jail_user, + chroot_base_dir: &config.chroot_base_dir, + netns: &config.netns, + api_socket: jail.api_socket(), + log_level: config.log_level.as_str(), housekeeping_cores, - )?; + cgroup_procs, + })?; - // Move Firecracker process into cgroup for CPU isolation - if let Some(cg) = &cgroup - && let Err(e) = cg.add_pid(fc_process.pid()) - { - eprintln!("Warning: failed to add Firecracker to cgroup: {e}"); - } + // Step 1b: Verify the placement landed. + // + // `spawn` returns only after `pre_exec` and the exec have completed, so + // this read is race free. A failed write already surfaced as a failed + // spawn; this catches a write that succeeded against the wrong cgroup. + verify_placement(cgroup.as_ref(), fc_process.pid())?; let client = fc_process.client(); @@ -173,26 +231,33 @@ pub fn run_firecracker( smt: false, })?; + // Every path in an API body is the chroot view: these resolve inside the + // jail, not on the host filesystem the runner sees. client.put_boot_source(&BootSource { - kernel_image_path: config.kernel_path.to_string(), + kernel_image_path: jail.kernel().chroot().clone(), boot_args: config.boot_args.clone(), })?; client.put_drive(&Drive { drive_id: "rootfs".to_owned(), - path_on_host: config.rootfs_path.to_string(), + path_on_host: jail.rootfs().chroot().clone(), is_root_device: true, is_read_only: false, })?; client.put_vsock(&VsockConfig { guest_cid: GUEST_CID, - uds_path: vsock_uds_path.clone(), + uds_path: jail.vsock().chroot().clone(), })?; // Step 3: Create vsock listeners (must be before boot) println!("Setting up vsock listeners..."); - let vsock_listener = VsockListener::new(&vsock_uds_path)?; + let vsock_listener = VsockListener::new(jail.vsock())?; + // Firecracker connects out to these as the unprivileged jail user, so it + // needs write access to the inodes. After bind and before InstanceStart. + vsock_listener + .chown_to_jail(config.jail_user) + .map_err(FirecrackerError::Chown)?; // Step 4: Boot the VM println!("Booting VM..."); @@ -286,6 +351,44 @@ pub fn run_firecracker( }) } +/// Open the descriptor the VMM will be placed through, if there is a cgroup. +/// +/// Placement is conditional on the cgroup existing, not unconditional: a host +/// with no CPU layout or no isolation gets no cgroup, and therefore no +/// placement, which is the existing degrade behavior. When the cgroup does +/// exist, failing to open it aborts the job. +fn placement_target( + cgroup: Option<&CgroupManager>, +) -> Result, FirecrackerError> { + cgroup + .map(CgroupManager::open_procs) + .transpose() + .map_err(FirecrackerError::CgroupPlacement) +} + +/// Confirm the VMM landed in its cgroup, if there is one. +/// +/// Skipped, not failed, when no cgroup exists. When one does, a pid that is +/// not a member aborts the job: a cgroup that exists but does not contain the +/// VMM is a silent lie about which cores the benchmark ran on, which is worse +/// than a declared absence of isolation. +fn verify_placement(cgroup: Option<&CgroupManager>, pid: u32) -> Result<(), FirecrackerError> { + let Some(cgroup) = cgroup else { + return Ok(()); + }; + let placed = cgroup + .contains_pid(pid) + .map_err(FirecrackerError::CgroupPlacement)?; + if placed { + Ok(()) + } else { + Err(FirecrackerError::CgroupMissingPid { + pid, + cgroup: cgroup.path().to_owned(), + }) + } +} + /// Decode the length-prefixed binary protocol for multiple output files. fn decode_output_files( data: &[u8], @@ -307,6 +410,48 @@ fn parse_exit_code(s: &str) -> i32 { mod tests { use super::*; + // --- placement is conditional on the cgroup existing --- + + #[test] + fn no_cgroup_skips_placement() { + // A host that cannot isolate is a declared limitation, so the job + // proceeds without a cgroup rather than failing. + assert!( + placement_target(None).unwrap().is_none(), + "no cgroup means nothing to place through" + ); + } + + #[test] + fn no_cgroup_skips_verification() { + verify_placement(None, 1).unwrap(); + } + + #[test] + fn a_cgroup_without_the_pid_aborts() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + std::fs::write(root.join("cgroup.procs"), "999\n").unwrap(); + let cgroup = CgroupManager::detached(root); + + let err = verify_placement(Some(&cgroup), 123).unwrap_err(); + + assert!( + matches!(err, FirecrackerError::CgroupMissingPid { pid: 123, .. }), + "a cgroup that does not contain the VMM must abort, got: {err}" + ); + } + + #[test] + fn a_cgroup_holding_the_pid_verifies() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + std::fs::write(root.join("cgroup.procs"), "123\n456\n").unwrap(); + let cgroup = CgroupManager::detached(root); + + verify_placement(Some(&cgroup), 123).unwrap(); + } + #[test] fn parse_exit_code_zero() { assert_eq!(parse_exit_code("0"), 0); diff --git a/plus/bencher_runner/src/firecracker/process.rs b/plus/bencher_runner/src/firecracker/process.rs index 230b7891a..51c6b2365 100644 --- a/plus/bencher_runner/src/firecracker/process.rs +++ b/plus/bencher_runner/src/firecracker/process.rs @@ -1,55 +1,142 @@ -//! Firecracker process management. +//! Jailed Firecracker process management. #![expect(clippy::print_stderr, reason = "process management prints diagnostics")] +use std::fs::File; +use std::os::unix::process::CommandExt as _; use std::process::{Child, Command}; use std::time::Duration; +use camino::Utf8Path; + use crate::firecracker::client::FirecrackerClient; use crate::firecracker::config::{Action, ActionType}; use crate::firecracker::error::FirecrackerError; +use crate::jail::{JailFile, JailUser, VmId}; + +/// How long to wait for the Firecracker API socket to appear. +/// +/// This budget used to cover Firecracker starting up on its own. It now also +/// has to cover the jailer building a chroot, copying a multi-megabyte exec +/// file into it, creating device nodes, chowning, `pivot_root`, and `setns`, +/// on a host that may be busy running someone else's benchmark. Five seconds +/// left no margin for that. +/// +/// Widening costs nothing on the failure path: a jailer that dies is detected +/// the moment it exits rather than at the deadline, so the only thing this +/// affects is how patient the runner is with a slow host. +const API_SOCKET_TIMEOUT: Duration = Duration::from_secs(30); + +/// Everything needed to spawn the VMM under the jailer. +#[derive(Debug)] +pub struct JailedSpawn<'a> { + /// The jailer binary, which runs as root and execs Firecracker in place. + pub jailer_bin: &'a Utf8Path, + /// The staged Firecracker binary, outside the jail. + /// + /// The jailer copies this into the chroot itself, and refuses to write + /// over a multiply linked destination, so it is neither placed in the + /// chroot by hand nor hardlinked there. Its base name determines the + /// chroot layout, so it is fixed rather than incidental. + pub exec_file: &'a Utf8Path, + /// The jailer `--id`, which is also the chroot name and the cgroup name. + pub vm_id: &'a VmId, + /// The unprivileged uid and gid the VMM drops to. + pub jail_user: JailUser, + /// The jailer `--chroot-base-dir`. + pub chroot_base_dir: &'a Utf8Path, + /// Handle of the empty network namespace the VMM joins. + pub netns: &'a Utf8Path, + /// The REST API socket, in both views. + pub api_socket: &'a JailFile, + /// Firecracker process log level. + pub log_level: &'a str, + /// Cores the stderr reader thread is pinned to. + pub housekeeping_cores: Vec, + /// Pre-opened `cgroup.procs`, when a cgroup exists to place the VMM in. + pub cgroup_procs: Option, +} -/// A running Firecracker process. +/// A running, jailed Firecracker process. pub struct FirecrackerProcess { child: Child, - api_socket_path: String, + api_socket: JailFile, stderr_thread: Option>, } impl FirecrackerProcess { - /// Start a new Firecracker process. + /// Start Firecracker under the jailer and wait for its API socket. + /// + /// The jailer builds the chroot, creates `/dev/kvm`, drops to the + /// unprivileged jail uid, joins the empty network namespace, and execs + /// Firecracker. Neither `--daemonize` nor `--new-pid-ns` is passed: + /// both make the jailer fork, which would break the pid identity that + /// [`Self::pid`] and [`Self::kill_after_grace_period`] rely on. Without + /// them the jailer execs in place and the child pid stays the VMM. /// - /// Spawns `firecracker --api-sock --id --level ` - /// and waits for the API socket to become ready. - /// A background thread reads stderr and prints lines prefixed with `[firecracker]`. - pub fn start( - firecracker_bin: &str, - api_socket_path: &str, - vm_id: &str, - log_level: &str, - housekeeping_cores: Vec, - ) -> Result { - // Remove stale socket if it exists - drop(std::fs::remove_file(api_socket_path)); - - let mut child = Command::new(firecracker_bin) - .arg("--api-sock") - .arg(api_socket_path) - .arg("--id") - .arg(vm_id) - .arg("--level") - .arg(log_level) + /// No cgroup flags are passed either. The runner owns the cgroup end to + /// end because it has to create, verify, read metrics from, and remove it, + /// and the cpuset partition specifically needs read-back verification that + /// the jailer's write-once interface cannot provide. + /// + /// A background thread reads stderr and prints lines prefixed with + /// `[firecracker]`. The jailer inherits that stdio and its own diagnostics + /// appear under the same prefix. + pub fn start(spawn: JailedSpawn<'_>) -> Result { + let args = jailer_args(&spawn); + + // Destructured rather than read field by field so that adding a field + // without deciding what it does here is a build error. + let JailedSpawn { + jailer_bin, + exec_file: _, + vm_id: _, + jail_user: _, + chroot_base_dir: _, + netns: _, + api_socket, + log_level: _, + housekeeping_cores, + cgroup_procs, + } = spawn; + + let mut command = Command::new(jailer_bin); + command + .args(&args) .stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::piped()) - .spawn() - .map_err(|e| { - FirecrackerError::ProcessStart(format!("failed to spawn {firecracker_bin}: {e}")) - })?; + .stderr(std::process::Stdio::piped()); - // Spawn a thread to read stderr line-by-line - let stderr = child.stderr.take().ok_or_else(|| { - FirecrackerError::ProcessStart("stderr was piped but not available".into()) + if let Some(procs) = cgroup_procs { + // Cgroup membership is inherited across `fork` and survives + // `execve`, so writing to the pre-opened descriptor here places + // the child before it execs the jailer, and Firecracker inherits + // the membership through the jailer's own exec. Doing it here + // rather than after spawn also means the VMM never boots its API + // or touches memory on the wrong cores first. + #[expect( + unsafe_code, + reason = "cgroup placement must happen between fork and exec" + )] + // SAFETY: the closure runs in the forked child before `execve`, + // where only async-signal-safe work is permitted. It performs a + // single `write` of a fixed one-byte buffer on a descriptor that + // was opened before the fork: no allocation, no path resolution, + // and no locks. A failed write is reported to the parent over the + // CLOEXEC pipe and surfaces as a failed `spawn`. + unsafe { + command.pre_exec(move || place_in_cgroup(&procs)); + } + } + + let mut child = command.spawn().map_err(|e| FirecrackerError::Spawn { + path: jailer_bin.to_owned(), + source: e, })?; + + // Spawn a thread to read stderr line-by-line + let stderr = child.stderr.take().ok_or(FirecrackerError::Stdio( + "stderr was piped but not available", + ))?; let stderr_thread = std::thread::spawn(move || { use std::io::BufRead as _; @@ -66,21 +153,54 @@ impl FirecrackerProcess { } }); - let process = Self { + let mut process = Self { child, - api_socket_path: api_socket_path.to_owned(), + api_socket: api_socket.clone(), stderr_thread: Some(stderr_thread), }; - // Wait for the API socket to become ready - process.client().wait_for_ready(Duration::from_secs(5))?; + process.wait_for_ready(API_SOCKET_TIMEOUT)?; Ok(process) } + /// Wait for the API socket, giving up the moment the jailer dies. + /// + /// Watching the child is what keeps a jailer that failed outright from + /// presenting as a socket timeout. A bad `--netns`, an unwritable + /// `--chroot-base-dir`, or a refused `mknod` makes it exit immediately, + /// and polling for a socket that will never appear would report + /// `SocketNotReady` and point at Firecracker instead of at the jailer's + /// own diagnostics, which are already on stderr. + fn wait_for_ready(&mut self, timeout: Duration) -> Result<(), FirecrackerError> { + let start = std::time::Instant::now(); + let poll_interval = Duration::from_millis(50); + + while start.elapsed() < timeout { + if self.client().try_ready()? { + return Ok(()); + } + if let Ok(Some(status)) = self.child.try_wait() { + return Err(FirecrackerError::JailedProcessExited { status }); + } + std::thread::sleep(poll_interval); + } + + // Once more before giving up. The loop sleeps between polls, so a + // process that exits during the last sleep would otherwise be reported + // as a socket that never became ready, which points at Firecracker + // taking too long when the truth is that it is gone. That confusion is + // the entire reason this error exists. + if let Ok(Some(status)) = self.child.try_wait() { + return Err(FirecrackerError::JailedProcessExited { status }); + } + + Err(FirecrackerError::SocketNotReady(timeout)) + } + /// Get a client for the Firecracker REST API. pub fn client(&self) -> FirecrackerClient { - FirecrackerClient::new(&self.api_socket_path) + FirecrackerClient::new(self.api_socket.socket()) } /// Get the PID of the Firecracker process. @@ -107,7 +227,12 @@ impl FirecrackerProcess { std::thread::sleep(poll_interval); } - // Force kill if still running + // Force kill if still running. Unlike the readiness wait above, a child + // that exits during the final sleep is not a missed case here: nothing + // was reaped, so the pid is still reserved by the child and cannot have + // been recycled, `kill` sends a signal that a zombie simply ignores, and + // `kill` then reaps it and joins the reader. That is precisely what the + // loop would have done, so there is no verdict to get wrong. self.kill(); } @@ -119,8 +244,18 @@ impl FirecrackerProcess { } /// Clean up socket files. + /// + /// The chroot itself is reclaimed wholesale by the jail teardown; this + /// only keeps the socket from outliving the process within a job. + /// + /// Unlinks through the host view. Unlinking has no `sun_path` limit, so + /// the socket view buys nothing here and costs a dependency on a + /// descriptor still being open. This runs from `Drop`, where a future + /// reordering could close that descriptor first, and where the failure + /// would not be an error but the deletion of whatever file inherited the + /// number. pub fn cleanup(&self) { - drop(std::fs::remove_file(&self.api_socket_path)); + drop(std::fs::remove_file(self.api_socket.host().as_path())); } /// Join the stderr reader thread if it exists. @@ -137,3 +272,186 @@ impl Drop for FirecrackerProcess { self.cleanup(); } } + +/// Build the jailer's argument vector. +/// +/// Split out from the spawn so it can be asserted without a host that can +/// boot a VM. Getting this wrong fails every job at startup and, before the +/// VM boots, produces errors that point at Firecracker rather than at the +/// command line that caused them. +fn jailer_args(spawn: &JailedSpawn<'_>) -> Vec { + vec![ + "--id".to_owned(), + spawn.vm_id.to_string(), + "--exec-file".to_owned(), + spawn.exec_file.to_string(), + "--uid".to_owned(), + spawn.jail_user.uid().to_string(), + "--gid".to_owned(), + spawn.jail_user.gid().to_string(), + "--chroot-base-dir".to_owned(), + spawn.chroot_base_dir.to_string(), + "--netns".to_owned(), + spawn.netns.to_string(), + // No cgroup flags of any kind, and neither --daemonize nor + // --new-pid-ns: see `FirecrackerProcess::start`. + "--".to_owned(), + // `--id` is deliberately not forwarded: the jailer already passes it + // to Firecracker, which rejects the duplicate with DuplicateArgument + // and fails every job at startup. + "--api-sock".to_owned(), + // The chroot view. Firecracker binds this after it has been confined, + // so the host path would name a directory it cannot reach. + spawn.api_socket.chroot().as_str().to_owned(), + "--level".to_owned(), + spawn.log_level.to_owned(), + ] +} + +/// Join the calling task to the cgroup behind a pre-opened `cgroup.procs`. +/// +/// The kernel reads `0` as the calling task, which is why no pid has to be +/// formatted (and no allocation performed) inside the forked child. +fn place_in_cgroup(mut procs: &File) -> std::io::Result<()> { + use std::io::Write as _; + + procs.write_all(b"0") +} + +#[cfg(test)] +mod tests { + use camino::Utf8Path; + + use super::*; + use crate::jail::JailPaths; + + fn spawn_for<'a>(jail: &'a JailPaths, vm_id: &'a VmId) -> JailedSpawn<'a> { + JailedSpawn { + jailer_bin: Utf8Path::new("/tmp/work/jailer"), + exec_file: Utf8Path::new("/tmp/work/firecracker"), + vm_id, + jail_user: JailUser::default(), + chroot_base_dir: Utf8Path::new("/var/lib/bencher-runner/jail"), + netns: Utf8Path::new("/run/netns/bencher-jail"), + api_socket: jail.api_socket(), + log_level: "Warning", + housekeeping_cores: Vec::new(), + cgroup_procs: None, + } + } + + fn args() -> Vec { + let (_dir, jail) = jail_in_tmpdir(); + jailer_args(&spawn_for(&jail, &vm_id())) + } + + /// A stand-in identity for tests. + fn vm_id() -> VmId { + VmId::from_chroot_name("vm-1".to_owned()) + } + + /// The jail root has to exist: the paths hold a descriptor on it. + fn jail_in_tmpdir() -> (tempfile::TempDir, JailPaths) { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8Path::from_path(dir.path()).unwrap(); + let jail = JailPaths::new(root).unwrap(); + (dir, jail) + } + + /// The value following `flag`, if the flag is present. + fn value_of<'a>(args: &'a [String], flag: &str) -> Option<&'a str> { + let index = args.iter().position(|arg| arg == flag)?; + args.get(index + 1).map(String::as_str) + } + + #[test] + fn confinement_flags_are_all_present() { + let args = args(); + + assert_eq!(value_of(&args, "--id"), Some("vm-1")); + assert_eq!( + value_of(&args, "--exec-file"), + Some("/tmp/work/firecracker") + ); + assert_eq!(value_of(&args, "--uid"), Some("61016")); + assert_eq!(value_of(&args, "--gid"), Some("61016")); + assert_eq!( + value_of(&args, "--chroot-base-dir"), + Some("/var/lib/bencher-runner/jail") + ); + assert_eq!(value_of(&args, "--netns"), Some("/run/netns/bencher-jail")); + } + + #[test] + fn id_appears_exactly_once() { + // The jailer passes `--id` to Firecracker itself. Forwarding it again + // after the separator makes Firecracker reject the duplicate and fail + // every job at startup. + let args = args(); + + assert_eq!( + args.iter().filter(|arg| *arg == "--id").count(), + 1, + "--id must be given to the jailer only: {args:?}" + ); + } + + #[test] + fn only_the_api_socket_and_level_are_forwarded() { + let args = args(); + let separator = args + .iter() + .position(|arg| arg == "--") + .expect("the jailer needs a -- separator before Firecracker's own arguments"); + + assert_eq!( + args.get(separator + 1..), + Some( + [ + "--api-sock".to_owned(), + "/api.sock".to_owned(), + "--level".to_owned(), + "Warning".to_owned(), + ] + .as_slice() + ) + ); + } + + #[test] + fn the_api_socket_is_the_chroot_view() { + // Firecracker binds the socket after it has been confined, so it must + // receive the path as it will exist inside the chroot. The host view + // names a directory the jailed process cannot reach. + let (_dir, jail) = jail_in_tmpdir(); + let args = jailer_args(&spawn_for(&jail, &vm_id())); + + assert_eq!(value_of(&args, "--api-sock"), Some("/api.sock")); + let jail_root = jail.root().as_str(); + assert!( + !args.iter().any(|arg| arg.contains(jail_root)), + "no host-side jail path may reach the jailed process: {args:?}" + ); + } + + #[test] + fn no_cgroup_or_forking_flags_are_passed() { + // The runner owns the cgroup end to end, and both --daemonize and + // --new-pid-ns make the jailer fork, which breaks the pid identity the + // process management relies on. + let args = args(); + + for forbidden in [ + "--cgroup", + "--parent-cgroup", + "--cgroup-version", + "--daemonize", + "--new-pid-ns", + ] { + assert!( + !args.iter().any(|arg| arg == forbidden), + "{forbidden} must not be passed: {args:?}" + ); + } + } +} diff --git a/plus/bencher_runner/src/firecracker/vsock.rs b/plus/bencher_runner/src/firecracker/vsock.rs index 4043de3ff..3a59b6ce2 100644 --- a/plus/bencher_runner/src/firecracker/vsock.rs +++ b/plus/bencher_runner/src/firecracker/vsock.rs @@ -2,8 +2,12 @@ //! //! Firecracker's vsock implementation uses Unix domain sockets on the host side. //! When the guest connects to CID 2 (host) on port N, Firecracker connects to -//! `{uds_path}_{N}` on the host. The host must have Unix listeners at those -//! paths before VM boot. +//! `{uds_path}_{N}`. The runner binds those sockets, from outside the chroot, +//! before VM boot; Firecracker reaches the same inodes at the chroot view of +//! the path and creates the base `uds_path` itself. +//! +//! Unix domain sockets are scoped by the filesystem, not by the network +//! namespace, so the empty namespace the VMM joins does not affect them. use std::io::Read as _; use std::os::fd::AsFd as _; @@ -12,9 +16,12 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; +use camino::Utf8Path; use nix::poll::{PollFd, PollFlags, PollTimeout, poll}; use crate::firecracker::error::FirecrackerError; +use crate::jail::chroot::chown_to_jail; +use crate::jail::{JailFile, JailUser}; /// Poll timeout for vsock listeners (50ms). /// @@ -32,6 +39,9 @@ mod ports { pub const STDERR: u32 = 5001; pub const EXIT_CODE: u32 = 5002; pub const OUTPUT_FILES: u32 = 5005; + + /// Every port the runner listens on. + pub const ALL: [u32; 4] = [STDOUT, STDERR, EXIT_CODE, OUTPUT_FILES]; } /// Results collected from the guest via vsock. @@ -49,8 +59,12 @@ pub struct VsockResults { /// Host-side vsock listener that accepts connections from Firecracker. pub struct VsockListener { - /// Base path for the vsock UDS. - vsock_uds_path: String, + /// Both views of the vsock base path. + /// + /// Binding needs the socket view, because of the `sun_path` limit. + /// Everything else uses the host view, which carries no dependency on a + /// descriptor staying open. + vsock: JailFile, /// Listeners for each port. stdout_listener: UnixListener, stderr_listener: UnixListener, @@ -61,13 +75,16 @@ pub struct VsockListener { impl VsockListener { /// Create vsock listeners for all expected ports. /// - /// Creates Unix listeners at `{vsock_uds_path}_{port}` for each port. - /// These must be created before the VM boots. - pub fn new(vsock_uds_path: &str) -> Result { - let stdout_path = format!("{vsock_uds_path}_{}", ports::STDOUT); - let stderr_path = format!("{vsock_uds_path}_{}", ports::STDERR); - let exit_code_path = format!("{vsock_uds_path}_{}", ports::EXIT_CODE); - let output_files_path = format!("{vsock_uds_path}_{}", ports::OUTPUT_FILES); + /// Creates Unix listeners at `{vsock}_{port}` for each port. Binding uses + /// the socket view, which is the only view short enough for `sun_path`; + /// unlinking and ownership use the host view, which does not depend on a + /// descriptor staying open. These must be created before the VM boots. + pub fn new(vsock: &JailFile) -> Result { + let socket = vsock.socket(); + let stdout_path = socket.with_suffix(&format!("_{}", ports::STDOUT)); + let stderr_path = socket.with_suffix(&format!("_{}", ports::STDERR)); + let exit_code_path = socket.with_suffix(&format!("_{}", ports::EXIT_CODE)); + let output_files_path = socket.with_suffix(&format!("_{}", ports::OUTPUT_FILES)); // Remove stale socket files for path in [ @@ -103,7 +120,7 @@ impl VsockListener { })?; Ok(Self { - vsock_uds_path: vsock_uds_path.to_owned(), + vsock: vsock.clone(), stdout_listener, stderr_listener, exit_code_listener, @@ -263,18 +280,37 @@ impl VsockListener { }) } + /// Hand the listener sockets to the jail uid and gid. + /// + /// Firecracker connects out to these sockets as the unprivileged jail + /// user, so it needs write permission on the inodes. After `pivot_root` + /// the only directory it traverses is `/`, which the jailer chowns itself, + /// so the inodes are all that is left to hand over. Must run after bind + /// and before `InstanceStart`. + pub fn chown_to_jail(&self, jail_user: JailUser) -> Result<(), crate::error::JailError> { + for port in ports::ALL { + chown_to_jail(Utf8Path::new(&self.host_path(port)), jail_user)?; + } + Ok(()) + } + /// Remove all socket files created by this listener. + /// + /// Unlinks through the host view, for the same reason as + /// [`crate::firecracker::process::FirecrackerProcess::cleanup`]: unlinking + /// has no `sun_path` limit, and this runs from `Drop`, where naming a + /// descriptor that may already be closed would delete an unrelated file + /// rather than fail. pub fn cleanup(&self) { - for port in [ - ports::STDOUT, - ports::STDERR, - ports::EXIT_CODE, - ports::OUTPUT_FILES, - ] { - let path = format!("{}_{port}", self.vsock_uds_path); - drop(std::fs::remove_file(path)); + for port in ports::ALL { + drop(std::fs::remove_file(self.host_path(port))); } } + + /// The host path of the listener socket for a port. + fn host_path(&self, port: u32) -> String { + format!("{}_{port}", self.vsock.host()) + } } impl Drop for VsockListener { @@ -324,6 +360,7 @@ fn try_accept_and_read(listener: &UnixListener, max_data_size: usize) -> Option< )] mod tests { use super::*; + use crate::jail::JailPaths; use std::io::Write as _; use std::os::unix::net::UnixStream; @@ -333,12 +370,24 @@ mod tests { /// Short grace period for tests to avoid slowing down the test suite. const TEST_GRACE_PERIOD: Duration = Duration::from_millis(50); - /// Helper: create a `VsockListener` in a temp directory. - fn listener_in_tmpdir() -> (tempfile::TempDir, VsockListener) { + /// Helper: a jail whose descriptor stays open for the whole test. + /// + /// The socket view names an open descriptor by number, so the `JailPaths` + /// has to outlive every path derived from it. Dropping it early leaves the + /// paths addressing whatever the kernel hands that number to next, which + /// is exactly the failure this binding prevents. + fn jail_in_tmpdir() -> (tempfile::TempDir, JailPaths) { let dir = tempfile::tempdir().unwrap(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); - let listener = VsockListener::new(&base).unwrap(); - (dir, listener) + let root = Utf8Path::from_path(dir.path()).unwrap(); + let jail = JailPaths::new(root).unwrap(); + (dir, jail) + } + + /// Helper: create a `VsockListener` on a jail that stays alive. + fn listener_in_tmpdir() -> (tempfile::TempDir, JailPaths, VsockListener) { + let (dir, jail) = jail_in_tmpdir(); + let listener = VsockListener::new(jail.vsock()).unwrap(); + (dir, jail, listener) } /// Helper: connect to a vsock port and write data. @@ -351,9 +400,8 @@ mod tests { #[test] fn vsock_listener_creates_socket_files() { - let dir = tempfile::tempdir().unwrap(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); - let _listener = VsockListener::new(&base).unwrap(); + let (_dir, jail, _listener) = listener_in_tmpdir(); + let base = jail.vsock().host().to_string(); for port in [5000, 5001, 5002, 5005] { let path = format!("{base}_{port}"); @@ -366,11 +414,11 @@ mod tests { #[test] fn vsock_listener_cleanup_removes_files() { - let dir = tempfile::tempdir().unwrap(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); + let (_dir, jail) = jail_in_tmpdir(); + let base = jail.vsock().host().to_string(); { - let _listener = VsockListener::new(&base).unwrap(); + let _listener = VsockListener::new(jail.vsock()).unwrap(); // listener drops here } @@ -385,8 +433,8 @@ mod tests { #[test] fn collect_all_ports() { - let (dir, listener) = listener_in_tmpdir(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); + let (_dir, jail, listener) = listener_in_tmpdir(); + let base = jail.vsock().host().to_string(); // Build protocol-encoded data: 1 file, path="out.bin", content=\x00\x01\x02 let mut encoded = Vec::new(); @@ -427,8 +475,8 @@ mod tests { #[test] fn collect_exit_code_only() { - let (dir, listener) = listener_in_tmpdir(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); + let (_dir, jail, listener) = listener_in_tmpdir(); + let base = jail.vsock().host().to_string(); let base_clone = base.clone(); let sender = std::thread::spawn(move || { @@ -454,7 +502,7 @@ mod tests { #[test] fn collect_timeout_returns_error() { - let (_dir, listener) = listener_in_tmpdir(); + let (_dir, _jail, listener) = listener_in_tmpdir(); // No data sent — should timeout with an error let result = listener.collect_results( @@ -473,8 +521,8 @@ mod tests { #[test] fn collect_non_utf8_stdout() { - let (dir, listener) = listener_in_tmpdir(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); + let (_dir, jail, listener) = listener_in_tmpdir(); + let base = jail.vsock().host().to_string(); let base_clone = base.clone(); let sender = std::thread::spawn(move || { @@ -502,8 +550,8 @@ mod tests { #[test] fn collect_exit_code_triggers_final_pass() { - let (dir, listener) = listener_in_tmpdir(); - let base = dir.path().join("vsock").to_str().unwrap().to_owned(); + let (_dir, jail, listener) = listener_in_tmpdir(); + let base = jail.vsock().host().to_string(); let base_clone = base.clone(); let sender = std::thread::spawn(move || { @@ -605,7 +653,7 @@ mod tests { #[test] fn collect_cancelled_returns_error() { - let (_dir, listener) = listener_in_tmpdir(); + let (_dir, _jail, listener) = listener_in_tmpdir(); // Set the cancel flag before collecting let cancel_flag = Arc::new(AtomicBool::new(true)); diff --git a/plus/bencher_runner/src/jail/cgroup.rs b/plus/bencher_runner/src/jail/cgroup.rs index 003cdd6a3..db5edeabc 100644 --- a/plus/bencher_runner/src/jail/cgroup.rs +++ b/plus/bencher_runner/src/jail/cgroup.rs @@ -9,7 +9,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use crate::RunnerError; use crate::cpu::CpuLayout; use crate::error::JailError; -use crate::jail::ResourceLimits; +use crate::jail::{JailSignals, VmId}; /// Default cgroup v2 mount point. const CGROUP_ROOT: &str = "/sys/fs/cgroup"; @@ -21,43 +21,87 @@ pub(crate) const BENCHER_CGROUP_BASE: &str = "bencher"; pub struct CgroupManager { cgroup_path: Utf8PathBuf, created: bool, + /// Raised when this cgroup could not be removed, which holds the chroot that + /// names it and earns a later job another sweep. + signals: JailSignals, } impl CgroupManager { - /// Create a new cgroup for the given run ID. - pub fn new(run_id: &str) -> Result { + /// Create a new cgroup for the given microVM. + /// + /// The signal is shared with the chroot of the same id: a cgroup that + /// cannot be removed has to keep that directory alive, because the + /// directory name is the only handle a later sweep has for finding this + /// cgroup again. + pub fn new(vm_id: &VmId, signals: JailSignals) -> Result { let cgroup_path = Utf8PathBuf::from(CGROUP_ROOT) .join(BENCHER_CGROUP_BASE) - .join(run_id); + .join(vm_id.as_str()); - // Ensure parent bencher cgroup exists + // Unconditional, because `create_dir_all` on a directory that is + // already there is a success. The `exists` check this replaces was a + // read that could only mislead: an error from it would have been read as + // an absent parent, and it gated nothing the create does not gate + // itself. let parent = Utf8PathBuf::from(CGROUP_ROOT).join(BENCHER_CGROUP_BASE); - if !parent.exists() { - fs::create_dir_all(&parent).map_err(|e| JailError::CreateCgroup { - path: parent.clone(), - source: e, - })?; - } + fs::create_dir_all(&parent).map_err(|e| JailError::CreateCgroup { + path: parent.clone(), + source: e, + })?; // Enable controllers in the parent. Always attempted (idempotent): // the parent may have been created without controllers, e.g. by // the tuning cpuset partition at startup. Self::enable_controllers(&parent)?; - // Create this run's cgroup - if !cgroup_path.exists() { - fs::create_dir_all(&cgroup_path).map_err(|e| JailError::CreateCgroup { - path: cgroup_path.clone(), - source: e, - })?; - } + // Create this run's cgroup, and remember whether we are the ones who + // made it. `Drop` removes what this created, so claiming ownership of + // a cgroup that was already there would have it rmdir something + // belonging to whoever did create it. Fresh ids make that unlikely, + // but this branch exists precisely for when the id is not fresh. + // `try_exists`, because this decides whether `Drop` may remove the + // directory. An error read as "not there" would have this claim a cgroup + // somebody else owns and then delete it on the way out, which is the one + // outcome the branch exists to prevent. + let created = match cgroup_path.try_exists() { + Ok(true) => false, + Ok(false) => { + fs::create_dir_all(&cgroup_path).map_err(|e| JailError::CreateCgroup { + path: cgroup_path.clone(), + source: e, + })?; + true + }, + Err(e) => { + return Err(JailError::ReadCgroup { + path: cgroup_path, + source: e, + } + .into()); + }, + }; Ok(Self { cgroup_path, - created: true, + created, + signals, }) } + /// Wrap an existing cgroup directory without creating or removing it. + /// + /// For tests that exercise the placement logic against a stand-in tree + /// rather than the real cgroup filesystem. + #[cfg(test)] + #[must_use] + pub fn detached(cgroup_path: Utf8PathBuf) -> Self { + Self { + cgroup_path, + created: false, + signals: JailSignals::unwatched(), + } + } + /// Enable controllers in a cgroup. /// /// Enables cpu, memory, and pids controllers (required), and io/cpuset controllers @@ -72,8 +116,14 @@ impl CgroupManager { .or_else(|_| fs::write(&subtree_control, "+cpu +memory +pids +io")) .or_else(|_| fs::write(&subtree_control, "+cpu +memory +pids")); - // Verify that required controllers are enabled - let enabled = fs::read_to_string(&subtree_control).unwrap_or_default(); + // Verify that required controllers are enabled. A read that failed is + // not an empty list: reporting one would blame the host for a + // controller it may well have enabled, on the strength of a question + // nobody answered. + let enabled = fs::read_to_string(&subtree_control).map_err(|e| JailError::ReadCgroup { + path: subtree_control.clone(), + source: e, + })?; if let Some(missing) = missing_required_controller(&enabled) { return Err(match write_result { Err(e) => JailError::EnableControllers { @@ -93,40 +143,6 @@ impl CgroupManager { Ok(()) } - /// Apply resource limits to this cgroup. - pub fn apply_limits(&self, limits: &ResourceLimits) -> Result<(), RunnerError> { - // CPU limit - if let Some(quota) = limits.cpu_quota_us { - let cpu_max = format!("{quota} {}", limits.cpu_period_us); - self.write_file("cpu.max", &cpu_max)?; - } - - // Memory limit - if let Some(bytes) = limits.memory_bytes { - self.write_file("memory.max", &bytes.to_string())?; - - // Disable swap to ensure benchmark memory measurements are accurate - // and to prevent swap thrashing from affecting benchmark results. - drop(self.disable_swap()); - } - - // OOM group kill: when the cgroup hits its memory limit, kill ALL processes - // in the group together. This prevents partial kills that leave orphan processes. - drop(self.write_file("memory.oom.group", "1")); - - // PIDs limit - self.write_file("pids.max", &limits.max_procs.to_string())?; - - // I/O limits - applied to all block devices - // Note: This requires knowing the device major:minor. We attempt to - // discover common devices, but this may not work in all configuration. - if limits.io_read_bps.is_some() || limits.io_write_bps.is_some() { - self.apply_io_limits(limits); - } - - Ok(()) - } - /// Apply CPU pinning via cpuset controller. /// /// Restricts processes in this cgroup to run only on the specified CPUs. @@ -139,103 +155,117 @@ impl CgroupManager { /// /// # Errors /// - /// Returns Ok even if cpuset is not available (logs a warning). - /// CPU pinning is best-effort for isolation but not required for correctness. - pub fn apply_cpuset(&self, layout: &CpuLayout) -> Result<(), RunnerError> { + /// Returns an error when the cpuset controller is present but rejects the + /// write. That is a half-applied fidelity mechanism: the cgroup would + /// exist without confining the VMM to the benchmark cores, so the run + /// would report a number measured somewhere other than where it claims. + /// + /// A controller that is not there at all is a different thing and is not + /// an error. `enable_controllers` falls back as far as `+cpu +memory + /// +pids`, and only those three are required, so a host that does not + /// delegate `cpuset` (a containerized runner, or a cgroup namespace + /// without it in `subtree_control`) creates its cgroup successfully and + /// then has no `cpuset.cpus` to write. That is a declared absence of + /// isolation, which the caller degrades on rather than failing. + pub fn apply_cpuset(&self, layout: &CpuLayout) -> Result { if !layout.has_isolation() { - // No meaningful isolation possible (single core or overlapping sets) - return Ok(()); + // Single core, or overlapping sets: there is nothing to confine to. + return Ok(Cpuset::Unavailable("the CPU layout offers no isolation")); } let cpuset = layout.benchmark_cpuset(); if cpuset.is_empty() { - return Ok(()); + return Ok(Cpuset::Unavailable("the benchmark core set is empty")); } - // Try to write cpuset.cpus - may fail if cpuset controller is not available + // `try_exists`, not `exists`: the latter reports an error as an absent + // file, so a cgroup directory the runner cannot stat would be declared + // an undelegated controller. That is a claim about the host made from a + // question that failed, and it is the difference between a host with no + // isolation to offer and a host nobody could ask. let path = self.cgroup_path.join("cpuset.cpus"); - if let Err(e) = fs::write(&path, &cpuset) { - // Log warning but don't fail - cpuset is optional for isolation - eprintln!( - "Warning: failed to set cpuset.cpus to '{cpuset}' (cpuset controller may not be available): {e}" - ); - } else { - // Also need to set cpuset.mems for cpuset to work. Use the - // parent's effective memory nodes so multi-node NUMA hosts - // are not forced onto node 0. - let mems = self - .cgroup_path - .parent() - .map_or_else(|| "0".to_owned(), effective_mems); - let mems_path = self.cgroup_path.join("cpuset.mems"); - if let Err(e) = fs::write(&mems_path, &mems) { - eprintln!("Warning: failed to set cpuset.mems: {e}"); - } + match path.try_exists() { + Ok(true) => {}, + Ok(false) => return Ok(Cpuset::Unavailable(UNDELEGATED)), + Err(e) => return Err(JailError::ReadCgroup { path, source: e }.into()), } - - Ok(()) - } - - /// Apply I/O bandwidth limits. - /// - /// Attempts to apply io.max limits to discovered block devices. - /// The io.max format is: "MAJ:MIN rbps=BYTES wbps=BYTES" - fn apply_io_limits(&self, limits: &ResourceLimits) { - use std::fmt::Write as _; - - // Try to find block devices to apply limits to - let devices = Self::discover_block_devices(); - - if devices.is_empty() { - // No devices found, skip I/O limits silently - return; + // Every failure, absence included. Delegation was settled by the check + // above, so a `cpuset.cpus` that has gone missing between that stat and + // this write is a cgroup disappearing underneath the runner, not a host + // that never had the controller. `verify_cpuset` reasons the same way + // about the same question one step later; the two used to disagree. + if let Err(e) = fs::write(&path, &cpuset) { + return Err(JailError::WriteCgroup { path, source: e }.into()); } - let read_limit = limits - .io_read_bps - .map_or("max".to_owned(), |v| v.to_string()); - let write_limit = limits - .io_write_bps - .map_or("max".to_owned(), |v| v.to_string()); - - let mut io_max_content = String::new(); - for (major, minor) in devices { - // Format: "MAJ:MIN rbps=BYTES wbps=BYTES" - let _unused = writeln!( - io_max_content, - "{major}:{minor} rbps={read_limit} wbps={write_limit}" - ); + // Also need to set cpuset.mems for cpuset to work. Use the parent's + // effective memory nodes so multi-node NUMA hosts are not forced onto + // node 0. Applied cpus without mems is the half-applied case. + let mems = match self.cgroup_path.parent() { + Some(parent) => effective_mems(parent).map_err(|e| JailError::ReadCgroup { + path: parent.join(MEMS_EFFECTIVE), + source: e, + })?, + None => NODE_ZERO.to_owned(), + }; + let mems_path = self.cgroup_path.join("cpuset.mems"); + if let Err(e) = fs::write(&mems_path, &mems) { + return Err(JailError::WriteCgroup { + path: mems_path, + source: e, + } + .into()); } - // Try to write io.max - may fail if io controller is not available - let path = self.cgroup_path.join("io.max"); - if let Err(e) = fs::write(&path, &io_max_content) { - // Log warning but don't fail - io controller may not be available - eprintln!("Warning: failed to set io.max (io controller may not be available): {e}"); - } + self.verify_cpuset(&cpuset, &mems) } - /// Discover block devices on the system. + /// Confirm the kernel actually gave the cgroup the cores that were asked + /// for. /// - /// Returns a list of (major, minor) device numbers for block devices. - fn discover_block_devices() -> Vec<(u32, u32)> { - let mut devices = Vec::new(); - - // Try to read /sys/block to find block devices - if let Ok(entries) = fs::read_dir("/sys/block") { - for entry in entries.flatten() { - let dev_path = entry.path().join("dev"); - if let Ok(content) = fs::read_to_string(&dev_path) - && let Some((major_str, minor_str)) = content.trim().split_once(':') - && let (Ok(major), Ok(minor)) = - (major_str.parse::(), minor_str.parse::()) - { - devices.push((major, minor)); + /// A successful write proves nothing here. Under cgroup v2 the effective + /// set is the written set intersected with the parent's effective set, so + /// a `cpuset.cpus` that reaches past the parent is accepted and then + /// silently narrowed, possibly to nothing at all, in which case the VMM + /// simply inherits the parent's CPUs. (An exclusive sibling narrows a set + /// the same way, though these cgroups never claim exclusivity.) The whole + /// point of separating applied from half-applied is lost if the applied + /// case is taken on trust, so both effective sets are read back and have + /// to match. Every other fidelity mechanism here already reads back: the + /// partition mode does, and so does cgroup placement. + fn verify_cpuset(&self, cpus: &str, mems: &str) -> Result { + // Memory nodes as well as cpus. The mems value is derived from the + // parent's effective set, so narrowing is unlikely, but it is written + // exactly the same way and would otherwise be the last write on this + // path still taken on trust. + for (file, requested) in [ + ("cpuset.cpus.effective", cpus), + ("cpuset.mems.effective", mems), + ] { + let path = self.cgroup_path.join(file); + // Every failure, absence included. Reaching this function means the + // write to `cpuset.cpus` succeeded, which is proof the controller is + // delegated, so a missing effective file here cannot mean it is not: + // reporting an undelegated controller would tell the operator + // something this code just observed to be false, while the run went + // ahead on a cpuset that was never verified. The read is the whole + // mechanism, so a read that did not happen is a failure of it. + let effective = fs::read_to_string(&path).map_err(|e| JailError::ReadCgroup { + path: path.clone(), + source: e, + })?; + + if parse_cpuset(&effective) != parse_cpuset(requested) { + return Err(JailError::CpusetNarrowed { + path, + requested: requested.to_owned(), + effective: effective.trim().to_owned(), } + .into()); } } - devices + Ok(Cpuset::Applied) } /// Disable swap for this cgroup. @@ -246,15 +276,33 @@ impl CgroupManager { self.write_file("memory.swap.max", "0") } - /// Add the current process to this cgroup. - pub fn add_self(&self) -> Result<(), RunnerError> { - let pid = std::process::id(); - self.write_file("cgroup.procs", &pid.to_string()) + /// Open this cgroup's `cgroup.procs` for writing. + /// + /// The descriptor is opened before the fork so the `pre_exec` closure that + /// joins the cgroup performs only a `write` of a fixed byte on an existing + /// descriptor: no allocation, no path resolution, nothing that is not + /// async-signal-safe. + pub fn open_procs(&self) -> Result { + let path = self.cgroup_path.join("cgroup.procs"); + fs::OpenOptions::new() + .write(true) + .open(&path) + .map_err(|e| JailError::OpenCgroupProcs { path, source: e }) } - /// Add a process by PID to this cgroup. - pub fn add_pid(&self, pid: u32) -> Result<(), RunnerError> { - self.write_file("cgroup.procs", &pid.to_string()) + /// Whether `pid` is a member of this cgroup. + /// + /// A failed `pre_exec` write already surfaces as a failed spawn; this is + /// for the different case of a write that succeeded against the wrong + /// destination. A cgroup that exists but does not contain the VMM is a + /// silent lie about where the benchmark ran. + pub fn contains_pid(&self, pid: u32) -> Result { + let path = self.cgroup_path.join("cgroup.procs"); + let procs = fs::read_to_string(&path).map_err(|e| JailError::ReadCgroup { + path: path.clone(), + source: e, + })?; + Ok(procs_contains_pid(&procs, pid)) } /// Write to a cgroup file. @@ -276,6 +324,10 @@ impl CgroupManager { /// that survive a direct-child kill, e.g. on timeout or cancellation, /// so no stray work lingers on benchmark cores and the cgroup can be /// removed. + /// + /// Best effort is sound here only because something else catches it: + /// whatever this fails to kill is exactly what makes [`Self::cleanup`]'s + /// `rmdir` fail, and that arms the reclaim signal. See [`crate::jail`]. pub fn kill_all(&self) { if let Err(e) = self.write_file("cgroup.kill", "1") { eprintln!("Warning: failed to kill cgroup subtree: {e}"); @@ -283,36 +335,182 @@ impl CgroupManager { } /// Clean up the cgroup. - pub fn cleanup(&mut self) -> Result<(), RunnerError> { - if self.created && self.cgroup_path.exists() { - if let Err(e) = fs::remove_dir(&self.cgroup_path) { - // Log but don't fail - cgroup might still have processes - eprintln!("Warning: failed to remove cgroup {}: {e}", self.cgroup_path); - } else { - self.created = false; - } + /// + /// A `rmdir` the kernel refuses is raised on the reclaim signal, not just + /// logged: it means something is still in this cgroup, and the only way to + /// get to it later is through the chroot of the same id, so the signal both + /// holds that directory and earns the next job a sweep. + /// + /// Returns nothing, because the signal is where a failure goes. A `Result` + /// here would be a channel with nothing in it that every caller, `Drop` + /// included, would have to discard. + pub fn cleanup(&mut self) { + if !self.created { + return; + } + // A stat that failed is not a cgroup that is gone. Reading it as one + // would skip both the removal and the signal, so nothing would be armed + // and nothing would ever come back for it. + match self.cgroup_path.try_exists() { + Ok(false) => self.created = false, + Ok(true) => { + if let Err(e) = fs::remove_dir(&self.cgroup_path) { + eprintln!( + "Warning: failed to remove cgroup {}: {e}. Something is still in it, so the next job sweeps it along with the jail that names it.", + self.cgroup_path + ); + self.signals.cgroup_survived(); + } else { + self.created = false; + } + }, + Err(e) => { + eprintln!( + "Warning: cannot tell whether cgroup {} is still there: {e}. It is treated as still there, so the next job sweeps it along with the jail that names it.", + self.cgroup_path + ); + self.signals.cgroup_survived(); + }, } - Ok(()) } } impl Drop for CgroupManager { fn drop(&mut self) { - drop(self.cleanup()); + self.cleanup(); } } +/// The `cpuset.mems.effective` file, read to mirror a parent's memory nodes. +pub(crate) const MEMS_EFFECTIVE: &str = "cpuset.mems.effective"; + +/// The single memory node a host without a readable node set is assumed to have. +const NODE_ZERO: &str = "0"; + /// Read a cgroup's effective memory nodes (`cpuset.mems.effective`). /// -/// Falls back to node `0` when the file is missing or empty (e.g., the -/// cpuset controller is not enabled). Using effective mems instead of a -/// hardcoded node keeps multi-node NUMA hosts from forcing all benchmark +/// Falls back to node `0` when the file is missing or empty, which is what the +/// cpuset controller not being enabled looks like. Using effective mems instead +/// of a hardcoded node keeps multi-node NUMA hosts from forcing all benchmark /// memory onto node 0. -pub(crate) fn effective_mems(cgroup: &Utf8Path) -> String { - match fs::read_to_string(cgroup.join("cpuset.mems.effective")) { - Ok(mems) if !mems.trim().is_empty() => mems.trim().to_owned(), - _ => "0".to_owned(), +/// +/// Which is exactly why any other failure is an error rather than the fallback. +/// This value is written to `cpuset.mems`, so answering node 0 to a read that +/// did not happen confines the guest's memory to one node on a host that may +/// have several, and the run then reports a number measured under a constraint +/// nobody chose. An absent file says the host has nothing to tell; a failed read +/// says nobody asked it. +pub(crate) fn effective_mems(cgroup: &Utf8Path) -> Result { + match fs::read_to_string(cgroup.join(MEMS_EFFECTIVE)) { + Ok(mems) if !mems.trim().is_empty() => Ok(mems.trim().to_owned()), + Ok(_) => Ok(NODE_ZERO.to_owned()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(NODE_ZERO.to_owned()), + Err(e) => Err(e), + } +} + +/// Why a run has no CPU isolation, when it has none. +const UNDELEGATED: &str = "the cpuset controller is not delegated to this cgroup"; + +/// Whether the cpuset actually confined the VMM to the benchmark cores. +/// +/// Every variant that is not [`Self::Applied`] carries the reason, so no +/// variant can claim confinement without a verified write behind it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Cpuset { + /// The cgroup confines the VMM to the benchmark cores, read back and + /// confirmed. + Applied, + /// There is no CPU isolation to be had, for the reason given. + Unavailable(&'static str), +} + +/// Parse a kernel cpu list (`0-3,5,7-9`) into the set of cpus it names. +/// +/// Compared as sets rather than as strings, because the kernel is free to +/// render the same set differently from the way it was written. +fn parse_cpuset(cpuset: &str) -> std::collections::BTreeSet { + let mut cpus = std::collections::BTreeSet::new(); + for group in cpuset.trim().split(',').filter(|group| !group.is_empty()) { + match group.split_once('-') { + Some((start, end)) => { + if let (Ok(start), Ok(end)) = (start.trim().parse(), end.trim().parse::()) { + cpus.extend(start..=end); + } + }, + None => { + if let Ok(cpu) = group.trim().parse() { + cpus.insert(cpu); + } + }, + } } + cpus +} + +/// How long to keep trying to remove a stale cgroup. +/// +/// `rmdir` fails while the cgroup still holds a process, and the reap that +/// precedes it may need a moment to land. +const REMOVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// How often to retry. +const REMOVE_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50); + +/// Remove the cgroup a swept jail left behind. +/// +/// The cgroup and the chroot are named by the same VM id by construction, so +/// the id read off the chroot directory names the cgroup exactly. +/// +/// Must run after the VMM in it has been reaped: `rmdir` on a cgroup that +/// still holds a process fails, which is what forces that ordering. A cgroup +/// that survives is worth reporting, not because it claims anything (these +/// cgroups set no exclusive cpuset, so a leftover blocks nothing) but because +/// the usual reason `rmdir` fails is that something is still running in it. +pub(crate) fn remove_stale_cgroup(vm_id: &VmId) -> Result<(), JailError> { + let path = Utf8PathBuf::from(CGROUP_ROOT) + .join(BENCHER_CGROUP_BASE) + .join(vm_id.as_str()); + // The caller deletes the chroot that names this cgroup once this returns + // `Ok`, and that chroot is the only handle any later sweep has for finding + // the cgroup again. A stat error read as "already gone" would strand the + // cgroup permanently and delete the one thing that could have found it, + // which is exactly what the caller's ordering exists to prevent. + match path.try_exists() { + Ok(false) => return Ok(()), + Ok(true) => {}, + Err(e) => return Err(JailError::StaleCgroup { path, source: e }), + } + + let deadline = std::time::Instant::now() + REMOVE_TIMEOUT; + loop { + match fs::remove_dir(&path) { + Ok(()) => { + eprintln!("Removed stale cgroup {path} left by a previous runner"); + return Ok(()); + }, + // Someone else got there first, which is the outcome either way. + // Only a stat that succeeded and said absent counts: anything else + // falls through to the retry and, in the end, to the error. + Err(_) if path.try_exists().is_ok_and(|exists| !exists) => return Ok(()), + Err(e) if std::time::Instant::now() >= deadline => { + // Reported rather than warned, because failing here means the + // next job sweeps again rather than inheriting a host nobody + // is looking at. + return Err(JailError::StaleCgroup { path, source: e }); + }, + Err(_) => std::thread::sleep(REMOVE_INTERVAL), + } + } +} + +/// Whether a `cgroup.procs` listing contains `pid`. +/// +/// Matches whole lines: pid `7` must not be satisfied by pid `70`. +fn procs_contains_pid(procs: &str, pid: u32) -> bool { + procs + .lines() + .any(|line| line.trim().parse::() == Ok(pid)) } /// Return the first required controller missing from a @@ -325,15 +523,6 @@ fn missing_required_controller(enabled: &str) -> Option<&'static str> { .find(|required| !enabled.split_whitespace().any(|token| token == *required)) } -/// Check if cgroup v2 is available. -#[expect(dead_code, reason = "utility for future cgroup v2 feature detection")] -#[must_use] -pub fn is_cgroup_v2_available() -> bool { - Utf8Path::new(CGROUP_ROOT) - .join("cgroup.controllers") - .exists() -} - #[cfg(test)] mod tests { use camino::Utf8PathBuf; @@ -356,13 +545,290 @@ mod tests { assert_eq!(missing_required_controller("cpu memory"), Some("pids")); } + /// A stand-in cgroup tree with the cpuset controller delegated. + /// + /// `effective` is what the kernel would report back after the write, which + /// is the whole point: a real kernel may narrow it silently. + fn cpuset_tree(effective: &str) -> (tempfile::TempDir, CgroupManager) { + // The mems written here derive from a parent with no + // `cpuset.mems.effective`, which falls back to node 0. + cpuset_tree_with_mems(effective, "0") + } + + /// A stand-in tree where the effective memory nodes can also be chosen. + fn cpuset_tree_with_mems( + effective_cpus: &str, + effective_mems: &str, + ) -> (tempfile::TempDir, CgroupManager) { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + fs::write(root.join("cpuset.cpus"), "").unwrap(); + fs::write(root.join("cpuset.mems"), "").unwrap(); + fs::write(root.join("cpuset.cpus.effective"), effective_cpus).unwrap(); + fs::write(root.join("cpuset.mems.effective"), effective_mems).unwrap(); + (dir, CgroupManager::detached(root)) + } + + #[test] + fn a_delegated_cpuset_that_the_kernel_honors_is_applied() { + let (_dir, manager) = cpuset_tree("2-7\n"); + let layout = CpuLayout::with_core_count(8); + + assert_eq!(manager.apply_cpuset(&layout).unwrap(), Cpuset::Applied); + assert_eq!( + fs::read_to_string(manager.path().join("cpuset.cpus")).unwrap(), + "2-7" + ); + } + + #[test] + fn an_equivalent_rendering_still_counts_as_applied() { + // The kernel is free to render the same set differently from the way + // it was written, so the comparison is over sets and not strings. + let (_dir, manager) = cpuset_tree("2,3,4,5,6,7\n"); + let layout = CpuLayout::with_core_count(8); + + assert_eq!(manager.apply_cpuset(&layout).unwrap(), Cpuset::Applied); + } + + #[test] + fn a_narrowed_memory_node_set_is_an_error() { + // The last write on this path that used to be taken on trust. + let (_dir, manager) = cpuset_tree_with_mems("2-7\n", "\n"); + let layout = CpuLayout::with_core_count(8); + + manager.apply_cpuset(&layout).unwrap_err(); + } + + #[test] + fn a_memory_node_set_that_cannot_be_read_back_fails_the_job() { + // This asserted a degrade until the rule was written down. The cpus were + // applied and verified, so the controller is demonstrably delegated; + // reporting it undelegated because the mems could not be read back would + // tell the operator something this function just disproved, and the run + // would go ahead on a memory binding nobody confirmed. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + fs::write(root.join("cpuset.cpus"), "").unwrap(); + fs::write(root.join("cpuset.mems"), "").unwrap(); + fs::write(root.join("cpuset.cpus.effective"), "2-7\n").unwrap(); + let manager = CgroupManager::detached(root); + let layout = CpuLayout::with_core_count(8); + + let err = manager.apply_cpuset(&layout).unwrap_err().to_string(); + + assert!( + err.contains("cpuset.mems.effective"), + "names the read that did not happen: {err}" + ); + } + + #[test] + fn an_undelegated_cpuset_controller_degrades() { + // A host that does not delegate cpuset creates its cgroup fine and + // then has no cpuset.cpus to write. That is a declared absence of + // isolation, not a failure, and no CI runner reaches it. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let manager = CgroupManager::detached(root); + let layout = CpuLayout::with_core_count(8); + + assert_eq!( + manager.apply_cpuset(&layout).unwrap(), + Cpuset::Unavailable(UNDELEGATED) + ); + } + + #[test] + fn a_silently_narrowed_cpuset_is_an_error() { + // The kernel intersects the written set with the parent's effective + // set and reports the result. A successful write proves nothing. + let (_dir, manager) = cpuset_tree("2-3\n"); + let layout = CpuLayout::with_core_count(8); + + let err = manager.apply_cpuset(&layout).unwrap_err().to_string(); + + assert!(err.contains("2-7"), "names what was asked for: {err}"); + assert!(err.contains("2-3"), "names what was granted: {err}"); + } + + #[test] + fn an_emptied_cpuset_is_an_error() { + // The worst case: narrowed to nothing, so the VMM inherits the + // parent's CPUs and the run silently measures the whole machine. + let (_dir, manager) = cpuset_tree("\n"); + let layout = CpuLayout::with_core_count(8); + + manager.apply_cpuset(&layout).unwrap_err(); + } + + #[test] + fn a_layout_with_no_isolation_claims_nothing() { + // Every variant that is not Applied has to carry a reason, so no + // path can report confinement without a verified write behind it. + let (_dir, manager) = cpuset_tree("0\n"); + let layout = CpuLayout::with_core_count(1); + + assert!(matches!( + manager.apply_cpuset(&layout).unwrap(), + Cpuset::Unavailable(_) + )); + } + + #[test] + fn parse_cpuset_reads_kernel_cpu_lists() { + assert_eq!(parse_cpuset("2-7"), (2..=7).collect()); + assert_eq!(parse_cpuset("2,3,4,5,6,7\n"), (2..=7).collect()); + assert_eq!(parse_cpuset("0-1,4,6-7"), [0, 1, 4, 6, 7].into()); + assert_eq!(parse_cpuset("3"), [3].into()); + assert!(parse_cpuset("").is_empty()); + assert!(parse_cpuset("\n").is_empty()); + } + + #[test] + fn a_cgroup_that_already_existed_is_not_ours_to_remove() { + // Drop removes what this created. Claiming a cgroup that was already + // there would have it rmdir something belonging to whoever did. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + let ours = CgroupManager { + cgroup_path: root.join("ours"), + created: true, + signals: JailSignals::unwatched(), + }; + let theirs = CgroupManager { + cgroup_path: root.join("theirs"), + created: false, + signals: JailSignals::unwatched(), + }; + fs::create_dir_all(ours.path()).unwrap(); + fs::create_dir_all(theirs.path()).unwrap(); + let theirs_path = theirs.path().to_owned(); + + drop(ours); + drop(theirs); + + assert!(!root.join("ours").exists(), "we remove what we created"); + assert!(theirs_path.exists(), "we leave what we did not create"); + } + + #[test] + fn procs_contains_pid_matches_whole_lines() { + assert!(procs_contains_pid("7\n70\n701\n", 7)); + assert!(procs_contains_pid("7\n70\n701\n", 701)); + // A prefix match must not count: pid 7 is not pid 70. + assert!(!procs_contains_pid("70\n701\n", 7)); + assert!(!procs_contains_pid("", 7)); + assert!(!procs_contains_pid("\n", 7)); + } + + #[test] + fn a_cgroup_that_will_not_go_away_raises_the_reclaim_signal() { + // The kernel refuses `rmdir` while a cgroup still holds a process, and + // a non-empty ordinary directory refuses it the same way. Warning alone + // would let the chroot that names this cgroup be removed, leaving + // nothing for a later sweep to find it by. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let signals = JailSignals::unwatched(); + let mut manager = CgroupManager { + cgroup_path: root.join("stuck"), + created: true, + signals: signals.clone(), + }; + fs::create_dir_all(manager.path()).unwrap(); + fs::write(manager.path().join("cgroup.procs"), "42\n").unwrap(); + + manager.cleanup(); + + assert!( + signals.must_keep_chroot(), + "a cgroup that outlives its job holds the chroot that names it" + ); + assert!(manager.path().exists()); + } + + #[test] + fn a_removed_cgroup_leaves_the_signal_alone() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let signals = JailSignals::unwatched(); + let mut manager = CgroupManager { + cgroup_path: root.join("gone"), + created: true, + signals: signals.clone(), + }; + fs::create_dir_all(manager.path()).unwrap(); + + manager.cleanup(); + + assert!(!manager.path().exists()); + assert!( + !signals.must_keep_chroot(), + "a clean teardown must not hold the chroot back" + ); + } + + #[test] + fn open_procs_reports_a_missing_cgroup() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let manager = CgroupManager::detached(root.join("absent")); + + manager.open_procs().unwrap_err(); + manager.contains_pid(1).unwrap_err(); + } + + #[test] + fn contains_pid_reads_the_listing() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + fs::write(root.join("cgroup.procs"), "123\n456\n").unwrap(); + let manager = CgroupManager::detached(root); + + assert!(manager.contains_pid(456).unwrap()); + assert!(!manager.contains_pid(789).unwrap()); + } + + #[test] + fn an_unreadable_node_set_is_not_node_zero() { + // The value is written to `cpuset.mems`, so answering node 0 to a read + // that failed would confine the guest to one node on a host that may + // have several, and the run would report a number measured under a + // constraint nobody chose. A file in place of the cgroup directory reads + // back `ENOTDIR`, the same way an unlistable one reads back `EACCES`. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let not_a_dir = root.join("cgroup"); + fs::write(¬_a_dir, b"in the way").unwrap(); + + effective_mems(¬_a_dir).unwrap_err(); + } + + #[test] + fn a_cpuset_that_cannot_be_verified_is_not_a_degrade() { + // The write proves the controller is delegated, so a missing effective + // file cannot mean it is not. Degrading here would tell the operator the + // controller was never delegated while the run went ahead on a cpuset + // nobody read back. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + fs::write(root.join("cpuset.cpus"), "").unwrap(); + fs::write(root.join("cpuset.mems"), "").unwrap(); + let manager = CgroupManager::detached(root); + let layout = CpuLayout::with_core_count(8); + + manager.apply_cpuset(&layout).unwrap_err(); + } + #[test] fn effective_mems_reads_file() { let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); fs::write(root.join("cpuset.mems.effective"), "0-1\n").unwrap(); - assert_eq!(effective_mems(&root), "0-1"); + assert_eq!(effective_mems(&root).unwrap(), "0-1"); } #[test] @@ -370,7 +836,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); - assert_eq!(effective_mems(&root), "0"); + assert_eq!(effective_mems(&root).unwrap(), "0"); } #[test] @@ -379,6 +845,6 @@ mod tests { let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); fs::write(root.join("cpuset.mems.effective"), "\n").unwrap(); - assert_eq!(effective_mems(&root), "0"); + assert_eq!(effective_mems(&root).unwrap(), "0"); } } diff --git a/plus/bencher_runner/src/jail/chroot.rs b/plus/bencher_runner/src/jail/chroot.rs new file mode 100644 index 000000000..87b15ddbb --- /dev/null +++ b/plus/bencher_runner/src/jail/chroot.rs @@ -0,0 +1,226 @@ +//! The per-job chroot the jailer confines Firecracker to. +//! +//! The runner builds the job's artifacts directly inside the chroot rather +//! than copying them in afterwards, which is legal because the jailer uses +//! `create_dir_all` for the chroot and does nothing if the path already +//! exists. Because the artifacts no longer live in a `TempDir`, this type +//! carries the cleanup responsibility that `TempDir` used to. + +#![expect(clippy::print_stderr, reason = "chroot teardown prints diagnostics")] + +use std::fs; +use std::os::unix::fs::{PermissionsExt as _, chown}; + +use camino::{Utf8Path, Utf8PathBuf}; + +use crate::error::JailError; +use crate::jail::{JailSignals, JailUser, StateDir, VmId}; + +/// A job's chroot tree, removed when this value is dropped. +/// +/// The jailer cleans up nothing by design, so teardown is the runner's job. +/// `Drop` covers completion, timeout, cancellation, and every error return; +/// the sweep in `prepare_host` covers the exits that never unwind. +/// +/// The cgroup's teardown runs first, which is what makes its signal meaningful +/// here: `run_firecracker` owns this job's cgroup and returns before this guard +/// is dropped. The signal read is this job's own, not the runner-wide one a +/// stale jail elsewhere can raise. +#[derive(Debug)] +pub struct JailDir { + dir: Utf8PathBuf, + root: Utf8PathBuf, + signals: JailSignals, +} + +impl JailDir { + /// Create the chroot tree for `vm_id` at mode 0700. + pub fn create(state: &StateDir, vm_id: &VmId, signals: JailSignals) -> Result { + let dir = state.jail_dir(vm_id); + let root = state.jail_root(vm_id); + + fs::create_dir_all(&root).map_err(|e| JailError::CreateJail { + path: root.clone(), + source: e, + })?; + // The jailer eventually sets the chroot root to 0700 owned by the jail + // user, but only once it runs. The runner builds the guest rootfs in + // here before that, so the tree is private from the moment it exists. + for path in [&dir, &root] { + fs::set_permissions(path, fs::Permissions::from_mode(0o700)).map_err(|e| { + JailError::CreateJail { + path: path.clone(), + source: e, + } + })?; + } + + Ok(Self { dir, root, signals }) + } + + /// The chroot root, which becomes `/` inside the jail. + #[must_use] + pub fn root(&self) -> &Utf8Path { + &self.root + } +} + +impl Drop for JailDir { + fn drop(&mut self) { + // A cgroup *this job* could not remove keeps the chroot alive. The two + // are named by the same id, and this directory is the only handle a + // later sweep has for finding that cgroup again, so removing it here + // would strand the cgroup for good with whatever is still in it. The + // same failure already earned the next job a sweep, which reclaims both + // in the right order. + // + // This job's cgroup, and nothing else: the runner-wide reclaim signal is + // also raised by a stale jail some sweep could not reclaim, which is no + // reason to keep a chroot whose own cgroup came down cleanly. + if self.signals.must_keep_chroot() { + eprintln!( + "Warning: leaving jail {} in place because its cgroup could not be removed. The directory names that cgroup, so the next job sweeps both.", + self.dir + ); + return; + } + + if let Err(e) = fs::remove_dir_all(&self.dir) + && e.kind() != std::io::ErrorKind::NotFound + { + eprintln!( + "Warning: failed to remove jail {}: {e}. It holds a VMM binary and a full guest rootfs; the next job will sweep it.", + self.dir + ); + self.signals.chroot_survived(); + } + } +} + +/// Let the jailed VMM read a file without giving it away. +/// +/// Firecracker only ever reads the kernel image, so it gets read permission +/// and nothing more: the file stays owned by root, which means the VMM cannot +/// write it and cannot chmod it into something it can write. The mode is set +/// explicitly rather than inherited, because a bundled write or a copy from +/// the host can land at 0600 and leave the VMM unable to read its own kernel. +pub fn grant_jail_read(path: &Utf8Path) -> Result<(), JailError> { + // Reported as a mode failure, not an ownership one. This function + // deliberately leaves the file owned by root, so an operator sent looking + // at ownership would be chasing the opposite of what went wrong. + fs::set_permissions(path, fs::Permissions::from_mode(0o644)).map_err(|e| JailError::ChmodJail { + path: path.to_owned(), + source: e, + }) +} + +/// Hand a file the runner placed inside the chroot to the jail uid and gid. +/// +/// The jailer chowns the chroot root and the device nodes it makes, but that +/// chown is not recursive: files the runner placed inside keep the ownership +/// they were created with, which is root. Every artifact Firecracker *writes* +/// has to be handed over explicitly, and getting it wrong produces an opaque +/// boot failure, so each one is checked. Anything it only reads gets +/// [`grant_jail_read`] instead. +pub fn chown_to_jail(path: &Utf8Path, jail_user: JailUser) -> Result<(), JailError> { + chown(path, Some(jail_user.uid()), Some(jail_user.gid())).map_err(|e| JailError::ChownJail { + path: path.to_owned(), + source: e, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A stand-in identity for tests. + fn vm_id() -> VmId { + VmId::from_chroot_name("vm-1".to_owned()) + } + + fn state_in_tmpdir() -> (tempfile::TempDir, StateDir) { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let state = StateDir::new(root.join("state")).unwrap(); + state.create().unwrap(); + (dir, state) + } + + #[test] + fn create_builds_a_private_chroot_tree() { + let (_dir, state) = state_in_tmpdir(); + + let jail = JailDir::create(&state, &vm_id(), JailSignals::unwatched()).unwrap(); + + assert_eq!(jail.root(), state.jail_root(&vm_id())); + assert!(jail.root().is_dir()); + for path in [state.jail_dir(&vm_id()), state.jail_root(&vm_id())] { + let mode = fs::metadata(&path).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o700, "{path} should be private"); + } + } + + #[test] + fn create_tolerates_an_existing_directory() { + let (_dir, state) = state_in_tmpdir(); + fs::create_dir_all(state.jail_root(&vm_id())).unwrap(); + + JailDir::create(&state, &vm_id(), JailSignals::unwatched()).unwrap(); + } + + #[test] + fn drop_removes_the_whole_tree() { + let (_dir, state) = state_in_tmpdir(); + + { + let jail = JailDir::create(&state, &vm_id(), JailSignals::unwatched()).unwrap(); + fs::write(jail.root().join("rootfs.ext4"), b"guest").unwrap(); + fs::create_dir_all(jail.root().join("dev")).unwrap(); + } + + assert!( + !state.jail_dir(&vm_id()).exists(), + "the chroot is the runner's to reclaim, not the jailer's" + ); + assert!(state.jail_parent().exists()); + } + + #[test] + fn an_unbuildable_chroot_is_an_error_not_a_warning() { + // A chroot that cannot be built is a confinement failure, so it has + // to abort the job rather than degrade into an unjailed run. + let (_dir, state) = state_in_tmpdir(); + // A file where the jail directory has to go makes the tree + // impossible to create. + fs::write(state.jail_dir(&vm_id()), b"in the way").unwrap(); + + JailDir::create(&state, &vm_id(), JailSignals::unwatched()).unwrap_err(); + } + + #[test] + fn a_surviving_cgroup_holds_the_chroot_that_names_it() { + // The cgroup is torn down first, and a removal it could not finish + // raises this. Removing the chroot anyway would leave nothing for a + // later sweep to find the cgroup by. + let (_dir, state) = state_in_tmpdir(); + let signals = JailSignals::unwatched(); + let jail = JailDir::create(&state, &vm_id(), signals.clone()).unwrap(); + fs::write(jail.root().join("rootfs.ext4"), b"guest").unwrap(); + + signals.cgroup_survived(); + drop(jail); + + assert!( + state.jail_dir(&vm_id()).exists(), + "the chroot names the cgroup that still has to be removed" + ); + } + + #[test] + fn drop_tolerates_an_already_removed_tree() { + let (_dir, state) = state_in_tmpdir(); + let jail = JailDir::create(&state, &vm_id(), JailSignals::unwatched()).unwrap(); + fs::remove_dir_all(state.jail_dir(&vm_id())).unwrap(); + drop(jail); + } +} diff --git a/plus/bencher_runner/src/jail/lock.rs b/plus/bencher_runner/src/jail/lock.rs new file mode 100644 index 000000000..cd3b7fc82 --- /dev/null +++ b/plus/bencher_runner/src/jail/lock.rs @@ -0,0 +1,181 @@ +//! Advisory lock serializing the jail lifecycle across runner processes. +//! +//! The sweep reclaims every chroot it finds, on the reasoning that jobs are +//! serial and so anything left is stale. That reasoning is only sound while it +//! is enforced: a one-shot `runner run` started on a host where the `runner up` +//! daemon has a job in flight would otherwise `remove_dir_all` the live chroot +//! out from under a running VMM. The same lock closes the race on the network +//! namespace handle, where two processes clearing and rebinding it at once can +//! stack mounts. +//! +//! Unlike the host tuning lock, which degrades to skipping tuning when it is +//! contended, this one waits. A runner that proceeded without it would destroy +//! another runner's work, so declining to hold it is not an option. + +#![expect(clippy::print_stdout, reason = "prints why the runner is waiting")] + +use std::fs::{File, OpenOptions}; +use std::os::fd::AsRawFd as _; + +use camino::Utf8Path; + +use crate::error::JailError; + +/// Lock file name inside the state directory. +/// +/// It lives beside the chroot base rather than inside it, so the sweep (which +/// only removes directories under `/jail/firecracker`) can never +/// reach it. +/// +/// Private again, and only spelled here. The state directory used to count this +/// name among the marks of a directory it owns, until proving ownership by a +/// name turned out to be the way a populated system directory could pass that +/// guard. Ownership is proven by the chroot tree now, so nothing outside this +/// module needs the name. +const LOCK_FILE: &str = ".lock"; + +/// Holds the jail lock for as long as it is alive. +/// +/// The kernel releases a `flock` when the holder exits or dies, so a crashed +/// runner cannot wedge future runs. +/// +/// `flock` is per open file description, not per process, so a second +/// `acquire` on the same path from a process that already holds it opens a new +/// description and blocks on itself forever. Nothing nests today: host +/// preparation takes and releases this lock before a job takes it, and the +/// network namespace uses a different lock file. Any new caller has to keep it +/// that way. +#[derive(Debug)] +pub struct JailLock { + /// The locked file, held only for its `flock`. + _file: File, +} + +impl JailLock { + /// Take the jail lock, waiting for whichever runner holds it. + /// + /// The state directory must already exist: the lock guards the contents, + /// so creating the directory is not something it can protect. + pub fn acquire(state_dir: &Utf8Path) -> Result { + let path = state_dir.join(LOCK_FILE); + let file = OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(&path) + .map_err(|e| JailError::OpenJailLock { + path: path.clone(), + source: e, + })?; + + // Try once without blocking, so waiting can be announced rather than + // looking like a hang. + if flock_nonblocking(&file).is_ok() { + return Ok(Self { _file: file }); + } + println!(" Waiting for another bencher runner to release {path}..."); + + flock_exclusive(&file).map_err(|e| JailError::JailLock { + path: path.clone(), + source: e, + })?; + + Ok(Self { _file: file }) + } +} + +/// Take an exclusive `flock`, waiting for whichever holder has it. +pub(super) fn flock_exclusive(file: &File) -> std::io::Result<()> { + flock(file, libc::LOCK_EX) +} + +/// Try for an exclusive `flock` without waiting. +/// +/// Shared with the network namespace lock so both locks announce a wait the same +/// way: an unexplained pause is the worst thing either of them can do. +pub(super) fn flock_nonblocking(file: &File) -> std::io::Result<()> { + flock(file, libc::LOCK_EX | libc::LOCK_NB) +} + +/// Apply `flock` to a file, retrying if a signal interrupts the wait. +fn flock(file: &File, operation: libc::c_int) -> std::io::Result<()> { + loop { + #[expect( + unsafe_code, + reason = "flock has no std wrapper; the fd is owned and valid" + )] + // SAFETY: `file` is an open, owned descriptor for the duration of the + // call; flock does not touch memory. + let ret = unsafe { libc::flock(file.as_raw_fd(), operation) }; + if ret == 0 { + return Ok(()); + } + let err = std::io::Error::last_os_error(); + if err.kind() != std::io::ErrorKind::Interrupted { + return Err(err); + } + } +} + +#[cfg(test)] +mod tests { + use camino::Utf8PathBuf; + + use super::*; + + fn state_in_tmpdir() -> (tempfile::TempDir, Utf8PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + (dir, root) + } + + #[test] + fn the_lock_can_be_taken() { + let (_dir, state) = state_in_tmpdir(); + + let lock = JailLock::acquire(&state).unwrap(); + + assert!(state.join(LOCK_FILE).exists()); + drop(lock); + } + + #[test] + fn a_released_lock_can_be_retaken() { + let (_dir, state) = state_in_tmpdir(); + drop(JailLock::acquire(&state).unwrap()); + + JailLock::acquire(&state).unwrap(); + } + + #[test] + fn a_held_lock_makes_a_second_runner_wait() { + // `flock` is per open file description, so a second `acquire` in this + // process contends exactly as another process would. The waiter is + // released only once the holder drops, which is what keeps a sweep + // from running while another runner has a job in flight. + let (_dir, state) = state_in_tmpdir(); + let held = JailLock::acquire(&state).unwrap(); + + let waiter = { + let state = state.clone(); + std::thread::spawn(move || JailLock::acquire(&state)) + }; + + // The waiter must still be blocked while the lock is held. + std::thread::sleep(std::time::Duration::from_millis(200)); + assert!( + !waiter.is_finished(), + "a second runner must wait rather than proceed to sweep" + ); + + drop(held); + waiter.join().unwrap().unwrap(); + } + + #[test] + fn a_missing_state_directory_is_an_error() { + let (_dir, state) = state_in_tmpdir(); + + JailLock::acquire(&state.join("absent")).unwrap_err(); + } +} diff --git a/plus/bencher_runner/src/jail/mod.rs b/plus/bencher_runner/src/jail/mod.rs index 558c03a84..a55a2c36a 100644 --- a/plus/bencher_runner/src/jail/mod.rs +++ b/plus/bencher_runner/src/jail/mod.rs @@ -1,187 +1,969 @@ -//! Resource management for Firecracker microVMs. +//! Confinement for Firecracker microVMs. //! -//! This module provides cgroup-based resource limits -//! for controlling Firecracker microVM processes. +//! Managed runners execute arbitrary code submitted by anyone, so the VMM must +//! not inherit the runner's root. This module owns everything that confines it: +//! the persistent state directory the chroots are built under, the empty +//! network namespace the VMM joins, and the cgroup that both places it on the +//! benchmark cores and bounds its resources. +//! +//! # What a failing step does +//! +//! Teardown is where this module is easiest to get wrong, because every step of +//! it runs after the thing it is cleaning up already happened, so a failure has +//! no obvious caller to tell. Measurement is easiest to get wrong for the +//! mirror-image reason: a read that fails still leaves a value in hand, and a +//! plausible one. Several separate defects here were the same mistake in those +//! two dresses, a fallible step whose failure either never reached the mechanism +//! built to retry it or was reported as a state that had been observed. So every +//! fallible step does exactly one of three things, and adding a step means +//! choosing which, rather than reaching for `eprintln!` or a fallback value: +//! +//! - **Fails the job.** The host cannot be trusted to measure. Used wherever +//! something may still be running on the benchmark cores, wherever the runner +//! could not establish that nothing is, and wherever a confinement it applied +//! could not be read back. Recoverable by construction: nothing latches, so +//! the next job tries the whole thing again. +//! - **Arms the retry, or declares the absence.** For teardown that is +//! [`ReclaimFailed`], read by [`HostPreparation::ensure`], used where the cost +//! is disk rather than a contended benchmark and where the code has no caller +//! to report to at all, which is every step reached from a `Drop`. For +//! measurement it is `Cpuset::Unavailable` or an absent metric: isolation the +//! host cannot offer is reported as isolation this run did not have, loudly, +//! and never as a number. +//! - **Ignored.** Only where the failure is itself the answer, or where a later +//! step is guaranteed to catch it. Each one below says which. +//! +//! What no step does is answer a question it could not ask. An unreadable file is +//! not an empty one, an unstattable path is not an absent one, and a field that +//! was never read is not zero. +//! +//! That covers reads which produce a value. It covers reads which gate an action +//! just as strictly, and those are the easier ones to miss: an `exists` before a +//! destructive step is a read whose failure authorizes destruction, and it does +//! not look like a measurement at all. A stat that fails before a `remove_dir` +//! is the same defect as a metric that fabricates a zero, and costs more. +//! +//! The rows below were found by grepping this crate for every idiom that can +//! turn a failure into an answer, rather than by reasoning about which functions +//! looked relevant. Reasoning missed instances three times running. +//! +//! Nothing enforces this table, deliberately. It is a specification a person +//! reads, and its value is that adding a step means picking a column and writing +//! the reason down next to the others. A checker that proved the code matched it +//! would be worth having, and is not worth building here: the gap is a decision, +//! not an oversight. +//! +//! | Step | On failure | +//! |---|---| +//! | [`HostPreparation::ensure`]: the root check | fails the job | +//! | [`HostPreparation::ensure`]: creating the state directory | fails the job | +//! | [`HostPreparation::ensure`]: reading `/etc/passwd`, `/etc/group` | ignored: the check is advisory and cannot see a directory service anyway | +//! | [`HostPreparation::ensure`]: taking the jail lock | fails the job | +//! | [`HostPreparation::ensure`]: a sweep that returns an error | fails the job | +//! | [`HostPreparation::ensure`]: a sweep that leaves a chroot behind | arms the retry | +//! | `sweep_jails`: the jail parent is absent | nothing to sweep | +//! | `sweep_jails`: the jail parent cannot be read | fails the job | +//! | `sweep_jails`: an entry cannot be read | fails the job | +//! | `sweep_jails`: an entry's kind cannot be read | fails the job: it may be a jail | +//! | `sweep_jails`: a name that is not UTF-8 | ignored: every name here is a UUID this runner minted, so it is not ours | +//! | `sweep_jails`: the reap reports a live VMM | fails the job | +//! | `sweep_jails`: the reap could not examine the jail | fails the job | +//! | `sweep_jails`: removing the cgroup | fails the job, and the chroot is kept because its name is the cgroup's only handle | +//! | `sweep_jails`: removing the chroot | arms the retry | +//! | `reap_jailed_vmm`: the jail root is absent | clear: nothing can be chrooted into a directory that is not there | +//! | `reap_jailed_vmm`: the jail root cannot be stat'ed | reported unexaminable, which fails the job | +//! | `reap_jailed_vmm`: `/proc` cannot be listed | reported unexaminable | +//! | `reap_jailed_vmm`: a `/proc//root` cannot be read | ignored: that process is gone or is not this jail's | +//! | `reap_jailed_vmm`: pinning or signalling the VMM | reported still running, which fails the job | +//! | `reap_jailed_vmm`: a VMM that will not exit | reported still running | +//! | `reap_jailed_vmm`: the rescan bound runs out | reported still running | +//! | `reap_jailed_vmm`: a `/proc//status` that cannot be read | ignored: the process is gone, which is what was being asked | +//! | `JailDir` teardown: the chroot is already gone | ignored: that is the goal state | +//! | `JailDir` teardown: removing the chroot | arms the retry | +//! | `JailDir` teardown: the retry is already armed | keeps the chroot, since its name is the cgroup's only handle | +//! | `CgroupManager` teardown: the cgroup cannot be stat'ed | arms the retry: it is treated as still there | +//! | `CgroupManager` teardown: `rmdir` of the cgroup | arms the retry, this job's and the runner's both | +//! | `CgroupManager` creation: the cgroup cannot be stat'ed | fails the job: this decides whether `Drop` may remove it | +//! | `remove_stale_cgroup`: the cgroup cannot be stat'ed | fails the job: the caller deletes the chroot on an `Ok` here | +//! | `StateDir::create`: the chroot tree cannot be stat'ed | fails the job: the 0700 chmod follows | +//! +//! And the same three columns for the reads that decide what a run measured: +//! +//! | Step | On failure | +//! |---|---| +//! | `apply_cpuset`: no isolation in the layout, or an empty core set | declares the absence | +//! | `apply_cpuset`: `cpuset.cpus` is absent | declares the absence: the controller is not delegated | +//! | `apply_cpuset`: `cpuset.cpus` cannot be stat'ed | fails the job: an error is not an absence | +//! | `apply_cpuset`: the kernel rejects a cpuset write | fails the job: half-applied confinement | +//! | `apply_cpuset`: the parent's node set is absent | node 0, which is what an undelegated controller means | +//! | `apply_cpuset`: the parent's node set cannot be read | fails the job: this value is written, so a guess confines the guest's memory | +//! | `apply_cpuset`: the effective set cannot be read back | fails the job: the read back *is* the mechanism, and the write already proved the controller delegated | +//! | `apply_cpuset`: the kernel narrowed the set | fails the job | +//! | `enable_controllers`: `cgroup.subtree_control` cannot be read | fails the job: an unreadable list is not an empty one | +//! | `BencherPartition::apply`: any read or write in the partition path | declares the absence: the level achieved is reported, down to `member` | +//! | `CpuLayout::detect`: the online CPU list cannot be read or parsed | declares the absence: the counted layout is announced as a guess | +//! | `CpuLayout::detect`: the core count cannot be read | one core, which reports as a layout with no isolation | +//! | `metrics`: a cgroup that is not there | no metrics, reported as absent | +//! | `metrics`: a field that cannot be read or parsed | absent, never zero | +//! | `tuning::preflight`: any check that cannot be performed | ignored: advisory only, and nothing reads it to decide whether the host can measure. A quiet preflight is not evidence of a quiet host | +//! | `pin_vcpu_threads`: a thread list that cannot be read | declares the absence: it reports how many of the vCPUs it pinned | +//! | `FirecrackerClient`: a status line that cannot be parsed | fails the request: no status is invented for a response Firecracker did not send | +//! | `find_binary`: a candidate path that cannot be stat'ed | ignored: the search is a list of guesses, and finding nothing is reported by name | +//! +//! `CgroupManager::kill_all` is deliberately not in either table. +//! Nothing on the jailed path calls it: the VMM is killed with a grace period +//! before its cgroup is torn down, and anything that somehow survives is caught +//! by the `rmdir` above, which arms the retry. The non-sandboxed path calls it +//! explicitly on timeout or cancellation, where a failure is warned and then +//! caught the same way. A row saying teardown kills survivors described a step +//! that does not exist, which is worse than no row at all: this table is read as +//! a specification. #[cfg(target_os = "linux")] mod cgroup; - #[cfg(target_os = "linux")] -pub use cgroup::CgroupManager; +pub mod chroot; +#[cfg(target_os = "linux")] +pub mod lock; +#[cfg(target_os = "linux")] +pub mod netns; +#[cfg(target_os = "linux")] +pub mod paths; +#[cfg(target_os = "linux")] +pub mod reap; +#[cfg(target_os = "linux")] +pub mod state; + #[cfg(target_os = "linux")] pub(crate) use cgroup::{BENCHER_CGROUP_BASE, effective_mems}; +#[cfg(target_os = "linux")] +pub use cgroup::{CgroupManager, Cpuset}; +#[cfg(target_os = "linux")] +pub use chroot::JailDir; +#[cfg(target_os = "linux")] +pub use lock::JailLock; +#[cfg(target_os = "linux")] +pub use paths::{ChrootPath, HostPath, JailFile, JailPaths, SocketPath}; +#[cfg(target_os = "linux")] +pub use state::StateDir; -use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; -/// Resource limits for the Firecracker microVM process. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ResourceLimits { - /// Maximum CPU time in microseconds per second. - /// E.g., 100000 = 100ms per 100ms period = 1 full CPU. - #[serde(default)] - pub cpu_quota_us: Option, +/// Default location of the runner's persistent state directory. +pub const DEFAULT_STATE_DIR: &str = "/var/lib/bencher-runner"; - /// CPU period in microseconds (default: 100000 = 100ms). - #[serde(default = "default_cpu_period")] - pub cpu_period_us: u64, +/// Default unprivileged uid the jailed Firecracker VMM runs as. +/// +/// One dedicated id, not one per job: jobs run serially and each gets a fresh +/// chroot that is swept, so a per-job allocator adds a scheme without closing +/// a live vector. +/// +/// Serially per runner, which is what the jail lock enforces. Two runners with +/// different `--state-dir` values hold different locks and can therefore have +/// jobs in flight at the same time, as the same uid, and that is a supported +/// configuration: the network namespace module is built around it. Two VMMs +/// sharing a uid can signal each other and, where `ptrace_scope` permits, trace +/// each other, which is the hazard [`JailUser`] describes. A per-runner id would +/// close it, and is not attempted here: the ids would have to be allocated +/// without coordination between runners that by construction do not know about +/// each other, which trades a known hazard for an unknown one. An operator +/// running two runners on one host should give them different `--jail-uid` +/// values, which is what that flag is for. +/// +/// The number is Bencher's historic default self-hosted API server port, +/// retired in favor of the IANA-registered 6610, so it reads as a project +/// convention rather than an arbitrary pick. It also lands in the unallocated +/// gap between the ids `systemd-homed` claims (60001-60513) and the +/// `DynamicUser` range (61184-65519), clear of both the regular user range and +/// `nobody` (65534). No passwd entry is needed: the jailer sets the numeric id +/// directly. +/// +/// This is a default rather than a fixed constant because self-hosted runners +/// land on hardware whose id allocation Bencher does not control. See +/// `--jail-uid`. +pub const DEFAULT_JAIL_UID: u32 = 61016; - /// Maximum memory in bytes. - #[serde(default)] - pub memory_bytes: Option, +/// Default unprivileged gid the jailed Firecracker VMM runs as. +/// +/// See [`DEFAULT_JAIL_UID`]. +pub const DEFAULT_JAIL_GID: u32 = 61016; - /// Maximum number of open file descriptors. - #[serde(default = "default_max_fds")] - pub max_fds: u64, +/// The unprivileged uid and gid the jailed Firecracker VMM drops to. +/// +/// A host process owning this uid can signal the VMM and, depending on the +/// `ptrace` scope, trace it, so it must not be an id the host allocates to +/// anything else. +/// +/// The fields are private because `0` must never reach them. The whole +/// sandbox is built by dropping privilege, so a jail user of root is not a +/// weaker jail, it is no jail at all: untrusted code would run against a root +/// VMM, which is the one thing the confinement exists to prevent. An operator +/// hitting a permission error is exactly the person most likely to try it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JailUser { + uid: u32, + gid: u32, +} - /// Maximum number of processes/threads. - #[serde(default = "default_max_procs")] - pub max_procs: u64, +impl JailUser { + /// Build a jail user, rejecting root. + pub fn new(uid: u32, gid: u32) -> Result { + if uid == 0 { + return Err(crate::error::JailError::PrivilegedJailUser { field: "uid" }); + } + if gid == 0 { + return Err(crate::error::JailError::PrivilegedJailUser { field: "gid" }); + } + Ok(Self { uid, gid }) + } - /// Maximum I/O read bandwidth in bytes per second. - /// Applied via cgroup v2 io.max. - #[serde(default)] - pub io_read_bps: Option, + /// The uid the VMM drops to. + #[must_use] + pub fn uid(self) -> u32 { + self.uid + } - /// Maximum I/O write bandwidth in bytes per second. - /// Applied via cgroup v2 io.max. - #[serde(default)] - pub io_write_bps: Option, + /// The gid the VMM drops to. + #[must_use] + pub fn gid(self) -> u32 { + self.gid + } } -const fn default_cpu_period() -> u64 { - 100_000 // 100ms +impl Default for JailUser { + fn default() -> Self { + Self { + uid: DEFAULT_JAIL_UID, + gid: DEFAULT_JAIL_GID, + } + } } -const fn default_max_fds() -> u64 { - 1024 +/// Require an absolute state directory. +/// +/// The invariant belongs to the library, not to one command line. The path +/// reaches the jailer as `--chroot-base-dir`, which the jailer resolves against +/// its own working directory rather than the runner's, so a relative value +/// builds the chroot somewhere the runner does not look: the sweep never reaches +/// it and the lock does not protect it. Every caller that hands this crate a +/// state directory is exposed to that, and the CLI is only one of them. +/// +/// Shared with the argument parser rather than restated there, so the rule has +/// one implementation and the operator still hears about it before the runner +/// starts. +pub fn check_absolute_state_dir(path: &camino::Utf8Path) -> Result<(), crate::error::JailError> { + if path.is_absolute() { + Ok(()) + } else { + Err(crate::error::JailError::RelativeStateDir { + path: path.to_owned(), + }) + } } -const fn default_max_procs() -> u64 { - 64 +/// The identity of one microVM. +/// +/// The same string is the jailer's `--id`, the name of the chroot directory, +/// and the name of the cgroup, by construction. Naming it once keeps the three +/// from drifting, and keeps a bare directory name read off the filesystem from +/// being mistaken for an identity that was minted. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VmId(String); + +impl VmId { + /// Mint a fresh identity for a job. + #[must_use] + pub fn new() -> Self { + Self(uuid::Uuid::new_v4().to_string()) + } + + /// Recover the identity of a jail from its chroot directory name. + /// + /// The sweep works backwards from the filesystem, and the directory name + /// is the identity that created it. + #[must_use] + pub fn from_chroot_name(name: String) -> Self { + Self(name) + } + + /// The identity as a string. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } } -impl Default for ResourceLimits { +impl Default for VmId { fn default() -> Self { - Self { - cpu_quota_us: None, - cpu_period_us: default_cpu_period(), - memory_bytes: None, - max_fds: default_max_fds(), - max_procs: default_max_procs(), - io_read_bps: None, - io_write_bps: None, - } + Self::new() + } +} + +impl std::fmt::Display for VmId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) } } -impl ResourceLimits { - /// Set CPU limit as a fraction of CPUs (e.g., 0.5 = half a CPU, 2.0 = 2 CPUs). +/// Tracks whether this runner process has prepared the host. +/// +/// Owned and threaded through the callers rather than kept in a global, +/// so that the latch belongs to one runner and cannot be observed, or reset, +/// by anything else. Tests get their own. +#[derive(Debug, Default)] +pub struct HostPreparation { + /// Whether the jail user warning has already been given. + /// + /// Advisory and unchanging, so it is worth saying once and not once a job. + /// Set where the warning is printed, so a preparation that fails afterwards + /// does not earn the operator the same advice again. + #[cfg_attr( + not(target_os = "linux"), + expect(dead_code, reason = "host preparation is Linux-only") + )] + warned_jail_user: bool, + /// Only the jail reads this, and the jail is Linux-only. + #[cfg_attr( + not(target_os = "linux"), + expect(dead_code, reason = "host preparation is Linux-only") + )] + prepared: bool, + /// Set when a job's teardown could not reclaim its chroot. + reclaim_failed: ReclaimFailed, +} + +/// Shared signal that a jail could not be reclaimed. +/// +/// `Drop` has nowhere to report a failure, and a chroot that outlives its job +/// holds a copy of the VMM binary and a full guest rootfs. Because the sweep +/// otherwise runs once per process, a long-lived daemon would carry that leak +/// until a restart. Setting this makes the next job sweep again, which is the +/// mechanism that already exists for exactly this. +/// +/// Owned by the runner's [`HostPreparation`] and cloned into each jail, never +/// global. +#[derive(Debug, Clone, Default)] +pub struct ReclaimFailed(Arc); + +impl ReclaimFailed { + /// Record that a jail could not be reclaimed. + pub fn set(&self) { + self.0.store(true, Ordering::SeqCst); + } + + /// Consume the signal. + /// + /// Deliberately not one swap together with [`Self::is_set`]. The signal asks + /// for a sweep, so it is spent only once a sweep has actually finished: + /// spending it up front would let a sweep that failed clear the one thing + /// that makes a later job try again. Jobs are serial, so nothing can raise + /// it between the sweep finishing and this call. + /// + /// Only the jail reads it, and the jail is Linux-only. + #[cfg(target_os = "linux")] + fn clear(&self) { + self.0.store(false, Ordering::SeqCst); + } + + /// Whether the signal is set, without consuming it. + /// + /// Read by the chroot teardown, which must not remove a directory whose + /// cgroup is still there: the directory name is the only handle a later + /// sweep has for finding that cgroup again. + #[cfg(target_os = "linux")] #[must_use] - #[expect( - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - clippy::cast_precision_loss, - reason = "CPU fraction to microsecond quota conversion" + pub(crate) fn is_set(&self) -> bool { + self.0.load(Ordering::SeqCst) + } +} + +/// Whether this job's own cgroup outlived its teardown. +/// +/// Job-scoped, where [`ReclaimFailed`] is runner-scoped, and that distinction is +/// the point. The chroot has to be kept when *this* job's cgroup could not be +/// removed, because the directory name is the only handle a later sweep has for +/// finding that cgroup. A runner-wide signal cannot say that: it is also raised +/// by an unrelated stale jail the sweep could not reclaim, and reading it made +/// the current job hold a chroot whose own cgroup came down cleanly. Two +/// questions on one flag is also what made the earlier ordering bugs easy to +/// write. +#[derive(Debug, Clone, Default)] +pub struct CgroupSurvived(Arc); + +impl CgroupSurvived { + /// Record that this job's cgroup could not be removed. + pub fn set(&self) { + self.0.store(true, Ordering::SeqCst); + } + + /// Whether this job's cgroup is still there. + /// + /// Only the chroot teardown reads it, and the jail is Linux-only. + #[cfg(target_os = "linux")] + fn is_set(&self) -> bool { + self.0.load(Ordering::SeqCst) + } +} + +/// The signals one job's teardown raises. +/// +/// Carried together because every teardown step needs both: a cgroup that will +/// not go away has to hold this job's chroot *and* earn the next job a sweep. +/// Keeping them in one value with two names is what stops the two being confused +/// for each other again. +#[derive(Debug, Clone)] +pub struct JailSignals { + /// Runner-wide: a later job owes another sweep. + /// + /// Only the jail raises and reads these, and the jail is Linux-only. + #[cfg_attr( + not(target_os = "linux"), + expect(dead_code, reason = "the jail is Linux-only") + )] + reclaim_failed: ReclaimFailed, + /// This job only: its cgroup outlived its teardown. + #[cfg_attr( + not(target_os = "linux"), + expect(dead_code, reason = "the jail is Linux-only") )] - pub fn with_cpu_limit(mut self, cpus: f64) -> Self { - let quota = (cpus * self.cpu_period_us as f64) as u64; - self.cpu_quota_us = Some(quota); - self + cgroup_survived: CgroupSurvived, +} + +impl JailSignals { + /// The signals for one job, sharing the runner's reclaim signal. + #[must_use] + pub fn for_job(reclaim_failed: ReclaimFailed) -> Self { + Self { + reclaim_failed, + cgroup_survived: CgroupSurvived::default(), + } + } + + /// Signals nothing reads, for a cgroup no sweep can find again. + /// + /// A non-sandboxed run has no chroot to hold and no directory the sweep + /// walks, so there is no handle for a later sweep to work from and nothing a + /// raised signal could change. Named rather than defaulted so the call site + /// says which of the two it is. + #[cfg(target_os = "linux")] + #[must_use] + pub fn unwatched() -> Self { + Self::for_job(ReclaimFailed::default()) + } + + /// Record that this job's cgroup could not be removed. + /// + /// Both signals, because both are true: the chroot that names this cgroup + /// has to stay, and the next job has to sweep for it. + #[cfg(target_os = "linux")] + pub fn cgroup_survived(&self) { + self.cgroup_survived.set(); + self.reclaim_failed.set(); + } + + /// Record that this job's chroot could not be removed. + #[cfg(target_os = "linux")] + pub fn chroot_survived(&self) { + self.reclaim_failed.set(); + } + + /// Whether this job's cgroup is still there, so its chroot must stay. + #[cfg(target_os = "linux")] + #[must_use] + pub(crate) fn must_keep_chroot(&self) -> bool { + self.cgroup_survived.is_set() } +} - /// Set memory limit in bytes. +impl HostPreparation { + /// A runner process that has not prepared the host yet. #[must_use] - pub fn with_memory_limit(mut self, bytes: u64) -> Self { - self.memory_bytes = Some(bytes); - self + pub fn new() -> Self { + Self::default() } - /// Set I/O bandwidth limits in bytes per second. + /// A handle each jail uses to report that it could not be reclaimed. #[must_use] - pub fn with_io_limits(mut self, read_bps: u64, write_bps: u64) -> Self { - self.io_read_bps = Some(read_bps); - self.io_write_bps = Some(write_bps); - self + pub fn reclaim_signal(&self) -> ReclaimFailed { + self.reclaim_failed.clone() + } + + /// Prepare the host for jailed execution, at most once. + /// + /// Called on demand, immediately before the first job builds a jail, never + /// at startup. The daemon learns which Specs it serves from the server, so + /// at startup it cannot know whether it will ever need a jail, and a + /// Runner that serves only non-sandboxed Specs is a supported + /// configuration that must come up on a host where the runner is not root. + /// Preparing eagerly would make `runner up` require root just to start. + /// + /// Failure is fatal. Untrusted code never runs with silently degraded + /// confinement, so a host that cannot be prepared does not execute a job. + /// A failure is not remembered, so the next job retries rather than + /// needing a restart. That matters most for a sweep that could not reclaim + /// a stale cgroup: the orphan may simply not have exited yet, and a + /// latched failure would leave every later job failing with no retry. + /// + /// Requires root, and says so by name rather than letting the operator + /// infer it from a permission error several layers down. + #[cfg(target_os = "linux")] + pub fn ensure( + &mut self, + state_dir: &camino::Utf8Path, + jail_user: JailUser, + ) -> Result<(), crate::error::JailError> { + self.ensure_as(current_euid(), state_dir, jail_user) + } + + /// Prepare the host, with the effective uid supplied. + /// + /// The uid is a parameter for the same reason the sweep's reap is one: the + /// check refuses every uid but root, so a test that had to be root to reach + /// anything past it would be exercising the harness rather than this. + #[cfg(target_os = "linux")] + fn ensure_as( + &mut self, + euid: u32, + state_dir: &camino::Utf8Path, + jail_user: JailUser, + ) -> Result<(), crate::error::JailError> { + // A jail that could not be reclaimed earns another sweep, whatever + // this process has already done. + if self.prepared && !self.reclaim_failed.is_set() { + return Ok(()); + } + // The flag is passed in rather than set after, because preparation can + // fail at a step past the warning: recording it here would repeat the + // advice on every job of a host whose sweep keeps failing, and recording + // it only on success would do the same. + let swept = prepare_host(euid, state_dir, jail_user, &mut self.warned_jail_user)?; + // Spent only on a sweep that finished, and armed by one that did not. + // Spending it any earlier disarms the mechanism precisely when it is + // needed: the signal would be gone, this process would still count as + // prepared, and every later job would return early while the jail nobody + // could reclaim sat there. That is true of a sweep that failed outright, + // which never reaches here, and equally of one that returned `Ok` having + // left a chroot on disk. + if swept.is_complete() { + self.reclaim_failed.clear(); + } else { + self.reclaim_failed.set(); + } + self.prepared = true; + Ok(()) + } + + /// Prepare the host for jailed execution, at most once. + /// + /// The jail is Linux-only, as is the VM executor it protects. + #[cfg(not(target_os = "linux"))] + pub fn ensure( + &mut self, + _state_dir: &camino::Utf8Path, + _jail_user: JailUser, + ) -> Result<(), crate::error::JailError> { + Ok(()) } } -#[cfg(test)] +/// Create the state directory and reclaim what a previous runner left behind. +/// +/// The sweep is taken under the jail lock: it removes every chroot it finds on +/// the reasoning that jobs are serial, so it must not run while another runner +/// has one in flight. Running before any jail exists in this process is what +/// the sweep's purpose actually requires. +/// +/// The network namespace is deliberately not built here. It is a process- +/// global object on a tmpfs, so it is rebuilt per job rather than once per +/// daemon lifetime. +#[cfg(target_os = "linux")] +#[expect(clippy::print_stdout, reason = "host preparation reports what it did")] +fn prepare_host( + euid: u32, + state_dir: &camino::Utf8Path, + jail_user: JailUser, + jail_user_announced: &mut bool, +) -> Result { + // Checked first, and by name. Without it the most likely upgrade failure + // surfaces as a permission error on a directory, or a bare EPERM out of + // `unshare`, neither of which mentions root or the flag that avoids it. + check_root(euid)?; + + let state = StateDir::new(state_dir.to_owned())?; + state.create()?; + + // Once per runner, not once per job, and recorded at the moment it is given + // rather than at the end of a preparation that may not reach one. This runs + // again whenever a sweep is owed, and a host that keeps failing to reclaim a + // jail would otherwise repeat the advice until an operator stopped reading + // any of it. + if !*jail_user_announced { + warn_on_named_account(jail_user); + *jail_user_announced = true; + } + + let _lock = JailLock::acquire(state.path())?; + let swept = state::sweep_jails(&state.jail_parent())?; + if swept.reclaimed() > 0 { + // Each one held a copy of the VMM binary and a full guest rootfs + // image, so an operator should hear about it. + println!( + " Reclaimed {} stale jail(s) from {state_dir}", + swept.reclaimed() + ); + } + Ok(swept) +} + +/// Refuse to build a jail without the privileges building one needs. +/// +/// The failure an operator actually hits on upgrade is this one, so it says +/// what the release notes say rather than leaving them to infer it from a +/// permission error several layers down. +#[cfg(target_os = "linux")] +fn check_root(euid: u32) -> Result<(), crate::error::JailError> { + if euid == 0 { + Ok(()) + } else { + Err(crate::error::JailError::NotRoot { euid }) + } +} + +/// The effective uid of this process. +#[cfg(target_os = "linux")] +#[expect( + unsafe_code, + reason = "geteuid has no std wrapper and cannot fail or touch memory" +)] +fn current_euid() -> u32 { + // SAFETY: `geteuid` takes no arguments, returns a plain integer, and is + // documented as always succeeding. + unsafe { libc::geteuid() } +} + +/// Warn when the jail uid or gid belongs to a named account. +/// +/// The jailer needs no passwd entry, so a name resolving here is the cheap +/// signal that the host allocates ids in this range: whatever owns that +/// account can signal the VMM and may be able to trace it. A warning rather +/// than a refusal, because an operator who deliberately created the account is +/// a legitimate setup and only they can tell the two apart. +#[cfg(target_os = "linux")] +#[expect(clippy::print_stderr, reason = "host preparation prints diagnostics")] +fn warn_on_named_account(jail_user: JailUser) { + let (uid, gid) = (jail_user.uid(), jail_user.gid()); + if let Some(name) = passwd_name(uid) { + eprintln!( + "Warning: jail uid {uid} belongs to the existing account '{name}'. That account can signal the jailed VMM; pass --jail-uid to pick an unallocated id." + ); + } + if let Some(name) = group_name(gid) { + eprintln!( + "Warning: jail gid {gid} belongs to the existing group '{name}'. Pass --jail-gid to pick an unallocated id." + ); + } +} + +/// The account name for a uid, read from `/etc/passwd`. +/// +/// Best effort, and blind to anything the local files do not know about: a +/// host backed by LDAP, Active Directory, or SSSD allocates ids that never +/// appear here, and those are the hosts most likely to allocate in this range +/// at all. Deliberately not a `getpwuid` call even so, because the runner +/// ships as a self-contained binary and NSS would make it depend on the host's +/// resolver configuration. This catches the cheap case; it is not a guarantee +/// that the id is unallocated. +#[cfg(target_os = "linux")] +fn passwd_name(uid: u32) -> Option { + lookup_name("/etc/passwd", uid) +} + +/// The group name for a gid, read from `/etc/group`. +#[cfg(target_os = "linux")] +fn group_name(gid: u32) -> Option { + lookup_name("/etc/group", gid) +} + +/// Find the name whose record carries `id` in a colon-separated database. +/// +/// Both `/etc/passwd` and `/etc/group` put the name first and the numeric id +/// third. +#[cfg(target_os = "linux")] +fn lookup_name(path: &str, id: u32) -> Option { + let database = std::fs::read_to_string(path).ok()?; + lookup_name_in(&database, id) +} + +/// Find the name whose record carries `id`, given the database contents. +#[cfg(target_os = "linux")] +fn lookup_name_in(database: &str, id: u32) -> Option { + database.lines().find_map(|line| { + let mut fields = line.split(':'); + let name = fields.next()?; + let _password = fields.next()?; + (fields.next()?.parse::().ok()? == id).then(|| name.to_owned()) + }) +} + +// Everything the jail prepares is Linux-only, and so is every test of it. +#[cfg(all(test, target_os = "linux"))] mod tests { use super::*; + /// The only uid that can build a jail. + /// + /// Supplied rather than inherited from the test process, which is usually + /// not root and would otherwise never reach the preparation being tested. + const ROOT_EUID: u32 = 0; + #[test] - fn resource_limits_defaults() { - let limits = ResourceLimits::default(); - assert_eq!(limits.cpu_quota_us, None); - assert_eq!(limits.cpu_period_us, 100_000); - assert_eq!(limits.memory_bytes, None); - assert_eq!(limits.max_fds, 1024); - assert_eq!(limits.max_procs, 64); - assert_eq!(limits.io_read_bps, None); - assert_eq!(limits.io_write_bps, None); + fn a_runner_that_is_not_root_is_refused_by_name() { + // The upgrade failure an operator actually hits. Without this it + // surfaces as a permission error on a directory, or a bare EPERM out of + // `unshare`, neither of which mentions root or the flag that avoids it. + let dir = tempfile::tempdir().unwrap(); + let root = camino::Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let state_dir = root.join("state"); + + let mut host = HostPreparation::new(); + // Nothing latches, so every job says it again rather than only the + // first one. + for attempt in 1..=3 { + let err = host + .ensure_as(1000, &state_dir, JailUser::default()) + .unwrap_err(); + let message = err.to_string(); + assert!( + message.contains("1000"), + "attempt {attempt} must name the uid: {message}" + ); + assert!( + message.contains("root"), + "attempt {attempt} must name root: {message}" + ); + // Both escapes, because both subcommands reach this error and each + // has only one of them: `--danger-allow-no-sandbox` exists on + // `runner up`, and a one-shot `runner run` gives up the sandbox by + // omitting `--sandbox`. Naming only the daemon's flag sends a + // `runner run` operator to an argument it does not accept. + assert!( + message.contains("--danger-allow-no-sandbox"), + "attempt {attempt} must name the daemon's escape hatch: {message}" + ); + assert!( + message.contains("without --sandbox"), + "attempt {attempt} must name the one-shot escape hatch: {message}" + ); + } + + assert!( + !state_dir.exists(), + "a refused runner must not have touched the state directory" + ); } #[test] - fn with_cpu_limit_one_cpu() { - let limits = ResourceLimits::default().with_cpu_limit(1.0); - assert_eq!(limits.cpu_quota_us, Some(100_000)); + fn the_default_jail_user_is_outside_the_allocated_ranges() { + // systemd-homed takes 60001-60513 and DynamicUser takes 61184-65519. + // An id inside either would collide with something the host allocates. + for id in [DEFAULT_JAIL_UID, DEFAULT_JAIL_GID] { + assert_eq!(id, 61016, "the jail id is a project convention"); + assert!(id > 60513, "{id} must clear the systemd-homed range"); + assert!(id < 61184, "{id} must clear the DynamicUser range"); + } } #[test] - fn with_cpu_limit_half_cpu() { - let limits = ResourceLimits::default().with_cpu_limit(0.5); - assert_eq!(limits.cpu_quota_us, Some(50_000)); + fn preparation_is_lazy_and_happens_at_most_once() { + // A daemon that prepared at startup would need root just to come up, + // which breaks a Runner serving only non-sandboxed Specs. Nothing may + // touch the state directory until a job actually builds a jail. + let dir = tempfile::tempdir().unwrap(); + let root = camino::Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let state_dir = root.join("state"); + assert!(!state_dir.exists(), "startup has not prepared anything"); + + let mut host = HostPreparation::new(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + assert!(state_dir.join("jail").is_dir(), "the first job prepares"); + + // A second job must not redo it: proven by removing the tree and + // seeing that it is not rebuilt. Owning the token is what makes this + // independent of every other test in the process. + std::fs::remove_dir_all(&state_dir).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + assert!(!state_dir.exists(), "preparation happens at most once"); + + // A different runner prepares its own host. + let mut other = HostPreparation::new(); + other + .ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + assert!( + state_dir.join("jail").is_dir(), + "a fresh token prepares again" + ); } #[test] - fn with_cpu_limit_two_cpus() { - let limits = ResourceLimits::default().with_cpu_limit(2.0); - assert_eq!(limits.cpu_quota_us, Some(200_000)); + fn a_failure_is_not_latched_and_self_heals() { + // Fatal to the job, not to the runner. A host that cannot be prepared + // has to fail every job that needs a jail, and recover on its own the + // moment the cause goes away, rather than wedging until a restart. + let dir = tempfile::tempdir().unwrap(); + let root = camino::Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let state_dir = root.join("state"); + // A populated directory the runner did not create is refused. + std::fs::create_dir_all(state_dir.join("someone-elses-data")).unwrap(); + + let mut host = HostPreparation::new(); + for attempt in 1..=3 { + assert!( + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .is_err(), + "attempt {attempt} must fail" + ); + } + + // Remove the cause and the very next job succeeds, with no restart. + std::fs::remove_dir(state_dir.join("someone-elses-data")).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + assert!(state_dir.join("jail").is_dir()); } #[test] - fn with_memory_limit() { - let limits = ResourceLimits::default().with_memory_limit(1024 * 1024 * 512); - assert_eq!(limits.memory_bytes, Some(0x2000_0000)); + fn a_jail_that_could_not_be_reclaimed_earns_another_sweep() { + // The sweep otherwise runs once per process, so a teardown that failed + // in a long-lived daemon would leak a chroot holding a VMM copy and a + // full guest rootfs until a restart. Drop has nowhere to report, so it + // raises this instead. + let dir = tempfile::tempdir().unwrap(); + let root = camino::Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let state_dir = root.join("state"); + + let mut host = HostPreparation::new(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + + // Already prepared: a second job does not redo the work. + std::fs::remove_dir_all(&state_dir).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + assert!(!state_dir.exists(), "preparation happens at most once"); + + // A jail that could not be reclaimed changes that. + host.reclaim_signal().set(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + assert!( + state_dir.join("jail").is_dir(), + "a failed teardown must earn another sweep" + ); + + // And the signal is consumed, not sticky. + std::fs::remove_dir_all(&state_dir).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + assert!(!state_dir.exists(), "the signal is consumed once"); } #[test] - fn with_io_limits() { - let limits = ResourceLimits::default().with_io_limits(1_000_000, 500_000); - assert_eq!(limits.io_read_bps, Some(1_000_000)); - assert_eq!(limits.io_write_bps, Some(500_000)); + fn a_sweep_that_failed_does_not_disarm_the_next_one() { + // The signal is spent on a sweep that finished, never on one that was + // merely attempted. Consuming it up front costs nothing on the first + // failure and everything on the second: this process still counts as + // prepared, so with the signal gone every later job returns early and + // the jail that could not be reclaimed is never swept again for the + // lifetime of the daemon. If the reason it could not be reclaimed is a + // VMM still in its cgroup, that orphan holds the benchmark cores while + // every later job measures through it and reports clean. + let dir = tempfile::tempdir().unwrap(); + let root = camino::Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let state_dir = root.join("state"); + + let mut host = HostPreparation::new(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + + // A teardown that could not reclaim its jail asks for another sweep. + host.reclaim_signal().set(); + + // Preparation now fails, twice, with the signal still outstanding. + std::fs::remove_dir_all(&state_dir).unwrap(); + std::fs::create_dir_all(state_dir.join("someone-elses-data")).unwrap(); + for attempt in 1..=2 { + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap_err(); + assert!( + host.reclaim_signal().is_set(), + "attempt {attempt} failed, so the sweep it asked for is still owed" + ); + } + + // Remove the cause: the next job must still sweep rather than return + // early on the strength of a signal an earlier failure ate. + std::fs::remove_dir_all(&state_dir).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + assert!( + state_dir.join("jail").is_dir(), + "the owed sweep must survive every failed attempt at it" + ); + + // And it is spent now that one has finished. + assert!(!host.reclaim_signal().is_set()); + std::fs::remove_dir_all(&state_dir).unwrap(); + host.ensure_as(ROOT_EUID, &state_dir, JailUser::default()) + .unwrap(); + assert!(!state_dir.exists(), "a finished sweep spends the signal"); } #[test] - fn builder_chain() { - let limits = ResourceLimits::default() - .with_cpu_limit(2.0) - .with_memory_limit(1024) - .with_io_limits(100, 200); - assert_eq!(limits.cpu_quota_us, Some(200_000)); - assert_eq!(limits.memory_bytes, Some(1024)); - assert_eq!(limits.io_read_bps, Some(100)); - assert_eq!(limits.io_write_bps, Some(200)); + fn the_jail_user_rejects_root() { + // Untrusted code against a root VMM is the one thing the confinement + // exists to prevent, so this must not be reachable by a typo. + JailUser::new(0, DEFAULT_JAIL_GID).unwrap_err(); + JailUser::new(DEFAULT_JAIL_UID, 0).unwrap_err(); + JailUser::new(0, 0).unwrap_err(); + + let user = JailUser::new(1234, 5678).unwrap(); + assert_eq!(user.uid(), 1234); + assert_eq!(user.gid(), 5678); } #[test] - fn serde_round_trip() { - let limits = ResourceLimits::default() - .with_cpu_limit(1.5) - .with_memory_limit(2048); - let json = serde_json::to_string(&limits).unwrap(); - let parsed: ResourceLimits = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed.cpu_quota_us, limits.cpu_quota_us); - assert_eq!(parsed.memory_bytes, Some(2048)); - assert_eq!(parsed.cpu_period_us, 100_000); + fn the_default_jail_user_is_unprivileged() { + let default = JailUser::default(); + assert_eq!(default.uid(), DEFAULT_JAIL_UID); + assert_eq!(default.gid(), DEFAULT_JAIL_GID); + JailUser::new(default.uid(), default.gid()).unwrap(); } #[test] - fn serde_deserialize_minimal() { - let json = "{}"; - let limits: ResourceLimits = serde_json::from_str(json).unwrap(); - assert_eq!(limits.cpu_quota_us, None); - assert_eq!(limits.cpu_period_us, 100_000); - assert_eq!(limits.max_procs, 64); + fn a_named_account_is_found_by_id() { + let passwd = "root:x:0:0:root:/root:/bin/bash\nbuild:x:61016:61016:CI build user:/home/build:/bin/sh\n"; + + assert_eq!(lookup_name_in(passwd, 61016).as_deref(), Some("build")); + assert_eq!(lookup_name_in(passwd, 0).as_deref(), Some("root")); + } + + #[test] + fn an_unallocated_id_has_no_name() { + let passwd = + "root:x:0:0:root:/root:/bin/bash\ndaemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin\n"; + + assert_eq!(lookup_name_in(passwd, 61016), None); + } + + #[test] + fn malformed_records_are_skipped() { + let passwd = "\nnot-a-record\nshort:x\nbuild:x:61016:61016::/home/build:/bin/sh\n"; + + assert_eq!(lookup_name_in(passwd, 61016).as_deref(), Some("build")); + assert_eq!(lookup_name_in("", 61016), None); } } diff --git a/plus/bencher_runner/src/jail/netns.rs b/plus/bencher_runner/src/jail/netns.rs new file mode 100644 index 000000000..f9aa27343 --- /dev/null +++ b/plus/bencher_runner/src/jail/netns.rs @@ -0,0 +1,307 @@ +//! The empty network namespace the jailed VMM joins. +//! +//! The guest has no network device and never will; this namespace is for the +//! VMM process itself. A compromised Firecracker with host network access can +//! exfiltrate, and an empty namespace removes that reach. The vsock transport +//! is unaffected: its host side is filesystem-scoped Unix domain sockets, not +//! network-namespace-scoped. + +use std::fs; +use std::os::unix::fs::MetadataExt as _; + +use camino::{Utf8Path, Utf8PathBuf}; +use nix::mount::{MntFlags, MsFlags, mount, umount2}; +use nix::sched::{CloneFlags, unshare}; + +use crate::error::JailError; +use crate::jail::lock::{flock_exclusive, flock_nonblocking}; + +/// Directory holding named network namespace handles. +/// +/// This follows the `ip netns` convention (iproute2's `NETNS_RUN_DIR`), so the +/// runner's namespace shows up in `ip netns list` for operators. `/run` is a +/// tmpfs, so handles do not survive a reboot, which is exactly right for a +/// handle onto a kernel object. +const NETNS_DIR: &str = "/run/netns"; + +/// Name of the empty network namespace the jailed VMM joins. +const NETNS_NAME: &str = "bencher-jail"; + +/// The runner's own network namespace, used as the reference for deciding +/// whether a handle is a live namespace distinct from the host's. +const SELF_NETNS: &str = "/proc/self/ns/net"; + +/// How many stacked mounts to unwind at the handle before giving up. +/// +/// Bounded rather than unbounded: a path that keeps reporting a successful +/// unmount forever is a kernel fault, and the failed unlink that follows +/// reports the real state either way. +const MAX_STACKED_MOUNTS: usize = 32; + +/// Lock file serializing access to the global network namespace handle. +const NETNS_LOCK_PATH: &str = "/run/bencher_runner_netns.lock"; + +/// The calling *thread's* network namespace. +/// +/// `/proc/self` resolves through the thread group leader, so it must not be +/// used from the namespace-creating thread: it would name the runner's own +/// namespace and pin the host network instead of the new one. +const THREAD_NETNS: &str = "/proc/thread-self/ns/net"; + +/// The path of the network namespace handle. +#[must_use] +pub fn handle_path() -> Utf8PathBuf { + Utf8Path::new(NETNS_DIR).join(NETNS_NAME) +} + +/// Build a fresh empty network namespace, returning its handle path. +/// +/// The handle is always cleared and recreated rather than reused. Proving a +/// handle is a namespace and is not the runner's own does not prove it is +/// empty: a `bencher-jail` left by an operator experimenting with `ip netns`, +/// or by a name collision, could hold interfaces, and the VMM would silently +/// regain the host network reach this module exists to remove. Recreating is +/// both cheaper and stronger than trying to assert a namespace holds nothing +/// but a down `lo`. +/// +/// Called per job rather than once per daemon lifetime. `/run` is a tmpfs and +/// the handle is a shared, operator-visible object, so an `ip netns del` or a +/// remount would otherwise break every subsequent job until a restart. +/// +/// The namespace is process-global while the jail lock is per state directory, +/// so this takes its own lock: two runners started with different +/// `--state-dir` values hold different jail locks and would otherwise clear +/// and rebind the same handle concurrently. +/// +/// The lock covers the rebuild, not the use. The jailer opens the handle +/// itself, after this returns and after the lock is released, so a second +/// runner rebuilding the handle in that window can make the first runner's +/// jailer see `ENOENT` between the unlink and the bind. That is narrow, it +/// requires two runners on one host, and it fails the job loudly rather than +/// silently leaving the VMM on the host network, which is the failure that +/// would matter. Holding the lock until the VMM has started would close it, +/// at the cost of serializing every job on a process-global lock. +pub fn ensure() -> Result { + let handle = handle_path(); + + fs::create_dir_all(NETNS_DIR).map_err(|e| JailError::NetnsDir { + path: Utf8PathBuf::from(NETNS_DIR), + source: e, + })?; + + let _lock = NetnsLock::acquire()?; + + clear(&handle)?; + + // The bind mount needs a regular file to land on. + fs::File::create(&handle).map_err(|e| JailError::NetnsHandle { + path: handle.clone(), + source: e, + })?; + + // Unwound through `clear`, never a bare unlink. Reaching the second arm + // means the bind mount is definitely there, since that is what makes the + // namespace live, and unlinking a mounted path fails with `EBUSY`: the + // handle would stay mounted, which is the state that makes every later + // `ensure` fail on a stacked mount. `clear` is the only way it is removed. + if let Err(e) = create(&handle) { + drop(clear(&handle)); + return Err(e); + } + + // The namespace has to be a real one and not the runner's own, or the VMM + // would keep host network reach. Cheap, and the whole point of the module. + if !is_live_netns(&handle) { + drop(clear(&handle)); + return Err(JailError::NetnsNotDistinct { + path: handle.clone(), + }); + } + + Ok(handle) +} + +/// Remove whatever is at the handle path, mounts included. +/// +/// Bind mounting over a file does not report `EBUSY`, so mounts stack: a +/// handle that has been recreated more than once carries more than one. A +/// single detach unwinds only the top mount, the unlink of the still-mounted +/// path then fails with `EBUSY`, and `File::create` on the surviving nsfs +/// mount fails with `EPERM` even as root. Unwinding one mount at a time and +/// reporting a failed unlink is what keeps a stacked handle from wedging the +/// host: without it, `ensure` fails permanently and every sandboxed job with +/// it, until an operator loops `umount` by hand. +fn clear(handle: &Utf8Path) -> Result<(), JailError> { + for _ in 0..MAX_STACKED_MOUNTS { + if umount2(handle.as_std_path(), MntFlags::MNT_DETACH).is_err() { + break; + } + } + + match fs::remove_file(handle) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(JailError::NetnsHandle { + path: handle.to_owned(), + source: e, + }), + } +} + +/// Advisory lock over the process-global network namespace handle. +/// +/// Separate from the jail lock, which is scoped to a state directory: the +/// handle is a single global object and two runners with different state +/// directories must still not rebind it at the same time. It lives beside the +/// host tuning lock, in root-writable tmpfs that clears on reboot. +struct NetnsLock { + _file: fs::File, +} + +impl NetnsLock { + /// Take the lock, waiting for whichever runner holds it. + /// + /// Announces the wait, as the jail lock does. This handle is global to the + /// host, so two runners with unrelated state directories contend here, and a + /// runner that sat silently on it would look hung rather than queued. + #[expect( + clippy::print_stdout, + reason = "prints why the runner is waiting, as the jail lock does" + )] + fn acquire() -> Result { + let path = Utf8Path::new(NETNS_LOCK_PATH); + let file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(path) + .map_err(|e| JailError::OpenNetnsLock { + path: path.to_owned(), + source: e, + })?; + + // Try once without blocking, so the wait can be announced rather than + // looking like a hang. + if flock_nonblocking(&file).is_ok() { + return Ok(Self { _file: file }); + } + println!(" Waiting for another bencher runner to release {path}..."); + + flock_exclusive(&file).map_err(|e| JailError::NetnsLock { + path: path.to_owned(), + source: e, + })?; + Ok(Self { _file: file }) + } +} + +/// Whether `handle` is a live network namespace other than the runner's own. +/// +/// Every namespace inode lives on the single kernel `nsfs`, so sharing a +/// device with a known namespace proves the handle is one, and a differing +/// inode proves it is not the host namespace the runner itself is in. A +/// leftover placeholder file sits on the `/run` tmpfs and fails the device +/// check. +fn is_live_netns(handle: &Utf8Path) -> bool { + let (Ok(own), Ok(candidate)) = (fs::metadata(SELF_NETNS), fs::metadata(handle)) else { + return false; + }; + own.dev() == candidate.dev() && own.ino() != candidate.ino() +} + +/// Create the namespace and bind its handle into place. +/// +/// The namespace is unshared on a dedicated thread rather than in the runner +/// itself. Network namespaces are per-task, so only this thread moves and the +/// runner stays on the host network; the bind mount then holds a reference +/// that keeps the namespace alive once the thread exits. The thread is not +/// reused for anything else, precisely because it never returns to the host +/// namespace. +fn create(handle: &Utf8Path) -> Result<(), JailError> { + let target = handle.to_owned(); + std::thread::spawn(move || -> Result<(), JailError> { + unshare(CloneFlags::CLONE_NEWNET).map_err(JailError::Unshare)?; + mount( + Some(THREAD_NETNS), + target.as_std_path(), + None::<&str>, + MsFlags::MS_BIND, + None::<&str>, + ) + .map_err(|e| JailError::BindNetns { + path: target.clone(), + source: e, + }) + }) + .join() + .map_err(|_panic| JailError::NetnsThread)? +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn handle_follows_the_ip_netns_convention() { + assert_eq!(handle_path(), "/run/netns/bencher-jail"); + } + + #[test] + fn a_plain_file_is_not_a_live_netns() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let path = root.join("net"); + fs::write(&path, b"").unwrap(); + + assert!(!is_live_netns(&path)); + } + + #[test] + fn a_missing_handle_is_not_a_live_netns() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + assert!(!is_live_netns(&root.join("absent"))); + } + + #[test] + fn clear_removes_a_plain_handle() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let path = root.join("net"); + fs::write(&path, b"").unwrap(); + + clear(&path).unwrap(); + + assert!(!path.exists()); + } + + #[test] + fn clear_is_idempotent_on_a_missing_handle() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + clear(&root.join("absent")).unwrap(); + } + + #[test] + fn clear_reports_a_handle_it_cannot_remove() { + // A directory stands in for the unremovable handle: the real case is + // a still-mounted path, which unlinks with EBUSY. Either way the + // failure has to surface rather than be swallowed into a confusing + // File::create error further down. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let path = root.join("net"); + fs::create_dir(&path).unwrap(); + + clear(&path).unwrap_err(); + } + + #[test] + fn the_runners_own_namespace_is_not_a_distinct_netns() { + // The handle must be a namespace *other* than the one the runner is + // in, or the VMM would keep host network reach. + assert!(!is_live_netns(Utf8Path::new(SELF_NETNS))); + } +} diff --git a/plus/bencher_runner/src/jail/paths.rs b/plus/bencher_runner/src/jail/paths.rs new file mode 100644 index 000000000..3ed9a5ac1 --- /dev/null +++ b/plus/bencher_runner/src/jail/paths.rs @@ -0,0 +1,433 @@ +//! The views of every file inside the jail chroot. +//! +//! Once Firecracker is jailed, every path it receives resolves inside the +//! chroot, while the runner reaches the same file from outside. The views are +//! different types rather than strings, so handing Firecracker a host path (or +//! the runner a chroot path) is a compile error instead of a boot that hangs +//! waiting for a socket that will never appear. +//! +//! There is a third view because of a hard kernel limit. `sockaddr_un.sun_path` +//! is 108 bytes, and the limit applies to the string handed to `bind` and +//! `connect`, before any resolution. A jail deep under an operator's +//! `--state-dir` blows that limit long before it comes near `PATH_MAX`, so the +//! runner addresses the jail's sockets through a descriptor it holds open on +//! the chroot. Resolution happens after the length check, so a short string +//! naming a long directory is exactly what is needed. + +use std::fs::File; +use std::os::fd::AsRawFd as _; +use std::os::unix::fs::OpenOptionsExt as _; + +use camino::{Utf8Path, Utf8PathBuf}; +use serde::Serialize; + +use crate::error::JailError; + +/// Size of `sockaddr_un.sun_path` on Linux. +const SUN_PATH_LEN: usize = 108; + +/// Longest path a socket name may occupy, leaving room for the NUL. +/// +/// Linux accepts a full 108 unterminated bytes when `addrlen` says so, but +/// `unix(7)` warns against relying on it and the standard library rejects +/// anything that does not leave room for the terminator. +const MAX_SOCKET_PATH: usize = SUN_PATH_LEN - 1; + +/// Room reserved after the vsock base path for its `_` suffix. +/// +/// Sized for the widest port a `u32` can print rather than for the ports in +/// use, so adding a port can never silently eat the margin. +const VSOCK_SUFFIX_RESERVE: usize = "_4294967295".len(); + +/// A path as the runner sees it: the host filesystem, outside the chroot. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostPath(Utf8PathBuf); + +impl HostPath { + /// The path as a [`Utf8Path`]. + #[must_use] + pub fn as_path(&self) -> &Utf8Path { + &self.0 + } + + /// The path as a string. + #[must_use] + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +impl std::fmt::Display for HostPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +/// A path as the jailed Firecracker process sees it, rooted at the chroot. +/// +/// Serializes as the bare path, since these are what the Firecracker API +/// request bodies carry. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(transparent)] +pub struct ChrootPath(Utf8PathBuf); + +impl ChrootPath { + /// The path as a string. + #[must_use] + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +impl std::fmt::Display for ChrootPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +/// A path the runner may hand to `bind` or `connect`. +/// +/// The value names a descriptor held open by the [`JailPaths`] it came from, +/// and it is only valid while that value is alive. That invariant is upheld by +/// ordering rather than by the compiler: `SocketPath` could borrow, which would +/// make a use-after-drop impossible, but the lifetime would ripple through +/// [`JailFile`], the Firecracker job config, the jailer spawn, and everything +/// that owns them. It is deliberately not enforced, not overlooked, and it is +/// a reasonable follow-up. +/// +/// Two things hold the line in the meantime. Only `bind` and `connect` take +/// this view, because they are the only callers subject to the length limit; +/// everything else, unlinking above all, takes the host view, which cannot go +/// stale. And a test drops the paths, claims the released descriptor number +/// with another directory, and asserts the same string no longer names the +/// jail, so the hazard is at least pinned. +/// +/// The type makes the length limit unforgeable: every value has been checked +/// against `sun_path`, so a path that would not fit is reported when the jail +/// is built, naming the limit and the offending string, rather than surfacing +/// later as a socket that never becomes ready. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SocketPath(String); + +impl SocketPath { + /// Check a path against the `sun_path` limit. + /// + /// `reserve` is the longest suffix that will later be appended, so a base + /// that fits only without its suffix is still rejected. + fn new(path: String, reserve: usize) -> Result { + let length = path.len() + reserve; + if length > MAX_SOCKET_PATH { + return Err(JailError::SocketPathTooLong { + path, + length, + limit: MAX_SOCKET_PATH, + }); + } + Ok(Self(path)) + } + + /// The path as a string. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + /// This path with a suffix appended. + /// + /// The suffix was reserved when the base was checked, so the result is + /// within the limit by construction. + #[must_use] + pub fn with_suffix(&self, suffix: &str) -> String { + format!("{}{suffix}", self.0) + } +} + +impl std::fmt::Display for SocketPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +/// One file in the jail, in every view that reaches it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JailFile { + host: HostPath, + chroot: ChrootPath, + socket: SocketPath, +} + +impl JailFile { + /// The path the runner uses to create, read, and own the file. + #[must_use] + pub fn host(&self) -> &HostPath { + &self.host + } + + /// The path Firecracker receives. + #[must_use] + pub fn chroot(&self) -> &ChrootPath { + &self.chroot + } + + /// The path the runner binds or connects to. + #[must_use] + pub fn socket(&self) -> &SocketPath { + &self.socket + } +} + +/// Every file the runner places in, or reaches inside, a jail chroot. +#[derive(Debug)] +pub struct JailPaths { + root: Utf8PathBuf, + /// Held open for the life of the job. + /// + /// The socket views name this descriptor by number, so closing it would + /// leave them addressing whatever directory the kernel hands that number + /// to next. `O_PATH` because the runner never reads or writes through it: + /// it exists only to be named. + _dir: File, + api_socket: JailFile, + kernel: JailFile, + rootfs: JailFile, + vsock: JailFile, +} + +impl JailPaths { + /// Resolve every view of every jail file for a chroot that already exists. + pub fn new(jail_root: &Utf8Path) -> Result { + let dir = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_PATH | libc::O_DIRECTORY) + .open(jail_root) + .map_err(|e| JailError::OpenJailRoot { + path: jail_root.to_owned(), + source: e, + })?; + + // Naming the directory by descriptor keeps the string short however + // deep the operator's state directory is. The descriptor also pins the + // directory's identity for the whole job. + let dir_path = format!("/proc/self/fd/{}", dir.as_raw_fd()); + + let file = |name: &str, reserve: usize| -> Result { + Ok(JailFile { + host: HostPath(jail_root.join(name)), + chroot: ChrootPath(Utf8Path::new("/").join(name)), + socket: SocketPath::new(format!("{dir_path}/{name}"), reserve)?, + }) + }; + + Ok(Self { + root: jail_root.to_owned(), + api_socket: file("api.sock", 0)?, + kernel: file("vmlinux", 0)?, + rootfs: file("rootfs.ext4", 0)?, + // The runner binds `{base}_{port}` for each vsock port, so the + // base has to leave room for the longest of those suffixes. + vsock: file("v.sock", VSOCK_SUFFIX_RESERVE)?, + _dir: dir, + }) + } + + /// The chroot root on the host, which becomes `/` inside the jail. + #[must_use] + pub fn root(&self) -> &Utf8Path { + &self.root + } + + /// The Firecracker REST API socket. + #[must_use] + pub fn api_socket(&self) -> &JailFile { + &self.api_socket + } + + /// The guest kernel image, which Firecracker reads. + #[must_use] + pub fn kernel(&self) -> &JailFile { + &self.kernel + } + + /// The guest rootfs image, which Firecracker reads and writes. + #[must_use] + pub fn rootfs(&self) -> &JailFile { + &self.rootfs + } + + /// The base path of the vsock Unix domain sockets. + #[must_use] + pub fn vsock(&self) -> &JailFile { + &self.vsock + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn jail_in_tmpdir() -> (tempfile::TempDir, JailPaths) { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8Path::from_path(dir.path()).unwrap(); + let paths = JailPaths::new(root).unwrap(); + (dir, paths) + } + + #[test] + fn chroot_view_is_rooted_at_the_chroot() { + let (_dir, paths) = jail_in_tmpdir(); + assert_eq!(paths.api_socket().chroot().as_str(), "/api.sock"); + assert_eq!(paths.kernel().chroot().as_str(), "/vmlinux"); + assert_eq!(paths.rootfs().chroot().as_str(), "/rootfs.ext4"); + assert_eq!(paths.vsock().chroot().as_str(), "/v.sock"); + } + + #[test] + fn host_view_is_under_the_jail_root() { + let (_dir, paths) = jail_in_tmpdir(); + let root = paths.root().to_owned(); + assert_eq!(paths.api_socket().host().as_path(), root.join("api.sock")); + assert_eq!(paths.kernel().host().as_path(), root.join("vmlinux")); + assert_eq!(paths.rootfs().host().as_path(), root.join("rootfs.ext4")); + assert_eq!(paths.vsock().host().as_path(), root.join("v.sock")); + } + + #[test] + fn the_chroot_and_host_views_round_trip_through_the_jail_root() { + let (_dir, paths) = jail_in_tmpdir(); + for file in [ + paths.api_socket(), + paths.kernel(), + paths.rootfs(), + paths.vsock(), + ] { + let relative = Utf8Path::new(file.chroot().as_str()) + .strip_prefix("/") + .unwrap(); + assert_eq!( + paths.root().join(relative), + file.host().as_path(), + "the chroot view of {file:?} must resolve to its host view" + ); + } + } + + #[test] + fn the_socket_view_resolves_to_the_same_file_as_the_host_view() { + // The whole point: a short string naming the same inode. If this ever + // stops holding, the runner and Firecracker stop meeting. + let (_dir, paths) = jail_in_tmpdir(); + std::fs::write(paths.rootfs().host().as_path(), b"guest").unwrap(); + + let through_socket_view = std::fs::read(paths.rootfs().socket().as_str()).unwrap(); + + assert_eq!(through_socket_view, b"guest"); + } + + #[test] + fn the_socket_view_stops_naming_the_jail_once_the_paths_are_dropped() { + // This is why `bind` and `connect` are the only callers of the socket + // view: the descriptor number is reused the moment it is released, and + // the identical string then resolves to a different directory with no + // error at all. Anything that only needs a path, unlinking above all, + // uses the host view, which cannot go stale. + let jail = tempfile::tempdir().unwrap(); + let jail_root = Utf8Path::from_path(jail.path()).unwrap(); + std::fs::write(jail_root.join("rootfs.ext4"), b"the jail").unwrap(); + + let socket_view = { + let paths = JailPaths::new(jail_root).unwrap(); + let view = paths.rootfs().socket().as_str().to_owned(); + assert_eq!(std::fs::read(&view).unwrap(), b"the jail"); + view + }; + + // Claim the number the jail's descriptor just released. + let impostor = tempfile::tempdir().unwrap(); + let impostor_root = Utf8Path::from_path(impostor.path()).unwrap(); + std::fs::write(impostor_root.join("rootfs.ext4"), b"somewhere else").unwrap(); + let _claim = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_PATH | libc::O_DIRECTORY) + .open(impostor_root) + .unwrap(); + + // The same string no longer names the jail. It either names the + // impostor or fails; what it must never do is still work. + let stale = std::fs::read(&socket_view); + assert_ne!( + stale.unwrap_or_default(), + b"the jail", + "a dropped descriptor must not leave the socket view pointing at the jail" + ); + } + + #[test] + fn every_socket_view_fits_the_sun_path_limit() { + let (_dir, paths) = jail_in_tmpdir(); + for file in [paths.api_socket(), paths.vsock()] { + assert!( + file.socket().as_str().len() <= MAX_SOCKET_PATH, + "{} must fit sun_path", + file.socket() + ); + } + // And the longest name the vsock base ever grows into still fits. + let longest = paths.vsock().socket().with_suffix("_4294967295"); + assert!(longest.len() <= MAX_SOCKET_PATH, "{longest} must fit"); + } + + #[test] + fn the_socket_view_survives_a_jail_root_far_past_the_limit() { + // A deep state directory is exactly the case that produced a five + // second timeout pointing at Firecracker instead of at the path. + let dir = tempfile::tempdir().unwrap(); + let mut root = Utf8Path::from_path(dir.path()).unwrap().to_owned(); + for _ in 0..8 { + root = root.join("a-fairly-long-directory-name"); + } + std::fs::create_dir_all(&root).unwrap(); + assert!( + root.as_str().len() > MAX_SOCKET_PATH, + "the host path has to be over the limit for this to prove anything" + ); + + let paths = JailPaths::new(&root).unwrap(); + + assert!(paths.api_socket().socket().as_str().len() <= MAX_SOCKET_PATH); + } + + #[test] + fn an_oversized_socket_path_names_the_limit_and_the_length() { + let err = SocketPath::new("/x".repeat(80), 0).unwrap_err(); + let message = err.to_string(); + + assert!(message.contains("107"), "the limit is named: {message}"); + assert!(message.contains("160"), "the length is named: {message}"); + } + + #[test] + fn a_reserved_suffix_counts_against_the_limit() { + let base = "/a".repeat(52); + assert_eq!(base.len(), 104); + + SocketPath::new(base.clone(), 0).unwrap(); + SocketPath::new(base, 8).unwrap_err(); + } + + #[test] + fn a_missing_jail_root_is_reported() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8Path::from_path(dir.path()).unwrap().join("absent"); + + JailPaths::new(&root).unwrap_err(); + } + + #[test] + fn chroot_paths_serialize_as_bare_strings() { + let (_dir, paths) = jail_in_tmpdir(); + assert_eq!( + serde_json::to_string(paths.rootfs().chroot()).unwrap(), + "\"/rootfs.ext4\"" + ); + } +} diff --git a/plus/bencher_runner/src/jail/reap.rs b/plus/bencher_runner/src/jail/reap.rs new file mode 100644 index 000000000..0b2ef4cc9 --- /dev/null +++ b/plus/bencher_runner/src/jail/reap.rs @@ -0,0 +1,633 @@ +//! Reaping the VMM a runner left behind. +//! +//! The sweep reclaims chroots because `Drop` does not run when a runner is +//! `SIGKILL`ed, crashes, or `exec`s itself during a self-update. The same exits +//! strand the jailed VMM: it is not signalled when its parent dies, so it is +//! reparented and keeps running, holding the benchmark cores through its +//! cgroup. Reclaiming only the disk leaves the more damaging half behind. +//! +//! The consequence is not leakage but wrong numbers that look right, and +//! nothing downstream catches it. The per-VM cgroups set only `cpuset.cpus` +//! and `cpuset.mems`, never `cpuset.cpus.exclusive`, so a leftover cgroup +//! claims nothing and does not narrow the next job's effective set: the next +//! job's cpuset applies cleanly and verifies cleanly while the stray VMM runs +//! untrusted guest code on the very same cores. The harm is contention, which +//! is invisible in the result. +//! +//! That is why a jail that cannot be cleared fails the job rather than merely +//! warning: it is the one remaining path where the runner would knowingly +//! emit a number it has reason to distrust. +//! +//! Killing a process the runner does not own is a destructive capability, so +//! the target is identified as narrowly as possible: only a process whose root +//! directory *is* the chroot being swept, compared by device and inode. A +//! process merely owned by the jail uid is not a target, because on a shared +//! host that uid may legitimately own something else. + +#![expect(clippy::print_stderr, reason = "reaping prints diagnostics")] + +use std::fs; +use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd}; +use std::os::unix::fs::MetadataExt as _; +use std::time::{Duration, Instant}; + +use camino::Utf8Path; + +/// How many processes to reap from one jail before giving up. +/// +/// A jail holds one VMM, so this is a bound on a loop that should run once, +/// not an expectation. +const MAX_JAILED_PROCESSES: usize = 64; + +/// How long to wait for a killed VMM to disappear. +const REAP_TIMEOUT: Duration = Duration::from_secs(5); + +/// How often to check whether it has. +const REAP_INTERVAL: Duration = Duration::from_millis(20); + +/// Whether a jail still has a VMM running in it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Reaped { + /// Nothing is running in the jail: either nothing was, or it has exited. + Clear, + /// A VMM is still running in the jail and could not be reaped. + /// + /// The caller must not remove the chroot: doing so would pull the rootfs + /// out from under a live process, and would destroy the only handle for + /// identifying that process on a later sweep. + StillRunning { + /// The VMM that is still running. + pid: u32, + }, + /// The jail could not be examined, so what is in it is unknown. + /// + /// Distinct from [`Self::Clear`], which is the whole reason this variant + /// exists: a jail root that cannot be stat'ed, or a `/proc` that cannot be + /// listed, says nothing about the jail, and reporting nothing as "empty" is + /// what would have the caller delete a tree with a live VMM in it. The + /// caller treats it exactly like [`Self::StillRunning`], with no pid to + /// name. + Unexaminable, +} + +/// Kill the VMM confined to `jail_root`, if one is still running. +/// +/// Best effort about *which* process it touches: a VMM that cannot be +/// identified is left alone, because the alternative to leaving an +/// unidentified process alone is killing the wrong one. Never best effort +/// about what it reports, because the caller decides whether to delete a +/// directory based on the answer. +pub fn reap_jailed_vmm(jail_root: &Utf8Path) -> Reaped { + reap_jailed_vmm_with(jail_root, find_jailed_vmm, reap_one) +} + +/// The reap, with the scan and the kill injectable. +/// +/// Both reach into `/proc` and signal processes on the machine running the +/// tests, so the exhaustion path is exercised through parameters rather than by +/// manufacturing sixty-four jailed processes. +fn reap_jailed_vmm_with(jail_root: &Utf8Path, find: F, reap: R) -> Reaped +where + F: Fn(&Utf8Path) -> std::io::Result>, + R: Fn(u32, &Utf8Path) -> Reaped, +{ + // Rescan after each reap rather than assuming one process per jail. That + // assumption holds today, since neither `--daemonize` nor `--new-pid-ns` + // is passed and the jailer execs in place as a single process, but the + // caller deletes a directory tree based on this answer. An invariant that + // load-bearing is worth enforcing rather than trusting, and a survivor + // would otherwise have the tree removed out from under it. + for _ in 0..MAX_JAILED_PROCESSES { + match find(jail_root) { + Ok(Some(pid)) => match reap(pid, jail_root) { + // Killed, or already gone: look again, because a jail is not + // clear until a scan finds nothing in it. + Reaped::Clear => {}, + Reaped::StillRunning { pid } => return Reaped::StillRunning { pid }, + // `reap_one` cannot return this today, and it is matched + // exhaustively so that a third outcome arriving here is a + // compile error rather than a silent fall-through into another + // pass of the loop. This module's whole argument is that a state + // nobody could examine must not read as a cleared one, and an + // `if let` is the construct that lets exactly that happen. + Reaped::Unexaminable => return Reaped::Unexaminable, + }, + Ok(None) => return Reaped::Clear, + // A scan that could not run has not found the jail empty, it has + // found out nothing. The caller deletes a tree on this answer. + Err(e) => { + eprintln!( + "Warning: cannot examine {jail_root} to see whether a VMM is still in it: {e}. It is left in place." + ); + return Reaped::Unexaminable; + }, + } + } + + // Exhausting the bound is anomalous by construction: Firecracker does not + // fork, the jailer execs in place, and a healthy jail therefore holds + // exactly one process. Reaching here means either the scan and the kill + // disagree, which is a bug and the likelier of the two, or something is + // spawning into this jail, which is precisely when the sweep must refuse. + // Neither says the jail is clear, so it is not reported clear. + // + // Nothing else would catch it. The cgroup removal that follows in the sweep + // refuses while anything is still in the cgroup, but a run with no CPU + // layout, or one whose host does not delegate cpuset, has no cgroup at all. + // Failing the job is recoverable, because nothing latches and the next job + // sweeps again; measuring through a jail that is still occupied is not, and + // that is the one outcome this module exists to prevent. + match find(jail_root) { + Ok(Some(pid)) => { + eprintln!( + "Warning: gave up scanning {jail_root} after {MAX_JAILED_PROCESSES} passes; pid {pid} still matches it while every reap reported the jail clear." + ); + Reaped::StillRunning { pid } + }, + Ok(None) => Reaped::Clear, + Err(e) => { + eprintln!( + "Warning: cannot examine {jail_root} to see whether a VMM is still in it: {e}. It is left in place." + ); + Reaped::Unexaminable + }, + } +} + +/// Kill one process known to be confined to `jail_root`. +fn reap_one(pid: u32, jail_root: &Utf8Path) -> Reaped { + // Pin the pid before signalling it. A pid found by scanning `/proc` can + // exit and have its number recycled before the signal lands, and this runs + // as root, so the signal would go to whatever inherited the number. A + // pidfd refers to one process for as long as it is open and keeps the + // number from being reused, which turns the check below into a guarantee + // rather than a narrow window. + let pidfd = match pidfd_open(pid) { + Ok(Some(pidfd)) => pidfd, + // Already gone, which is the common case and not a failure. + Ok(None) => return Reaped::Clear, + Err(e) => { + // Silence here is what wedges a runner: the orphan keeps the + // benchmark CPUs, the cgroup cannot be removed, and nothing says + // why. Kernels before 5.3 have no pidfd_open at all. + eprintln!( + "Warning: cannot pin orphaned VMM (pid {pid}) in {jail_root} to reap it: {e}. It is still running and still holds the benchmark CPUs." + ); + return Reaped::StillRunning { pid }; + }, + }; + + // Re-check now that the pid cannot change underneath us. If it is no + // longer the jail's VMM, the jail is clear whatever else is true. + if !is_jailed_vmm(pid, jail_root) { + return Reaped::Clear; + } + + if let Err(e) = pidfd_kill(&pidfd) { + eprintln!("Warning: failed to kill orphaned VMM (pid {pid}) in {jail_root}: {e}"); + return Reaped::StillRunning { pid }; + } + + if wait_for_exit(pid) { + eprintln!("Reaped orphaned VMM (pid {pid}) left behind in {jail_root}"); + Reaped::Clear + } else { + eprintln!( + "Warning: orphaned VMM (pid {pid}) in {jail_root} did not exit within {} seconds", + REAP_TIMEOUT.as_secs() + ); + Reaped::StillRunning { pid } + } +} + +/// Find the pid of the VMM whose root directory is `jail_root`. +/// +/// The jailer `chroot`s before exec, so the confined process's root *is* the +/// chroot. Comparing device and inode rather than the path is what makes this +/// exact: the jailer pivots into a private mount namespace, so the path reads +/// back as `/`, while the identity is preserved. +/// +/// `Ok(None)` is "nothing is confined here", which a jail root that is not there +/// at all also means: nothing can be chrooted into a directory that does not +/// exist. Every other failure is an error rather than an absence, because the +/// caller deletes a directory tree on the strength of this answer and a scan +/// that could not run has established nothing. +fn find_jailed_vmm(jail_root: &Utf8Path) -> std::io::Result> { + let jail = match fs::metadata(jail_root) { + Ok(jail) => jail, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e), + }; + for entry in fs::read_dir("/proc")? { + let entry = entry?; + let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { + continue; + }; + if matches_jail(pid, &jail) { + return Ok(Some(pid)); + } + } + Ok(None) +} + +/// Whether a process's root directory is `jail_root`. +/// +/// A jail root that cannot be stat'ed reads as "not this jail's VMM", which +/// keeps the kill from landing on a process this function cannot vouch for. The +/// caller's next scan turns the same failure into [`Reaped::Unexaminable`], so +/// nothing downstream mistakes it for an empty jail. +fn is_jailed_vmm(pid: u32, jail_root: &Utf8Path) -> bool { + fs::metadata(jail_root).is_ok_and(|jail| matches_jail(pid, &jail)) +} + +/// Whether a process's root directory is the same inode as `jail`. +/// +/// A `/proc//root` that cannot be read means the process is gone or is not +/// one this runner may inspect, and either way it is not the jail's VMM. That is +/// the one failure here that is genuinely nothing: processes come and go under a +/// scan of `/proc` constantly. +fn matches_jail(pid: u32, jail: &fs::Metadata) -> bool { + // Following this magic symlink crosses into the process's own mount + // namespace, which a privileged reader is allowed to do. + fs::metadata(format!("/proc/{pid}/root")) + .is_ok_and(|root| root.dev() == jail.dev() && root.ino() == jail.ino()) +} + +/// Open a descriptor pinned to a process. +/// +/// `Ok(None)` means the process is already gone, which is the common case and +/// not a failure. Everything else is distinguished and reported by the caller, +/// because a reap that quietly does nothing leaves an orphan holding the +/// benchmark CPUs: `ENOSYS` on a kernel before 5.3, `EPERM` under a +/// restrictive policy, and a pid too large to convert all look identical from +/// the outside otherwise. +fn pidfd_open(pid: u32) -> std::io::Result> { + let pid = libc::pid_t::try_from(pid).map_err(|_err| { + std::io::Error::new(std::io::ErrorKind::InvalidInput, "pid out of range") + })?; + + #[expect( + unsafe_code, + reason = "pidfd_open has no std wrapper; it takes plain integers" + )] + // SAFETY: `pidfd_open` takes a pid and a flag word and touches no memory. + // It returns a new descriptor or -1, and the descriptor is handed straight + // to `OwnedFd` so it is closed exactly once. Every argument is widened to + // `c_long` explicitly: `syscall` is variadic, so the width each value is + // passed at is the width written here rather than anything the signature + // enforces. + let raw = unsafe { + libc::syscall( + libc::SYS_pidfd_open, + libc::c_long::from(pid), + libc::c_long::from(0i32), + ) + }; + + if raw < 0 { + let error = std::io::Error::last_os_error(); + return if error.raw_os_error() == Some(libc::ESRCH) { + Ok(None) + } else { + Err(error) + }; + } + + let raw = libc::c_int::try_from(raw) + .map_err(|_err| std::io::Error::other("pidfd out of descriptor range"))?; + #[expect( + unsafe_code, + reason = "taking ownership of a descriptor this call just created" + )] + // SAFETY: `raw` is a fresh descriptor returned by the syscall above and is + // not owned by anything else. + let fd = unsafe { OwnedFd::from_raw_fd(raw) }; + Ok(Some(fd)) +} + +/// SIGKILL the process a descriptor is pinned to. +/// +/// SIGKILL rather than a graceful shutdown: the job that owned this VMM is +/// already gone, so there is nothing left to shut down cleanly for. +fn pidfd_kill(pidfd: &OwnedFd) -> std::io::Result<()> { + #[expect( + unsafe_code, + reason = "pidfd_send_signal has no std wrapper; the fd is owned and valid" + )] + // SAFETY: `pidfd` is an open, owned descriptor for the duration of the + // call. A null `siginfo` pointer is the documented way to ask the kernel + // to synthesize one, and the final argument is a reserved flag word. The + // integers are widened to `c_long` explicitly, since a variadic call passes + // each value at the width it is written at. + let ret = unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + libc::c_long::from(pidfd.as_raw_fd()), + libc::c_long::from(libc::SIGKILL), + std::ptr::null::(), + libc::c_long::from(0i32), + ) + }; + if ret == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + +/// Wait for a killed process to stop running. +/// +/// A zombie counts as exited. The orphan reparents to whatever is PID 1, and +/// if the runner is itself PID 1 (a container with no init) nothing ever reaps +/// it, so `/proc/` persists forever. Waiting on the directory alone would +/// then stall the full timeout on every sweep and warn about a process that is +/// already dead. +fn wait_for_exit(pid: u32) -> bool { + wait_for_exit_until(Instant::now() + REAP_TIMEOUT, || is_running(pid)) +} + +/// The wait, with the deadline and the liveness check supplied. +/// +/// Parameters for the same reason the scan and the kill are: the case worth +/// testing is a process that exits in the last interval of the budget, and +/// reproducing that against a real process means racing a sleep. +fn wait_for_exit_until(deadline: Instant, running: R) -> bool +where + R: Fn() -> bool, +{ + while Instant::now() < deadline { + if !running() { + return true; + } + std::thread::sleep(REAP_INTERVAL); + } + + // Once more before the verdict. The loop sleeps between checks, so a process + // that exits during the last of them was never looked at again, and a bare + // `false` here becomes `JailError::JailStillRunning`: a job failed on a host + // that is clean, with an error confidently naming a pid that is already + // gone. The same shape as the readiness wait in the VMM process, where it + // cost a misleading message rather than a spurious failure. + !running() +} + +/// Whether a process still exists and is not a zombie. +fn is_running(pid: u32) -> bool { + let Ok(status) = fs::read_to_string(format!("/proc/{pid}/status")) else { + return false; + }; + !is_zombie(&status) +} + +/// Whether a `/proc//status` listing describes a zombie. +fn is_zombie(status: &str) -> bool { + status + .lines() + .find_map(|line| line.strip_prefix("State:")) + .is_some_and(|state| state.trim_start().starts_with('Z')) +} + +#[cfg(test)] +mod tests { + use camino::Utf8PathBuf; + + use super::*; + + #[test] + fn no_process_is_rooted_at_an_ordinary_directory() { + // The identification has to be narrow enough that a directory no + // process is chrooted into matches nothing at all. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + assert_eq!(find_jailed_vmm(&root).unwrap(), None); + } + + #[test] + fn a_missing_jail_matches_nothing() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + assert_eq!(find_jailed_vmm(&root.join("absent")).unwrap(), None); + assert!(!is_jailed_vmm(std::process::id(), &root.join("absent"))); + } + + #[test] + fn the_runners_own_root_is_matched_by_identity_not_by_path() { + // The runner is rooted at `/`, so it is found when `/` is the jail and + // not otherwise. This is the mechanism the reap depends on: a jailed + // VMM's root path reads back as `/` too, so only the inode separates + // them. + // + // It also pins the subtler half. `/proc//root` is a magic + // symlink, and the comparison has to follow it: `metadata` does, + // `symlink_metadata` would return the procfs link's own inode and + // match nothing ever. This assertion fails if that changes. + let root = Utf8Path::new("/"); + + assert!(is_jailed_vmm(std::process::id(), root)); + + let dir = tempfile::tempdir().unwrap(); + let other = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + assert!(!is_jailed_vmm(std::process::id(), &other)); + } + + #[test] + fn a_pidfd_pins_a_live_process() { + let pidfd = pidfd_open(std::process::id()).expect("pidfd_open is available"); + assert!(pidfd.is_some(), "this process is alive"); + } + + #[test] + fn a_pidfd_distinguishes_gone_from_broken() { + // Pid 0 is never a process, and the syscall rejects it rather than + // signalling the caller's process group the way plain `kill` would. + // The distinction matters: an absent process is nothing to report, + // while a refusal leaves an orphan running and has to be said out loud. + match pidfd_open(0) { + Ok(None) => {}, + Ok(Some(_)) => panic!("pid 0 must never yield a pidfd"), + Err(e) => assert_ne!( + e.raw_os_error(), + Some(libc::ESRCH), + "ESRCH must be reported as gone, not as an error" + ), + } + } + + #[test] + fn a_process_that_exits_in_the_last_interval_counts_as_exited() { + // A deadline already spent, so the loop never runs and the check after + // it is the whole verdict. Without that check the caller fails the job + // on a host that is clean, naming a pid that no longer exists. + let checks = std::cell::Cell::new(0); + + let exited = wait_for_exit_until(Instant::now(), || { + checks.set(checks.get() + 1); + false + }); + + assert!(exited, "the process is gone, whatever the budget did"); + assert_eq!(checks.get(), 1, "the check after the loop is the verdict"); + } + + #[test] + fn a_process_that_outlives_the_budget_is_still_running() { + assert!(!wait_for_exit_until(Instant::now(), || true)); + } + + #[test] + fn a_zombie_counts_as_exited() { + // Without this the reap stalls its full timeout whenever the runner is + // PID 1 and never reaps what reparents to it. + let zombie = "Name:\tfirecracker\nUmask:\t0022\nState:\tZ (zombie)\nTgid:\t42\n"; + let running = "Name:\tfirecracker\nUmask:\t0022\nState:\tS (sleeping)\nTgid:\t42\n"; + + assert!(is_zombie(zombie)); + assert!(!is_zombie(running)); + assert!(!is_zombie("")); + } + + #[test] + fn this_process_is_running() { + assert!(is_running(std::process::id())); + } + + #[test] + fn reaping_an_unjailed_directory_kills_nothing_and_reports_clear() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + assert_eq!(reap_jailed_vmm(&root), Reaped::Clear); + } + + #[test] + fn a_survivor_is_reported_without_exhausting_the_scan() { + // A process the reap could not kill is the answer immediately: the + // caller must not delete the tree under it, and there is nothing to be + // gained by rescanning. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let scans = std::cell::Cell::new(0); + + let reaped = reap_jailed_vmm_with( + &root, + |_jail_root| { + scans.set(scans.get() + 1); + Ok(Some(99)) + }, + |pid, _jail_root| Reaped::StillRunning { pid }, + ); + + assert_eq!(reaped, Reaped::StillRunning { pid: 99 }); + assert_eq!(scans.get(), 1, "a survivor is the answer on the first pass"); + } + + #[test] + fn a_jail_that_keeps_producing_processes_is_never_called_clear() { + // A healthy jail holds one process, so exhausting the bound is either + // the scan disagreeing with the kill or something spawning into the + // jail. Neither is evidence the jail is clear, and reporting it clear + // would have the sweep delete the tree and the next job measure through + // whatever is left. The cgroup removal cannot be leaned on here: a run + // with no CPU layout has no cgroup to refuse. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let reaps = std::cell::Cell::new(0); + + let reaped = reap_jailed_vmm_with( + &root, + |_jail_root| Ok(Some(7)), + |_pid, _jail_root| { + reaps.set(reaps.get() + 1); + Reaped::Clear + }, + ); + + assert_eq!(reaped, Reaped::StillRunning { pid: 7 }); + assert_eq!( + reaps.get(), + MAX_JAILED_PROCESSES, + "the loop is bounded rather than endless" + ); + } + + #[test] + fn a_jail_that_empties_on_the_last_pass_is_clear() { + // The bound is a guard, not a verdict: a jail that no longer matches + // anything is clear however many passes it took to get there. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let scans = std::cell::Cell::new(0); + + let reaped = reap_jailed_vmm_with( + &root, + |_jail_root| { + scans.set(scans.get() + 1); + Ok((scans.get() <= MAX_JAILED_PROCESSES).then_some(7)) + }, + |_pid, _jail_root| Reaped::Clear, + ); + + assert_eq!(reaped, Reaped::Clear); + assert_eq!( + scans.get(), + MAX_JAILED_PROCESSES + 1, + "the scan after the loop is what decides" + ); + } + + #[test] + fn a_jail_that_cannot_be_examined_is_not_reported_clear() { + // The caller deletes a directory tree on this answer, so a scan that + // could not run must not read as an empty jail. A jail root that is + // simply absent is a different thing and stays clear: nothing can be + // chrooted into a directory that is not there. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + + let reaped = reap_jailed_vmm_with( + &root, + |_jail_root| Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)), + |_pid, _jail_root| Reaped::Clear, + ); + + assert_eq!(reaped, Reaped::Unexaminable); + assert_eq!(reap_jailed_vmm(&root.join("absent")), Reaped::Clear); + } + + #[test] + fn a_scan_that_fails_after_the_bound_is_not_reported_clear() { + // The same rule on the way out of the loop as on the way in. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + let scans = std::cell::Cell::new(0); + + let reaped = reap_jailed_vmm_with( + &root, + |_jail_root| { + scans.set(scans.get() + 1); + if scans.get() > MAX_JAILED_PROCESSES { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + } else { + Ok(Some(7)) + } + }, + |_pid, _jail_root| Reaped::Clear, + ); + + assert_eq!(reaped, Reaped::Unexaminable); + } + + #[test] + fn a_still_running_vmm_carries_its_pid() { + // The caller keys the decision not to delete a directory off this, so + // the variant has to name the process it is refusing to abandon. + let still = Reaped::StillRunning { pid: 4242 }; + assert_ne!(still, Reaped::Clear); + match still { + Reaped::StillRunning { pid } => assert_eq!(pid, 4242), + Reaped::Clear | Reaped::Unexaminable => panic!("expected StillRunning"), + } + } +} diff --git a/plus/bencher_runner/src/jail/state.rs b/plus/bencher_runner/src/jail/state.rs new file mode 100644 index 000000000..9e0d34996 --- /dev/null +++ b/plus/bencher_runner/src/jail/state.rs @@ -0,0 +1,921 @@ +//! The runner's persistent state directory. +//! +//! Everything the jail needs that must outlive a single job hangs off one +//! directory: the chroot base the jailer builds under, and the sweep that +//! reclaims chroots left behind by a runner that exited without unwinding. + +#![expect(clippy::print_stderr, reason = "host preparation prints diagnostics")] + +use std::fs; +use std::os::unix::fs::PermissionsExt as _; + +use camino::{Utf8Path, Utf8PathBuf}; + +use crate::error::JailError; +use crate::jail::VmId; +use crate::jail::reap::Reaped; + +/// Subdirectory of the state directory used as the jailer's chroot base. +const CHROOT_BASE: &str = "jail"; + +/// The chroot directory inside a jail, which the jailer makes `/`. +const JAIL_ROOT: &str = "root"; + +/// The `--exec-file` base name the jailer derives the chroot layout from. +/// +/// The jailer builds `///root`, so the +/// staged Firecracker binary must be named exactly this for the runner and +/// the jailer to agree on where the chroot lives. +pub(crate) const EXEC_FILE_NAME: &str = "firecracker"; + +/// The runner's persistent state directory. +/// +/// Created at mode 0700 owned by root: it holds every job's chroot, which +/// contains the guest rootfs and the copied VMM binary. +#[derive(Debug, Clone)] +pub struct StateDir { + root: Utf8PathBuf, +} + +impl StateDir { + /// Create a handle for the state directory rooted at `root`. + /// + /// Absolute, checked here so the type carries the invariant: a relative root + /// reaches the jailer as a `--chroot-base-dir` it resolves against its own + /// working directory, and every path this type hands out would then name a + /// different file for the runner than for the jailer. See + /// [`crate::jail::check_absolute_state_dir`]. + pub fn new(root: Utf8PathBuf) -> Result { + crate::jail::check_absolute_state_dir(&root)?; + Ok(Self { root }) + } + + /// The state directory itself. + #[must_use] + pub fn path(&self) -> &Utf8Path { + &self.root + } + + /// Refuse a state directory that belongs to the host rather than to us. + /// + /// A path that does not exist, or exists and is empty, is ours to take. A + /// populated one is only ours if it already carries something the runner + /// put there. Without this, `--state-dir /var/lib` would be chmodded to + /// 0700 and take the host down with it. + /// + /// A read that fails for any reason other than absence is refused rather + /// than treated as an empty directory. It is not evidence that the + /// directory is empty, and the chmod that follows is the thing this guard + /// exists to keep off a directory that is not the runner's: a path that is + /// really a file, one whose contents cannot be listed, or a listing that + /// breaks off partway would otherwise be taken on the strength of a failed + /// check. A listing that ends early is the same failure as one that never + /// started, so it is reported rather than dropped. + /// + /// What the filesystem itself put there is not somebody else's data. See + /// [`BENIGN_ENTRIES`]. + /// + /// Ownership is proven by the tree, never by a name. An entry called `jail` + /// proves nothing: `/var/lib` on a host running this runner has one, and so + /// does any directory somebody happened to name that way, and matching on + /// the name alone let a populated system directory pass this guard and be + /// chmodded 0700, which is the whole hazard it exists for. + /// `jail/firecracker` is a path only this runner builds, and it builds the + /// tree in one step, so there is no window where a shallower half stands for + /// the whole. + fn check_root_is_ours(&self) -> Result<(), JailError> { + match self.jail_parent().try_exists() { + Ok(true) => return Ok(()), + Ok(false) => {}, + Err(e) => { + return Err(JailError::ReadStateDir { + path: self.jail_parent(), + source: e, + }); + }, + } + + let entries = match fs::read_dir(&self.root) { + Ok(entries) => entries, + // Missing: creating it is the next step. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => { + return Err(JailError::ReadStateDir { + path: self.root.clone(), + source: e, + }); + }, + }; + let mut populated = false; + for entry in entries { + let entry = entry.map_err(|e| JailError::ReadStateDir { + path: self.root.clone(), + source: e, + })?; + let name = entry.file_name(); + if BENIGN_ENTRIES.iter().any(|benign| name == *benign) { + continue; + } + populated = true; + } + if populated { + return Err(JailError::ForeignStateDir { + path: self.root.clone(), + }); + } + Ok(()) + } + + /// The jailer's `--chroot-base-dir`. + #[must_use] + pub fn chroot_base(&self) -> Utf8PathBuf { + self.root.join(CHROOT_BASE) + } + + /// The directory holding one subdirectory per jailed VMM. + /// + /// This is the level the sweep operates on. + #[must_use] + pub fn jail_parent(&self) -> Utf8PathBuf { + self.chroot_base().join(EXEC_FILE_NAME) + } + + /// The jail directory for a VM, the tree teardown removes. + #[must_use] + pub fn jail_dir(&self, vm_id: &VmId) -> Utf8PathBuf { + self.jail_parent().join(vm_id.as_str()) + } + + /// The chroot root for a VM, which becomes `/` inside the jail. + #[must_use] + pub fn jail_root(&self, vm_id: &VmId) -> Utf8PathBuf { + self.jail_dir(vm_id).join(JAIL_ROOT) + } + + /// Create the state directory tree at mode 0700. + /// + /// Idempotent. The mode is applied on every call so a directory created + /// with a laxer mode by an older runner is tightened on upgrade. That + /// tightening is why the root has to be one the runner owns: pointed at a + /// populated system directory it would otherwise chmod that directory to + /// 0700 and break the host. + pub fn create(&self) -> Result<(), JailError> { + self.check_root_is_ours()?; + for dir in [&self.root, &self.chroot_base(), &self.jail_parent()] { + fs::create_dir_all(dir).map_err(|e| JailError::CreateStateDir { + path: dir.clone(), + source: e, + })?; + fs::set_permissions(dir, fs::Permissions::from_mode(0o700)).map_err(|e| { + JailError::CreateStateDir { + path: dir.clone(), + source: e, + } + })?; + } + Ok(()) + } +} + +/// Entries that do not make a directory somebody else's. +/// +/// A dedicated filesystem is the natural home for the chroots, since each holds +/// a copy of the VMM binary and a full guest rootfs, and moving that traffic +/// off the system disk is the recommended answer to its effect on a run. A +/// freshly created ext4 volume already contains `lost+found` at its mount +/// point, so counting that as somebody else's data would refuse the exact setup +/// the state directory exists to support, and would do it with an error saying +/// the directory was not created by the runner. +/// +/// Only what the filesystem itself creates belongs here. Anything a person or +/// another program put there is what the guard is for. +const BENIGN_ENTRIES: [&str; 1] = ["lost+found"]; + +/// What one sweep did. +/// +/// Separating "reclaimed" from "left behind" is what lets the caller decide +/// about the reclaim signal. A jail whose chroot would not go away is disk, not +/// a contended benchmark, so it does not fail the job; but the sweep is the +/// mechanism that reclaims it, so a sweep that left one behind has to leave the +/// signal armed rather than spend it. See [`crate::jail`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Swept { + /// Jails whose chroot is gone. + reclaimed: usize, + /// Jails still on disk that a later sweep owes another attempt. + left_behind: usize, +} + +impl Swept { + /// Jails whose chroot is gone. + #[must_use] + pub fn reclaimed(self) -> usize { + self.reclaimed + } + + /// Whether the sweep owes nothing further. + #[must_use] + pub fn is_complete(self) -> bool { + self.left_behind == 0 + } +} + +/// Remove every jail directory under `jail_parent`, reporting what it did. +/// +/// Jobs run serially, so anything found here is stale by construction. The +/// runner disappears without unwinding in several ordinary ways, including +/// SIGKILL, a crash, and the `exec` in a self-update, and `Drop` runs in +/// none of them. Each leftover chroot holds a copy of the VMM binary and a +/// full guest rootfs image, so leaving them is not an option. +/// +/// Non-directory entries are left alone: the jailer only ever creates +/// directories here, so anything else was put there by someone else. +pub fn sweep_jails(jail_parent: &Utf8Path) -> Result { + sweep_jails_with( + jail_parent, + super::reap::reap_jailed_vmm, + super::cgroup::remove_stale_cgroup, + ) +} + +/// The sweep, with the reap and the cgroup removal injectable. +/// +/// The branch that refuses to remove a directory is the one preventing a +/// destructive action, and it only runs when a real VMM survives a real kill. +/// Manufacturing that would be testing fault injection rather than this code, +/// so the reap is a parameter and the tests supply the answer. +/// +/// The cgroup removal is a parameter for a different reason: it reaches into +/// `/sys/fs/cgroup` on the machine running the tests, and a unit test that +/// passes only because a given id happens not to exist on a dev box is +/// reading host state, not this code. +fn sweep_jails_with( + jail_parent: &Utf8Path, + reap: R, + remove_cgroup: C, +) -> Result +where + R: Fn(&Utf8Path) -> Reaped, + C: Fn(&VmId) -> Result<(), JailError>, +{ + // Absence is the only reading that means there is nothing to sweep, and it + // is the ordinary one: this runs before any jail exists in this process, so + // a parent that is not there yet is a first run. Every other failure is + // reported, because "could not look" must not reach the caller as "nothing + // was there" in the one function whose job is finding what a previous runner + // left behind. The rule `check_root_is_ours` follows, one level up. + let entries = match fs::read_dir(jail_parent) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Swept::default()), + Err(e) => { + return Err(JailError::ReadJailParent { + path: jail_parent.to_owned(), + source: e, + }); + }, + }; + + let mut swept = Swept::default(); + // The first failure is remembered but does not abandon the rest: one jail + // whose cgroup will not go away must not leave every other stale jail + // unreaped, with its chroot and cgroup still in place. + let mut failure = None; + + for entry in entries { + let outcome = match entry { + Ok(entry) => match jail_id(jail_parent, &entry) { + Ok(Some(vm_id)) => reclaim_one( + &jail_parent.join(vm_id.as_str()), + &vm_id, + &reap, + &remove_cgroup, + ), + // Not a jail, or not ours: nothing owed either way. + Ok(None) => continue, + Err(e) => Reclamation::Failed(e), + }, + // A listing that breaks off partway leaves jails unexamined, so it + // is remembered like any other failure rather than passing for a + // sweep that found nothing. What the listing did yield is still + // worth reaping. + Err(e) => { + eprintln!( + "Warning: failed to read an entry under {jail_parent}: {e}. Jails there may not have been examined." + ); + Reclamation::Failed(JailError::ReadJailParent { + path: jail_parent.to_owned(), + source: e, + }) + }, + }; + + match outcome { + Reclamation::Reclaimed => swept.reclaimed += 1, + Reclamation::LeftBehind => swept.left_behind += 1, + Reclamation::Failed(e) => { + if failure.is_none() { + failure = Some(e); + } + }, + } + } + + match failure { + Some(e) => Err(e), + None => Ok(swept), + } +} + +/// What became of one stale jail. +/// +/// The three variants are the three columns of the table in [`crate::jail`], so +/// a step added to [`reclaim_one`] has to pick one. +enum Reclamation { + /// The cgroup and the chroot are both gone. + Reclaimed, + /// Still on disk. Costs disk rather than fidelity, so the job may run, but + /// the sweep is what reclaims it and this one did not, so another is owed. + LeftBehind, + /// The host cannot be trusted to measure until this is resolved. + Failed(JailError), +} + +/// The identity of the jail an entry names, if it is one of ours. +/// +/// `Ok(None)` is an entry that is not a jail. An entry whose kind cannot be read +/// is not one of those: it may be a jail, so skipping it silently would leave a +/// live VMM unexamined while still reporting a sweep that found nothing wrong. +fn jail_id(jail_parent: &Utf8Path, entry: &fs::DirEntry) -> Result, JailError> { + match entry.file_type() { + Ok(file_type) if file_type.is_dir() => {}, + Ok(_) => return Ok(None), + Err(e) => { + eprintln!( + "Warning: cannot tell what {} under {jail_parent} is: {e}. If it is a jail, it was not examined.", + entry.file_name().display() + ); + return Err(JailError::ReadJailParent { + path: jail_parent.to_owned(), + source: e, + }); + }, + } + + // Skipped rather than lossily converted. A lossy name rebuilds into a + // path naming a different file, and everything downstream then works + // on the wrong one: the reap stats a path that does not exist and + // reports the jail clear, so a live VMM is neither reaped nor + // mentioned, and the cgroup removal targets a name nobody created. + // The runner only ever creates UTF-8 names here, so anything else is + // not ours to touch. + let file_name = entry.file_name(); + let Some(name) = file_name.to_str() else { + eprintln!( + "Warning: skipping an entry with a non-UTF-8 name under {jail_parent}; the runner did not create it" + ); + return Ok(None); + }; + Ok(Some(VmId::from_chroot_name(name.to_owned()))) +} + +/// Reap, then unwind one stale jail: its cgroup first, then its chroot. +fn reclaim_one(jail_dir: &Utf8Path, vm_id: &VmId, reap: &R, remove_cgroup: &C) -> Reclamation +where + R: Fn(&Utf8Path) -> Reaped, + C: Fn(&VmId) -> Result<(), JailError>, +{ + // Reap before removing, and only remove once the jail is clear. Deleting the + // tree under a live VMM would not stop it, and it would destroy the only + // handle for identifying that process later: without the directory the next + // sweep never sees this id, never removes its cgroup, and the cgroup leaks + // for good. + // + // Fatal to the job, not to the runner, whether the VMM was found alive or + // could not be looked for at all. A stray VMM runs untrusted guest code on + // the benchmark cores, and nothing downstream catches it: these cgroups + // claim no exclusive cpuset, so the next job's cpuset applies and verifies + // cleanly while being contended the whole time. Refusing to measure is the + // only honest answer, and a jail that could not be examined has not earned a + // better one. + match reap(&jail_dir.join(JAIL_ROOT)) { + Reaped::Clear => {}, + Reaped::StillRunning { pid } => { + eprintln!( + "Warning: leaving stale jail {jail_dir} in place because VMM pid {pid} is still running on the benchmark cores." + ); + return Reclamation::Failed(JailError::JailStillRunning { + path: jail_dir.to_owned(), + pid, + }); + }, + Reaped::Unexaminable => { + eprintln!( + "Warning: leaving stale jail {jail_dir} in place because whether a VMM is still running in it could not be determined." + ); + return Reclamation::Failed(JailError::JailUnexaminable { + path: jail_dir.to_owned(), + }); + }, + } + + // The cgroup goes first, and the chroot only once the cgroup is gone. The + // two are named by the same id, and the directory is the only handle a later + // sweep has for finding the cgroup again, so removing the directory while the + // cgroup survives strands that cgroup for good: the next sweep never sees the + // id, never retries the removal, and something may still be running on the + // benchmark cores under it. A leftover cgroup claims nothing, since these + // cgroups set no exclusive cpuset, but a removal that fails usually means + // something is still in it, which is why it is reported rather than + // swallowed. + if let Err(e) = remove_cgroup(vm_id) { + eprintln!( + "Warning: leaving stale jail {jail_dir} in place because its cgroup could not be removed: {e}" + ); + return Reclamation::Failed(e); + } + + // A chroot that will not go away costs disk, not fidelity: the VMM is gone + // and the cgroup with it, so the job may run. But the sweep is the only thing + // that reclaims it, and this one just failed to, so it is counted as still + // owed. Warning alone would have the caller spend the reclaim signal on a + // sweep that did not finish, and the leak would then survive until the daemon + // restarted. + match fs::remove_dir_all(jail_dir) { + Ok(()) => Reclamation::Reclaimed, + Err(e) => { + eprintln!( + "Warning: failed to sweep stale jail {jail_dir}: {e}. It holds a VMM binary and a full guest rootfs; the next job will try again." + ); + Reclamation::LeftBehind + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_root() -> (tempfile::TempDir, Utf8PathBuf) { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8PathBuf::try_from(dir.path().to_path_buf()).unwrap(); + (dir, root) + } + + #[test] + fn jail_layout_matches_jailer_template() { + let state = StateDir::new(Utf8PathBuf::from("/var/lib/bencher-runner")).unwrap(); + assert_eq!(state.chroot_base(), "/var/lib/bencher-runner/jail"); + assert_eq!( + state.jail_parent(), + "/var/lib/bencher-runner/jail/firecracker" + ); + assert_eq!( + state.jail_dir(&VmId::from_chroot_name("abc".to_owned())), + "/var/lib/bencher-runner/jail/firecracker/abc" + ); + // ///root + assert_eq!( + state.jail_root(&VmId::from_chroot_name("abc".to_owned())), + state + .chroot_base() + .join(EXEC_FILE_NAME) + .join("abc") + .join("root") + ); + } + + #[test] + fn create_is_idempotent_and_private() { + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")).unwrap(); + + state.create().unwrap(); + state.create().unwrap(); + + for dir in [state.path(), &state.chroot_base(), &state.jail_parent()] { + let mode = fs::metadata(dir).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o700, "{dir} should be private"); + } + } + + #[test] + fn create_tightens_a_lax_directory() { + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")).unwrap(); + fs::create_dir_all(state.path()).unwrap(); + fs::set_permissions(state.path(), fs::Permissions::from_mode(0o755)).unwrap(); + + state.create().unwrap(); + + let mode = fs::metadata(state.path()).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o700); + } + + #[test] + fn a_populated_foreign_directory_is_refused() { + // The mode tightening would otherwise chmod a system directory to + // 0700: `--state-dir /var/lib` must not take the host down. + let (_dir, root) = temp_root(); + let foreign = root.join("var-lib"); + fs::create_dir_all(foreign.join("dpkg")).unwrap(); + fs::create_dir_all(foreign.join("systemd")).unwrap(); + + StateDir::new(foreign.clone()) + .unwrap() + .create() + .unwrap_err(); + + let mode = fs::metadata(&foreign).unwrap().permissions().mode(); + assert_ne!(mode & 0o777, 0o700, "a refused root must not be chmodded"); + } + + #[test] + fn a_root_that_cannot_be_read_is_not_assumed_to_be_ours() { + // A failed read is not an empty directory. A file where the state + // directory should be reads back `ENOTDIR`, the same way an unlistable + // directory reads back `EACCES`, and neither says the path is the + // runner's to chmod. + let (_dir, root) = temp_root(); + let not_a_dir = root.join("state"); + fs::write(¬_a_dir, b"operator note").unwrap(); + + let err = StateDir::new(not_a_dir).unwrap().create().unwrap_err(); + + assert!( + matches!(err, JailError::ReadStateDir { .. }), + "a read that failed is reported, not swallowed: {err}" + ); + } + + #[test] + fn a_dedicated_filesystem_is_ours_to_take() { + // A freshly created ext4 volume mounted at the state directory holds + // `lost+found`, which the filesystem made, not an operator. Refusing it + // would block the recommended setup with an error blaming the operator + // for a directory they did not populate. + let (_dir, root) = temp_root(); + let volume = root.join("volume"); + fs::create_dir_all(volume.join("lost+found")).unwrap(); + + StateDir::new(volume.clone()).unwrap().create().unwrap(); + + let mode = fs::metadata(&volume).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o700); + assert!(volume.join("lost+found").exists(), "left where it was"); + } + + #[test] + fn a_benign_entry_does_not_launder_a_populated_directory() { + // The exemption covers what a filesystem creates, not the directory it + // happens to sit in. + let (_dir, root) = temp_root(); + let foreign = root.join("var-lib"); + fs::create_dir_all(foreign.join("lost+found")).unwrap(); + fs::create_dir_all(foreign.join("dpkg")).unwrap(); + + StateDir::new(foreign.clone()) + .unwrap() + .create() + .unwrap_err(); + + let mode = fs::metadata(&foreign).unwrap().permissions().mode(); + assert_ne!(mode & 0o777, 0o700, "a refused root must not be chmodded"); + } + + #[test] + fn an_empty_directory_is_ours_to_take() { + let (_dir, root) = temp_root(); + let empty = root.join("empty"); + fs::create_dir_all(&empty).unwrap(); + + StateDir::new(empty).unwrap().create().unwrap(); + } + + #[test] + fn a_directory_the_runner_already_used_is_ours() { + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")).unwrap(); + state.create().unwrap(); + // Something the host put there afterwards does not disown it. + fs::write(state.path().join("notes.txt"), b"operator note").unwrap(); + + state.create().unwrap(); + } + + #[test] + fn a_directory_named_like_ours_is_not_ours() { + // The hazard this guard exists for, and what a name match let through: + // `/var/lib` on a host running this runner holds a directory called + // `jail`, and so does anything anyone happened to name that way. Only + // the tree the runner builds proves the directory is the runner's. + let (_dir, root) = temp_root(); + let foreign = root.join("var-lib"); + fs::create_dir_all(foreign.join("jail")).unwrap(); + fs::create_dir_all(foreign.join("dpkg")).unwrap(); + fs::write(foreign.join(".lock"), b"").unwrap(); + + StateDir::new(foreign.clone()) + .unwrap() + .create() + .unwrap_err(); + + let mode = fs::metadata(&foreign).unwrap().permissions().mode(); + assert_ne!(mode & 0o777, 0o700, "a refused root must not be chmodded"); + } + + #[test] + fn the_tree_is_what_proves_the_directory_is_ours() { + // A state directory the runner has used carries `jail/firecracker`, + // which it creates in one step, so a shallower half never stands for the + // whole. Anything the operator put there afterwards does not disown it. + let (_dir, root) = temp_root(); + let state = root.join("state"); + fs::create_dir_all(state.join("jail").join("firecracker")).unwrap(); + fs::write(state.join("notes.txt"), b"operator note").unwrap(); + + StateDir::new(state.clone()).unwrap().create().unwrap(); + + let mode = fs::metadata(&state).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o700); + } + + #[test] + fn a_relative_state_directory_is_refused() { + // The invariant lives on the type: the jailer resolves the path it is + // handed against its own working directory, so a relative one names a + // different place for the jailer than for the runner. + StateDir::new(Utf8PathBuf::from("bencher-runner")).unwrap_err(); + StateDir::new(Utf8PathBuf::from("./bencher-runner")).unwrap_err(); + StateDir::new(Utf8PathBuf::from("/var/lib/bencher-runner")).unwrap(); + } + + #[test] + fn sweep_removes_stale_jails() { + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")).unwrap(); + state.create().unwrap(); + + // Two stale jails, one with a nested chroot tree. + fs::create_dir_all(state.jail_root(&VmId::from_chroot_name("one".to_owned()))).unwrap(); + fs::write( + state + .jail_root(&VmId::from_chroot_name("one".to_owned())) + .join("rootfs.ext4"), + b"stale", + ) + .unwrap(); + fs::create_dir_all(state.jail_dir(&VmId::from_chroot_name("two".to_owned()))).unwrap(); + + assert_eq!( + sweep_jails_with(&state.jail_parent(), |_j| Reaped::Clear, |_v| Ok(())) + .unwrap() + .reclaimed(), + 2 + ); + assert!( + !state + .jail_dir(&VmId::from_chroot_name("one".to_owned())) + .exists() + ); + assert!( + !state + .jail_dir(&VmId::from_chroot_name("two".to_owned())) + .exists() + ); + assert!(state.jail_parent().exists()); + } + + #[test] + fn sweep_leaves_unrelated_entries_alone() { + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")).unwrap(); + state.create().unwrap(); + + let note = state.jail_parent().join("NOTES.txt"); + fs::write(¬e, b"not a jail").unwrap(); + fs::create_dir_all(state.jail_dir(&VmId::from_chroot_name("stale".to_owned()))).unwrap(); + + assert_eq!( + sweep_jails_with(&state.jail_parent(), |_j| Reaped::Clear, |_v| Ok(())) + .unwrap() + .reclaimed(), + 1 + ); + assert!( + !state + .jail_dir(&VmId::from_chroot_name("stale".to_owned())) + .exists() + ); + assert!(note.exists(), "non-directory entries are not the sweep's"); + } + + #[test] + fn a_jail_whose_vmm_survives_is_left_in_place() { + // Removing the tree would not stop the VMM, and it would destroy the + // only handle for identifying that process on a later sweep. + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")).unwrap(); + state.create().unwrap(); + let live = VmId::from_chroot_name("live".to_owned()); + let dead = VmId::from_chroot_name("dead".to_owned()); + fs::create_dir_all(state.jail_root(&live)).unwrap(); + fs::create_dir_all(state.jail_root(&dead)).unwrap(); + + let err = sweep_jails_with( + &state.jail_parent(), + |jail_root| { + if jail_root.as_str().contains("live") { + Reaped::StillRunning { pid: 4242 } + } else { + Reaped::Clear + } + }, + |_vm_id| Ok(()), + ) + .unwrap_err(); + + assert!( + state.jail_dir(&live).exists(), + "a jail with a live VMM must not be removed" + ); + assert!( + !state.jail_dir(&dead).exists(), + "one unreapable jail must not abandon the rest of the sweep" + ); + let message = err.to_string(); + assert!(message.contains("4242"), "names the pid: {message}"); + assert!(message.contains("live"), "names the jail: {message}"); + } + + #[test] + fn a_surviving_vmm_fails_every_attempt_not_just_the_first() { + // A host that can never clear a jail has to tell the operator on every + // job, not once. Nothing latches, so the sweep is re-attempted and + // reports again. + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")).unwrap(); + state.create().unwrap(); + let live = VmId::from_chroot_name("live".to_owned()); + fs::create_dir_all(state.jail_root(&live)).unwrap(); + + let stuck = |_jail_root: &Utf8Path| Reaped::StillRunning { pid: 7 }; + for attempt in 1..=3 { + let err = sweep_jails_with(&state.jail_parent(), stuck, |_vm_id| Ok(())).unwrap_err(); + assert!( + err.to_string().contains('7'), + "attempt {attempt} must report the pid" + ); + assert!(state.jail_dir(&live).exists()); + } + } + + #[test] + fn a_jail_whose_cgroup_survives_keeps_the_chroot_that_names_it() { + // The chroot name is the only handle a later sweep has for finding the + // cgroup, so a directory removed while its cgroup survives strands + // that cgroup for good: nothing ever sees the id again. One stuck + // cgroup must still not abandon the rest of the sweep. + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")).unwrap(); + state.create().unwrap(); + let stuck = VmId::from_chroot_name("stuck".to_owned()); + let clear = VmId::from_chroot_name("clear".to_owned()); + fs::create_dir_all(state.jail_root(&stuck)).unwrap(); + fs::create_dir_all(state.jail_root(&clear)).unwrap(); + + let err = sweep_jails_with( + &state.jail_parent(), + |_jail_root| Reaped::Clear, + |vm_id| { + if vm_id.as_str() == "stuck" { + Err(JailError::StaleCgroup { + path: Utf8PathBuf::from("/sys/fs/cgroup/bencher/stuck"), + source: std::io::Error::from(std::io::ErrorKind::DirectoryNotEmpty), + }) + } else { + Ok(()) + } + }, + ) + .unwrap_err(); + + assert!( + state.jail_dir(&stuck).exists(), + "the chroot names the cgroup that has to be retried" + ); + assert!( + !state.jail_dir(&clear).exists(), + "one stuck cgroup must not abandon the rest of the sweep" + ); + assert!(err.to_string().contains("stuck"), "names the cgroup: {err}"); + } + + #[test] + fn a_chroot_that_will_not_go_away_is_owed_another_sweep() { + // Disk, not fidelity: the VMM is gone and the cgroup with it, so the job + // runs. But the sweep is the only thing that reclaims the tree, and this + // sweep did not, so it reports the debt rather than passing for complete. + // Warning alone would have the caller spend the reclaim signal and the + // leak would outlive every later job. + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")).unwrap(); + state.create().unwrap(); + let stuck = VmId::from_chroot_name("stuck".to_owned()); + fs::create_dir_all(state.jail_root(&stuck)).unwrap(); + // A jail directory that `remove_dir_all` cannot finish: the tree is + // unsearchable, so the walk inside it fails. + fs::set_permissions(state.jail_dir(&stuck), fs::Permissions::from_mode(0o000)).unwrap(); + + let swept = sweep_jails_with( + &state.jail_parent(), + |_jail_root| Reaped::Clear, + |_vm_id| Ok(()), + ) + .unwrap(); + + // Restored before the assertions so the temp directory can be cleaned + // up whichever way they go. + fs::set_permissions(state.jail_dir(&stuck), fs::Permissions::from_mode(0o700)).unwrap(); + + assert_eq!(swept.reclaimed(), 0); + assert!( + !swept.is_complete(), + "a sweep that left a chroot behind still owes one" + ); + } + + #[test] + fn a_jail_that_could_not_be_examined_is_left_in_place() { + // The reap could not establish whether a VMM is in there. Removing the + // tree on that would be the same destructive step as removing it under a + // VMM known to be alive, so it gets the same answer. + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")).unwrap(); + state.create().unwrap(); + let unknown = VmId::from_chroot_name("unknown".to_owned()); + fs::create_dir_all(state.jail_root(&unknown)).unwrap(); + + let err = sweep_jails_with( + &state.jail_parent(), + |_jail_root| Reaped::Unexaminable, + |_vm_id| Ok(()), + ) + .unwrap_err(); + + assert!( + state.jail_dir(&unknown).exists(), + "not ours to delete blind" + ); + assert!( + matches!(err, JailError::JailUnexaminable { .. }), + "a jail that could not be checked fails the job: {err}" + ); + } + + #[test] + fn a_cleared_jail_is_still_swept() { + let (_dir, root) = temp_root(); + let state = StateDir::new(root.join("state")).unwrap(); + state.create().unwrap(); + fs::create_dir_all(state.jail_root(&VmId::from_chroot_name("one".to_owned()))).unwrap(); + + let swept = sweep_jails_with( + &state.jail_parent(), + |_jail_root| Reaped::Clear, + |_vm_id| Ok(()), + ) + .unwrap(); + + assert_eq!(swept.reclaimed(), 1); + assert!(swept.is_complete()); + } + + #[test] + fn a_jail_parent_that_cannot_be_read_is_not_an_empty_one() { + // The sweep exists to find what a previous runner left behind, so a + // read it could not perform must not reach the caller as a clean host. + // A file where the jail parent should be reads back `ENOTDIR`, the same + // way an unlistable directory reads back `EACCES`. + let (_dir, root) = temp_root(); + let not_a_dir = root.join("firecracker"); + fs::write(¬_a_dir, b"in the way").unwrap(); + + let err = sweep_jails_with(¬_a_dir, |_j| Reaped::Clear, |_v| Ok(())).unwrap_err(); + + assert!( + matches!(err, JailError::ReadJailParent { .. }), + "a read that failed is reported, not counted as zero jails: {err}" + ); + } + + #[test] + fn sweep_missing_parent_is_zero() { + let (_dir, root) = temp_root(); + assert_eq!( + sweep_jails_with(&root.join("nope"), |_j| Reaped::Clear, |_v| Ok(())) + .unwrap() + .reclaimed(), + 0 + ); + } +} diff --git a/plus/bencher_runner/src/jailer_bin.rs b/plus/bencher_runner/src/jailer_bin.rs new file mode 100644 index 000000000..9044c684d --- /dev/null +++ b/plus/bencher_runner/src/jailer_bin.rs @@ -0,0 +1,52 @@ +//! Bundled Firecracker jailer binary support. +//! +//! This module provides access to the bundled `jailer` binary, which confines +//! the Firecracker VMM to a chroot under an unprivileged uid. +//! +//! - In **release** builds: The binary is embedded directly in bencher-runner. +//! - In **debug** builds: The binary is loaded from disk (downloaded by build.rs). +//! +//! The jailer ships in the same release archive as Firecracker, so both are +//! bundled from a single download under a single hash. Extracting both per job +//! keeps the VMM and its jailer at the same version across a runner self-update. +//! +//! # Example +//! +//! ```ignore +//! use bencher_runner::jailer_bin::{write_jailer_to_file, JAILER_BUNDLED}; +//! +//! if JAILER_BUNDLED { +//! write_jailer_to_file("/tmp/jailer".as_ref())?; +//! } +//! ``` + +use std::io; + +use camino::Utf8Path; + +// Include the generated jailer module +include!(concat!(env!("OUT_DIR"), "/jailer_generated.rs")); + +/// Write the bundled jailer binary to a file. +/// +/// The file is written with executable permissions (0o755). +/// +/// # Arguments +/// +/// * `path` - The destination path for the jailer binary +/// +/// # Errors +/// +/// Returns an error if the file cannot be written. +pub fn write_jailer_to_file(path: &Utf8Path) -> io::Result<()> { + use std::os::unix::fs::PermissionsExt as _; + + std::fs::write(path, jailer_bytes())?; + + // Make it executable + let mut perms = std::fs::metadata(path)?.permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(path, perms)?; + + Ok(()) +} diff --git a/plus/bencher_runner/src/lib.rs b/plus/bencher_runner/src/lib.rs index c9a95a2c5..ad502659c 100644 --- a/plus/bencher_runner/src/lib.rs +++ b/plus/bencher_runner/src/lib.rs @@ -33,6 +33,8 @@ pub mod init; #[cfg(feature = "plus")] pub mod jail; #[cfg(all(feature = "plus", target_os = "linux"))] +pub mod jailer_bin; +#[cfg(all(feature = "plus", target_os = "linux"))] pub mod kernel; #[cfg(feature = "plus")] mod local; @@ -60,7 +62,10 @@ pub use config::Config; #[cfg(feature = "plus")] pub use error::{ConfigError, ExecutionError, JailError, RunnerError}; #[cfg(feature = "plus")] -pub use jail::ResourceLimits; +pub use jail::{ + DEFAULT_JAIL_GID, DEFAULT_JAIL_UID, DEFAULT_STATE_DIR, HostPreparation, JailUser, + check_absolute_state_dir, +}; #[cfg(feature = "plus")] pub use log_level::SandboxLogLevel; #[cfg(feature = "plus")] diff --git a/plus/bencher_runner/src/local_isolation.rs b/plus/bencher_runner/src/local_isolation.rs index ce25beae7..b7f356a54 100644 --- a/plus/bencher_runner/src/local_isolation.rs +++ b/plus/bencher_runner/src/local_isolation.rs @@ -57,12 +57,22 @@ impl LocalIsolation { }; let benchmark = layout.benchmark.clone(); - let run_id = format!("local-{}", uuid::Uuid::new_v4()); + let run_id = crate::jail::VmId::from_chroot_name(format!("local-{}", uuid::Uuid::new_v4())); - let cgroup = match crate::jail::CgroupManager::new(&run_id) { + // No chroot names this cgroup, and no sweep walks it, so a teardown it + // cannot finish has nothing to hand the work to. + let signals = crate::jail::JailSignals::unwatched(); + let cgroup = match crate::jail::CgroupManager::new(&run_id, signals) { Ok(cgroup) => { - if let Err(e) = cgroup.apply_cpuset(layout) { - eprintln!("Warning: failed to apply cpuset for local run: {e}"); + // Best effort here, unlike the sandboxed path: a local run + // makes no confinement claim to begin with, so losing the + // cpuset degrades its numbers rather than falsifying them. + match cgroup.apply_cpuset(layout) { + Ok(crate::jail::Cpuset::Applied) => {}, + Ok(crate::jail::Cpuset::Unavailable(reason)) => { + eprintln!("Warning: this local run has no CPU isolation ({reason})"); + }, + Err(e) => eprintln!("Warning: failed to apply cpuset for local run: {e}"), } // Keep benchmark memory resident, mirroring the Firecracker path if let Err(e) = cgroup.disable_swap() { diff --git a/plus/bencher_runner/src/metrics.rs b/plus/bencher_runner/src/metrics.rs index 4bba7d2dc..3ff8a97d4 100644 --- a/plus/bencher_runner/src/metrics.rs +++ b/plus/bencher_runner/src/metrics.rs @@ -47,19 +47,33 @@ pub struct CgroupMetrics { /// Read cgroup metrics from the given cgroup path. /// /// Reads `cpu.stat` and `memory.peak` from the cgroup directory. -/// Returns `None` if the path doesn't exist. +/// Returns `None` when there is no cgroup to read. +/// +/// Every field is optional, and a field that could not be read stays absent +/// rather than becoming a number. These are reported to an operator as measured +/// values, so a zero standing in for a failed read is the plainest form of the +/// one thing measurement is never allowed to do. Absence is already how this +/// reports "no cgroup at all", so it costs nothing to be honest per field. +/// +/// That is also why the stat below is the one gating read in this crate that may +/// stay: it can only withhold a reading, never invent one. Nothing here can +/// answer with a number it did not read, so a failure reaches the operator as a +/// field that is not there. A stat that fails does not suppress the reads either, +/// which is the one thing worth tightening: only a stat that succeeded and said +/// absent means there is nothing to read, and anything else goes on to ask the +/// files themselves, since they report what they can and nothing more. pub fn read_cgroup_metrics(cgroup_path: &Utf8Path) -> Option { - if !cgroup_path.exists() { + if cgroup_path.try_exists().is_ok_and(|exists| !exists) { return None; } - let cpu_stat = read_cpu_stat(cgroup_path); + let cpu_stat = read_cpu_stat(cgroup_path).unwrap_or_default(); let memory_peak = read_file_u64(&cgroup_path.join("memory.peak")); Some(CgroupMetrics { - cpu_usage_us: cpu_stat.as_ref().map(|s| s.usage_usec), - cpu_user_us: cpu_stat.as_ref().map(|s| s.user_usec), - cpu_system_us: cpu_stat.as_ref().map(|s| s.system_usec), + cpu_usage_us: cpu_stat.usage_usec, + cpu_user_us: cpu_stat.user_usec, + cpu_system_us: cpu_stat.system_usec, memory_peak_bytes: memory_peak, }) } @@ -72,37 +86,37 @@ pub fn format_metrics(metrics: &RunMetrics) -> Option { Some(format!("---BENCHER_METRICS:{json}---")) } +/// The three fields of `cpu.stat` this runner reports. +/// +/// Each one is what the file said, or nothing. A field the file did not carry, or +/// carried unparseably, is not zero usage: zero is a measurement, and this never +/// measured it. +#[derive(Default)] #[expect( clippy::struct_field_names, reason = "matches cgroup cpu.stat field names" )] struct CpuStat { - usage_usec: u64, - user_usec: u64, - system_usec: u64, + usage_usec: Option, + user_usec: Option, + system_usec: Option, } fn read_cpu_stat(cgroup_path: &Utf8Path) -> Option { let content = std::fs::read_to_string(cgroup_path.join("cpu.stat")).ok()?; - let mut usage = None; - let mut user = None; - let mut system = None; + let mut stat = CpuStat::default(); for line in content.lines() { let mut parts = line.split_whitespace(); match (parts.next(), parts.next()) { - (Some("usage_usec"), Some(v)) => usage = v.parse().ok(), - (Some("user_usec"), Some(v)) => user = v.parse().ok(), - (Some("system_usec"), Some(v)) => system = v.parse().ok(), + (Some("usage_usec"), Some(v)) => stat.usage_usec = v.parse().ok(), + (Some("user_usec"), Some(v)) => stat.user_usec = v.parse().ok(), + (Some("system_usec"), Some(v)) => stat.system_usec = v.parse().ok(), _ => {}, } } - Some(CpuStat { - usage_usec: usage.unwrap_or(0), - user_usec: user.unwrap_or(0), - system_usec: system.unwrap_or(0), - }) + Some(stat) } fn read_file_u64(path: &Utf8Path) -> Option { @@ -129,21 +143,23 @@ mod tests { fs::write(path.join("cpu.stat"), content).unwrap(); let stat = read_cpu_stat(path).unwrap(); - assert_eq!(stat.usage_usec, 12345); - assert_eq!(stat.user_usec, 6000); - assert_eq!(stat.system_usec, 6345); + assert_eq!(stat.usage_usec, Some(12345)); + assert_eq!(stat.user_usec, Some(6000)); + assert_eq!(stat.system_usec, Some(6345)); } #[test] - fn read_cpu_stat_missing_fields_default_to_zero() { + fn a_field_the_file_did_not_carry_is_absent_not_zero() { + // Zero is a measurement. A field that was never read has to reach the + // operator as missing, which is what the reported type already allows. let dir = tempfile::tempdir().unwrap(); let path = tempdir_utf8(&dir); fs::write(path.join("cpu.stat"), "usage_usec 100\n").unwrap(); let stat = read_cpu_stat(path).unwrap(); - assert_eq!(stat.usage_usec, 100); - assert_eq!(stat.user_usec, 0); - assert_eq!(stat.system_usec, 0); + assert_eq!(stat.usage_usec, Some(100)); + assert_eq!(stat.user_usec, None); + assert_eq!(stat.system_usec, None); } #[test] @@ -157,9 +173,9 @@ mod tests { .unwrap(); let stat = read_cpu_stat(path).unwrap(); - assert_eq!(stat.usage_usec, 0); // parse fails -> unwrap_or(0) - assert_eq!(stat.user_usec, 100); - assert_eq!(stat.system_usec, 0); // no value at all + assert_eq!(stat.usage_usec, None, "a value that would not parse"); + assert_eq!(stat.user_usec, Some(100)); + assert_eq!(stat.system_usec, None, "no value at all"); } #[test] @@ -169,9 +185,9 @@ mod tests { fs::write(path.join("cpu.stat"), "").unwrap(); let stat = read_cpu_stat(path).unwrap(); - assert_eq!(stat.usage_usec, 0); - assert_eq!(stat.user_usec, 0); - assert_eq!(stat.system_usec, 0); + assert_eq!(stat.usage_usec, None); + assert_eq!(stat.user_usec, None); + assert_eq!(stat.system_usec, None); } #[test] @@ -260,6 +276,22 @@ mod tests { assert!(read_cgroup_metrics(Utf8Path::new("/nonexistent")).is_none()); } + #[test] + fn a_cgroup_whose_files_cannot_be_read_reports_no_numbers() { + // The property that makes this module's gating stat harmless: every + // failure here withholds a field, and none of them invents one. An empty + // reading is honest; a zero would not be. + let dir = tempfile::tempdir().unwrap(); + let path = tempdir_utf8(&dir); + + let metrics = read_cgroup_metrics(path).unwrap(); + + assert_eq!(metrics.cpu_usage_us, None); + assert_eq!(metrics.cpu_user_us, None); + assert_eq!(metrics.cpu_system_us, None); + assert_eq!(metrics.memory_peak_bytes, None); + } + // --- format_metrics --- #[test] diff --git a/plus/bencher_runner/src/run.rs b/plus/bencher_runner/src/run.rs index a9405749e..4f2abeab8 100644 --- a/plus/bencher_runner/src/run.rs +++ b/plus/bencher_runner/src/run.rs @@ -71,6 +71,10 @@ pub struct RunArgs { pub sandbox_log_level: crate::SandboxLogLevel, /// Sandbox mode for benchmark execution. pub sandbox: Option, + /// The runner's persistent state directory. + pub state_dir: Utf8PathBuf, + /// The unprivileged uid and gid the jailed VMM drops to. + pub jail_user: crate::jail::JailUser, } /// Build a `Config` from CLI `RunArgs`. @@ -121,6 +125,8 @@ fn build_config_from_run_args(args: &RunArgs) -> Result Result<(), RunnerError> { } } + // Owned by this invocation rather than shared: the latch belongs to one + // runner process and nothing else can observe or reset it. + let mut host = crate::jail::HostPreparation::new(); + let iter_count = args.iter.as_usize(); for iteration in 0..iter_count { - match execute(&config, None) { + match execute(&config, &mut host, None) { Ok(output) => { println!("{}", output.stdout); if !output.stderr.is_empty() { @@ -396,15 +406,23 @@ pub fn resolve_oci_config( /// # Returns /// /// The benchmark output including exit code and stdout. +#[cfg_attr( + not(target_os = "linux"), + expect( + unused_variables, + reason = "host preparation is Linux-only, as is the VM executor it prepares for" + ) +)] pub fn execute( config: &crate::Config, + host: &mut crate::jail::HostPreparation, cancel_flag: Option<&Arc>, ) -> Result { match config.sandbox { Some(bencher_json::Sandbox::Firecracker) => { #[cfg(target_os = "linux")] { - crate::vm::vm_execute(config, cancel_flag) + crate::vm::vm_execute(config, host, cancel_flag) } #[cfg(not(target_os = "linux"))] { diff --git a/plus/bencher_runner/src/tuning/dma_latency.rs b/plus/bencher_runner/src/tuning/dma_latency.rs index 516c8854a..5ae1b4374 100644 --- a/plus/bencher_runner/src/tuning/dma_latency.rs +++ b/plus/bencher_runner/src/tuning/dma_latency.rs @@ -59,6 +59,7 @@ mod tests { TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), } } diff --git a/plus/bencher_runner/src/tuning/kernel_work.rs b/plus/bencher_runner/src/tuning/kernel_work.rs index 2ef2d5381..d613442a8 100644 --- a/plus/bencher_runner/src/tuning/kernel_work.rs +++ b/plus/bencher_runner/src/tuning/kernel_work.rs @@ -101,6 +101,7 @@ mod tests { TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), } } diff --git a/plus/bencher_runner/src/tuning/mod.rs b/plus/bencher_runner/src/tuning/mod.rs index 92a7b655b..dab6e03cb 100644 --- a/plus/bencher_runner/src/tuning/mod.rs +++ b/plus/bencher_runner/src/tuning/mod.rs @@ -158,6 +158,13 @@ struct SavedSetting { #[cfg(target_os = "linux")] pub struct TuningGuard { saved: Vec, + /// Cgroups to remove once every setting has been restored. + /// + /// Removed rather than reset, because an empty cgroup carrying a stale + /// cpuset is residue the runner owns and nothing else reads. `rmdir` is + /// self-guarding: it succeeds only when the cgroup is empty, so a concurrent + /// runner's job or a task this one could not reclaim leaves it in place. + remove_if_empty: Vec, /// File descriptors held open for the lifetime of the guard /// (e.g., the PM `QoS` constraint on `/dev/cpu_dma_latency`). /// Dropped after the saved settings are restored. @@ -174,6 +181,11 @@ impl TuningGuard { pub(crate) fn save_restore(&mut self, path: Utf8PathBuf, value: String, label: String) { self.saved.push(SavedSetting { path, value, label }); } + + /// Record a cgroup to remove once the settings have been restored. + pub(crate) fn remove_when_empty(&mut self, path: Utf8PathBuf) { + self.remove_if_empty.push(path); + } } #[cfg(target_os = "linux")] @@ -182,6 +194,28 @@ impl Drop for TuningGuard { for setting in self.saved.iter().rev() { restore(&setting.path, &setting.value, &setting.label); } + // After the settings, because a cgroup that is about to be removed still + // has to have its values put back for the case where the removal cannot + // happen. + for path in &self.remove_if_empty { + remove_empty_cgroup(path); + } + } +} + +/// Remove a cgroup the runner is done with, if nothing is left in it. +/// +/// The whole point is that this can fail and that failing is correct. A cgroup +/// still holding a task, or a concurrent runner's job, refuses `rmdir` with +/// `EBUSY`, and the alternative to leaving it is tearing a cgroup out from under +/// something that is using it. +#[cfg(target_os = "linux")] +#[expect(clippy::print_stdout, reason = "tuning reports what it unwound")] +fn remove_empty_cgroup(path: &Utf8Path) { + match std::fs::remove_dir(path) { + Ok(()) => println!(" Tuning: removed the cgroup {path}"), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}, + Err(e) => println!(" Tuning: left the cgroup {path} in place: {e}"), } } @@ -191,6 +225,7 @@ pub fn apply(config: &TuningConfig) -> TuningGuard { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; if config.disable_aslr { @@ -638,6 +673,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; guard.saved.push(SavedSetting { path: file_path.clone(), @@ -668,6 +704,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; guard.saved.push(SavedSetting { path: path1.clone(), @@ -691,6 +728,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; write_sysctl(&mut guard, "/nonexistent/path/value", "0", "test"); assert!( @@ -711,6 +749,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; write_sysctl(&mut guard, path.to_str().unwrap(), "0", "test"); assert!( @@ -750,6 +789,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; write_bracketed_sysctl(&mut guard, path.to_str().unwrap(), "never", "test"); @@ -770,6 +810,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; write_bracketed_sysctl(&mut guard, path.to_str().unwrap(), "never", "test"); @@ -793,6 +834,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; write_bracketed_sysctl(&mut guard, path.to_str().unwrap(), "never", "test"); @@ -812,6 +854,7 @@ mod tests { let mut guard = TuningGuard { saved: Vec::new(), held_fds: Vec::new(), + remove_if_empty: Vec::new(), }; write_sysctl(&mut guard, path.to_str().unwrap(), "10", "test"); diff --git a/plus/bencher_runner/src/tuning/partition.rs b/plus/bencher_runner/src/tuning/partition.rs index e20e7a78a..0ba0b8a97 100644 --- a/plus/bencher_runner/src/tuning/partition.rs +++ b/plus/bencher_runner/src/tuning/partition.rs @@ -71,13 +71,21 @@ impl BencherPartition { return PartitionLevel::Member; } - if !self.path.exists() - && let Err(e) = fs::create_dir_all(&self.path) - { + if let Err(e) = fs::create_dir_all(&self.path) { eprintln!("Warning: failed to create cgroup {}: {e}", self.path); return PartitionLevel::Member; } + // Removed on the way out, once the values have been restored. An empty + // `bencher` cgroup left carrying this run's cpuset is residue: nothing + // else reads that cgroup, the next job recreates it on demand, and + // clearing `cpuset.cpus` back to inherit-everything is refused with + // `EIO` while any task remains in a descendant, so the restore alone + // cannot always undo what this does. Registered whoever created the + // directory, because the residue is the same either way and `rmdir` + // refuses while anything is still inside. + guard.remove_when_empty(self.path.clone()); + // A partition needs explicit cpus and mems. Mems mirror the // root's effective nodes so multi-node NUMA hosts are not forced // onto node 0. @@ -89,10 +97,23 @@ impl BencherPartition { ) { return PartitionLevel::Member; } + // A node set that could not be read is not node 0. Degrading to member + // is the declared, loud absence of isolation; writing a guessed node set + // would confine the benchmark's memory on the strength of a read that + // did not happen. + let mems = match effective_mems(&self.root) { + Ok(mems) => mems, + Err(e) => { + eprintln!( + "Warning: failed to read the cgroup root's effective memory nodes ({e}); no cpuset partition" + ); + return PartitionLevel::Member; + }, + }; if !save_and_write( guard, &self.path.join("cpuset.mems"), - &effective_mems(&self.root), + &mems, "bencher cpuset.mems", ) { return PartitionLevel::Member; diff --git a/plus/bencher_runner/src/up/job.rs b/plus/bencher_runner/src/up/job.rs index 2f49dae2e..42fb8ae77 100644 --- a/plus/bencher_runner/src/up/job.rs +++ b/plus/bencher_runner/src/up/job.rs @@ -44,6 +44,7 @@ pub fn execute_job( config: &UpConfig, job: &JsonClaimedJob, ws: &Arc>, + host: &mut crate::jail::HostPreparation, ) -> JobFinishResult { // Only allow jobs with a known sandbox type or explicit opt-in for non-sandboxed. if let Err(reason) = check_sandbox_allowed(job.spec.sandbox, config.allow_no_sandbox) { @@ -101,7 +102,7 @@ pub fn execute_job( job.uuid ); let start = build_time.then(std::time::Instant::now); - let result = crate::execute(&job_config, Some(&cancel_flag)); + let result = crate::execute(&job_config, host, Some(&cancel_flag)); let elapsed = start.map(|s| s.elapsed()); match result { Ok(output) => { @@ -302,10 +303,37 @@ fn build_metric_output( /// authenticated image pulls. /// /// CPU layout from the up config is passed through for core isolation. +/// +/// The up config is destructured rather than read field by field: every +/// runner-level setting has to reach the job, and a field that is added but +/// never passed through is silently ignored at runtime. Destructuring makes +/// that a build error instead. fn build_config_from_job( up_config: &UpConfig, job: &JsonClaimedJob, ) -> Result { + let UpConfig { + // Protocol and identity settings, which do not describe a run. + host: _, + key: _, + runner: _, + poll_timeout_secs: _, + tuning: _, + no_auto_update: _, + update_channel: _, + max_download_size: _, + allow_no_sandbox: _, + // Run settings, every one of which must reach the config below. + cpu_layout, + max_output_size, + max_file_count, + max_symlinks, + grace_period, + sandbox_log_level, + state_dir, + jail_user, + } = up_config; + let spec = &job.spec; let config = &job.config; @@ -342,35 +370,40 @@ fn build_config_from_job( // Pass all file paths through for multi-file output extraction runner_config = runner_config.with_file_paths_opt(config.file_paths.clone()); + // Pass through the runner's state directory: the jail chroot for the + // job is built under it. + runner_config = runner_config.with_state_dir(state_dir.clone()); + runner_config = runner_config.with_jail_user(*jail_user); + // Pass through CPU layout for core isolation - if let Some(cpu_layout) = &up_config.cpu_layout + if let Some(cpu_layout) = cpu_layout && cpu_layout.has_isolation() { runner_config = runner_config.with_cpu_layout(cpu_layout.clone()); } // Pass through max output size if configured - if let Some(max_output_size) = up_config.max_output_size { + if let Some(max_output_size) = *max_output_size { runner_config = runner_config.with_max_output_size(max_output_size); } // Pass through max file count if configured - if let Some(max_file_count) = up_config.max_file_count { + if let Some(max_file_count) = *max_file_count { runner_config = runner_config.with_max_file_count(max_file_count); } // Pass through max symlinks if configured - if let Some(max_symlinks) = up_config.max_symlinks { + if let Some(max_symlinks) = *max_symlinks { runner_config = runner_config.with_max_symlinks(max_symlinks); } // Pass through grace period if configured - if let Some(grace_period) = up_config.grace_period { + if let Some(grace_period) = *grace_period { runner_config = runner_config.with_grace_period(grace_period); } // Pass through sandbox log level - runner_config.sandbox_log_level = up_config.sandbox_log_level; + runner_config.sandbox_log_level = *sandbox_log_level; // Pass through sandbox mode from the job spec runner_config = runner_config.with_sandbox(spec.sandbox); @@ -543,6 +576,8 @@ mod tests { no_auto_update: false, update_channel: bencher_valid::UpdateChannel::default(), max_download_size: None, + state_dir: Utf8PathBuf::from(crate::jail::DEFAULT_STATE_DIR), + jail_user: crate::jail::JailUser::default(), } } @@ -783,6 +818,48 @@ mod tests { ); } + #[test] + fn state_dir_passed_through() { + // The daemon prepares and sweeps the state directory it was given, so + // a job that builds its jail somewhere else leaks a chroot on every + // unclean exit and creates the real tree outside the swept location. + let mut up_config = test_up_config(); + up_config.state_dir = Utf8PathBuf::from("/mnt/fast/runner-state"); + let job = test_job(1, mib_to_bytes(512), mib_to_bytes(1024), 300, false); + + let result = build_config_from_job(&up_config, &job).unwrap(); + + assert_eq!( + result.state_dir, + Utf8PathBuf::from("/mnt/fast/runner-state") + ); + } + + #[test] + fn state_dir_defaults_when_the_daemon_uses_the_default() { + let up_config = test_up_config(); + let job = test_job(1, mib_to_bytes(512), mib_to_bytes(1024), 300, false); + + let result = build_config_from_job(&up_config, &job).unwrap(); + + assert_eq!(result.state_dir, crate::jail::DEFAULT_STATE_DIR); + } + + #[test] + fn jail_user_passed_through() { + // A host that allocates ids in the default range needs the override to + // reach the job, or the VMM shares a uid with a local account that can + // signal it. + let mut up_config = test_up_config(); + up_config.jail_user = crate::jail::JailUser::new(4242, 4243).unwrap(); + let job = test_job(1, mib_to_bytes(512), mib_to_bytes(1024), 300, false); + + let result = build_config_from_job(&up_config, &job).unwrap(); + + assert_eq!(result.jail_user.uid(), 4242); + assert_eq!(result.jail_user.gid(), 4243); + } + #[test] fn cpu_layout_passed_through() { let up_config = test_up_config(); diff --git a/plus/bencher_runner/src/up/mod.rs b/plus/bencher_runner/src/up/mod.rs index 212f97227..c4bca4aea 100644 --- a/plus/bencher_runner/src/up/mod.rs +++ b/plus/bencher_runner/src/up/mod.rs @@ -67,6 +67,10 @@ pub struct UpConfig { pub update_channel: bencher_valid::UpdateChannel, /// Maximum download size in bytes for self-update binaries. pub max_download_size: Option, + /// The runner's persistent state directory. + pub state_dir: camino::Utf8PathBuf, + /// The unprivileged uid and gid the jailed VMM drops to. + pub jail_user: crate::jail::JailUser, } pub struct Up { @@ -104,6 +108,13 @@ impl Up { // Warn about host conditions that limit benchmark accuracy (Linux only) preflight::print_host_warnings(); + // The host is deliberately NOT prepared here. A Runner that serves + // only non-sandboxed Specs is a supported configuration and must come + // up without root, and the daemon learns its Specs from the server, so + // it cannot know at startup whether it will ever build a jail. + // Preparation happens on demand, before the first job that does. + println!(" State directory: {}", self.config.state_dir); + // Serialize host-global tuning across runner processes. Declared // before the guard so the lock releases only after restore completes. let host_lock = crate::tuning::HostTuningLock::acquire(); @@ -173,6 +184,9 @@ fn run_driver(config: &UpConfig, channel_url: &Url, key: &str) -> Result<(), UpE { println!(" Update channel: {channel}"); } + // Owned by the daemon loop. The latch belongs to this runner process, and + // a failure is deliberately not remembered so the next job retries. + let mut host = crate::jail::HostPreparation::new(); let mut sm = ChannelStateMachine::new(config.poll_timeout_secs, runner_metadata); let mut effects: VecDeque = ChannelStateMachine::initial_effects().into_iter().collect(); @@ -186,7 +200,7 @@ fn run_driver(config: &UpConfig, channel_url: &Url, key: &str) -> Result<(), UpE continue; } - match execute_effect(effect, config, channel_url, key, &mut ws) { + match execute_effect(effect, config, channel_url, key, &mut ws, &mut host) { EffectResult::Continue => {}, EffectResult::Input(input) => { effects.clear(); @@ -225,6 +239,7 @@ fn execute_effect( channel_url: &Url, key: &str, ws: &mut Option>>, + host: &mut crate::jail::HostPreparation, ) -> EffectResult { match effect { Effect::Connect => match JobChannel::connect(channel_url, key) { @@ -253,7 +268,7 @@ fn execute_effect( eprintln!("Error: WS not connected during job execution"); return EffectResult::Input(Input::ConnectionFailed); }; - let result = execute_job(config, &job, ws_ref); + let result = execute_job(config, &job, ws_ref, host); EffectResult::Input(Input::JobFinished(result)) }, Effect::SleepBeforeReconnect(reason) => { diff --git a/plus/bencher_runner/src/vm.rs b/plus/bencher_runner/src/vm.rs index b1ef180fd..791d099ec 100644 --- a/plus/bencher_runner/src/vm.rs +++ b/plus/bencher_runner/src/vm.rs @@ -8,11 +8,16 @@ use std::sync::atomic::AtomicBool; use camino::{Utf8Path, Utf8PathBuf}; use crate::error::RunnerError; +use crate::jail::{ + HostPreparation, JailDir, JailLock, JailPaths, JailSignals, StateDir, VmId, chroot, netns, + state, +}; use crate::run::{RunOutput, prepare_oci_workspace}; -/// Execute a single benchmark run in a Firecracker microVM. +/// Execute a single benchmark run in a jailed Firecracker microVM. pub fn vm_execute( config: &crate::Config, + host: &mut HostPreparation, cancel_flag: Option<&Arc>, ) -> Result { use crate::firecracker::run_firecracker; @@ -27,25 +32,25 @@ pub fn vm_execute( println!(" Memory: {} MiB", config.memory.to_mib()); println!(" Timeout: {} seconds", config.timeout_secs); + let state_dir = StateDir::new(config.state_dir.clone())?; + + // Prepare the host on demand, before the first jail this process builds. + // Must come before the job lock is taken: preparation takes the same lock, + // and `flock` is per open file description, so nesting would block on + // itself. It is also the cheap check, so a host that cannot jail at all + // fails here rather than after pulling an image. + host.ensure(state_dir.path(), config.jail_user)?; + + // Everything that does not touch the jail happens before the lock. The + // image pull and unpack are the slow part of a job and need nothing from + // the chroot, so holding an exclusive per-state-directory lock across them + // would serialize concurrent `runner run` invocations on the download + // rather than on the jail. let workspace = prepare_oci_workspace(config)?; let work_dir = &workspace.work_dir; let unpack_dir = &workspace.unpack_dir; let oci_config = workspace.oci_config; - let rootfs_path = work_dir.join("rootfs.ext4"); - - // Get kernel path - use bundled, provided, or find system kernel - let kernel_path = if let Some(kernel) = &config.kernel { - kernel.clone() - } else if crate::kernel::KERNEL_BUNDLED { - let kernel_dest = work_dir.join("vmlinux"); - crate::kernel::write_kernel_to_file(&kernel_dest)?; - println!(" Extracted bundled kernel to {kernel_dest}"); - kernel_dest - } else { - find_kernel()? - }; - let command = oci_config.command; let working_dir = &oci_config.working_dir; let env = oci_config.env; @@ -65,38 +70,103 @@ pub fn vm_execute( println!("Installing init binary..."); install_init_binary(unpack_dir)?; - // Step 6: Create ext4 rootfs + // Held from here to the end of the job. Another runner's sweep removes + // every chroot it finds, so it must not run while this one is live. + // Declared before the jail guard so the lock outlives the teardown it + // protects. + let _lock = JailLock::acquire(state_dir.path())?; + + // Rebuilt per job rather than once per daemon lifetime: the handle lives + // on a tmpfs and is operator visible, so it has to be self-healing. + let netns = netns::ensure()?; + + // The jail root is a function of the VM id, and the job's artifacts are + // built inside it rather than copied in afterwards, so the id is minted + // before any of them exist. Dropping this guard removes the chroot tree, + // which is what the workspace temp directory used to cover. + let vm_id = VmId::new(); + // Minted per job, beside the id: the cgroup and the chroot of one job share + // these, and a stale jail some other sweep could not reclaim is not this + // job's business. + let signals = JailSignals::for_job(host.reclaim_signal()); + let jail_dir = JailDir::create(&state_dir, &vm_id, signals.clone())?; + let jail = JailPaths::new(jail_dir.root())?; + println!(" Jail: {}", jail.root()); + + // Everything Firecracker reads has to be inside the chroot, so the kernel + // lands in the jail root whatever its source: bundled, supplied by the + // job, or found on the host. + let kernel_dest = jail.kernel().host().as_path(); + if let Some(kernel) = &config.kernel { + println!(" Copying the job's kernel into the jail..."); + copy_file(kernel, kernel_dest)?; + } else if crate::kernel::KERNEL_BUNDLED { + crate::kernel::write_kernel_to_file(kernel_dest)?; + println!(" Extracted bundled kernel into the jail at {kernel_dest}"); + } else { + println!(" Copying the host's kernel into the jail..."); + copy_file(&find_kernel()?, kernel_dest)?; + } + + // Step 6: Create the ext4 rootfs directly in the jail root + let rootfs_dest = jail.rootfs().host().as_path(); println!( - "Creating ext4 at {rootfs_path} ({} MiB)...", + "Creating ext4 at {rootfs_dest} ({} MiB)...", config.disk.to_mib() ); - bencher_rootfs::create_ext4_with_size(unpack_dir, &rootfs_path, config.disk.to_mib())?; + bencher_rootfs::create_ext4_with_size(unpack_dir, rootfs_dest, config.disk.to_mib())?; - // Step 7–8: Build Firecracker config and run the microVM - let fc_config = build_firecracker_config(config, work_dir, kernel_path, rootfs_path)?; + // The jailer chowns the chroot root and the device nodes it creates, but + // not what the runner placed inside, so each artifact is handed over + // explicitly. The rootfs is written by Firecracker and is given away; the + // kernel is only read, so it stays owned by root and merely becomes + // readable. + chroot::chown_to_jail(rootfs_dest, config.jail_user)?; + chroot::grant_jail_read(kernel_dest)?; + + // Step 7-8: Build Firecracker config and run the microVM + let fc_config = + build_firecracker_config(config, work_dir, vm_id, &state_dir, jail, netns, signals)?; let run_output = run_firecracker(&fc_config, cancel_flag)?; Ok(run_output) } -/// Build the Firecracker job config: resolve the binary and convert types. +/// Build the Firecracker job config: stage the binaries and convert types. fn build_firecracker_config( config: &crate::Config, work_dir: &Utf8Path, - kernel_path: Utf8PathBuf, - rootfs_path: Utf8PathBuf, + vm_id: VmId, + state_dir: &StateDir, + jail: JailPaths, + netns: Utf8PathBuf, + signals: JailSignals, ) -> Result { - let firecracker_bin = if crate::firecracker_bin::FIRECRACKER_BUNDLED { - let fc_dest = work_dir.join("firecracker"); - crate::firecracker_bin::write_firecracker_to_file(&fc_dest)?; - println!(" Extracted bundled firecracker to {fc_dest}"); - fc_dest + // The jailer copies `--exec-file` into the chroot itself and rejects a + // multiply linked file, so Firecracker is staged outside the jail and is + // never placed in the chroot by hand or hardlinked. Its base name is what + // the jailer derives the chroot layout from, so it is fixed. + let firecracker_bin = work_dir.join(state::EXEC_FILE_NAME); + if crate::firecracker_bin::FIRECRACKER_BUNDLED { + crate::firecracker_bin::write_firecracker_to_file(&firecracker_bin)?; + println!(" Extracted bundled firecracker to {firecracker_bin}"); + } else { + copy_binary(&find_firecracker_binary()?, &firecracker_bin)?; + } + + // The jailer runs outside the chroot and is never copied into it, so it + // can be used wherever it is found. + let jailer_bin = if crate::jailer_bin::JAILER_BUNDLED { + let jailer_dest = work_dir.join("jailer"); + crate::jailer_bin::write_jailer_to_file(&jailer_dest)?; + println!(" Extracted bundled jailer to {jailer_dest}"); + jailer_dest } else { - find_firecracker_binary()? + find_jailer_binary()? }; - println!("Launching Firecracker microVM..."); + println!("Launching jailed Firecracker microVM..."); let vcpus = u8::try_from(u32::from(config.vcpus)).map_err(|_err| { crate::error::ConfigError::OutOfRange { name: "vCPU count", @@ -112,13 +182,17 @@ fn build_firecracker_config( Ok(crate::firecracker::FirecrackerJobConfig { firecracker_bin, - kernel_path, - rootfs_path, + jailer_bin, + vm_id, + jail, + jail_user: config.jail_user, + chroot_base_dir: state_dir.chroot_base(), + netns, + signals, vcpus, memory_mib, boot_args: config.kernel_cmdline.clone(), timeout_secs: config.timeout_secs, - work_dir: work_dir.to_owned(), cpu_layout: config.cpu_layout.clone(), log_level: config.sandbox_log_level, max_file_count: config.max_file_count, @@ -128,6 +202,32 @@ fn build_firecracker_config( }) } +/// Copy a file the job needs to a path the runner controls. +/// +/// Used both for artifacts placed inside the chroot and for binaries staged +/// outside it, so the message says where the copy landed rather than claiming +/// a destination it does not know about. +fn copy_file(src: &Utf8Path, dest: &Utf8Path) -> Result<(), RunnerError> { + std::fs::copy(src, dest).map_err(|e| crate::error::ConfigError::CopyFile { + src: src.to_owned(), + dest: dest.to_owned(), + source: e, + })?; + println!(" Copied {src} to {dest}"); + Ok(()) +} + +/// Stage an executable found on the host, preserving its executable bit. +fn copy_binary(src: &Utf8Path, dest: &Utf8Path) -> Result<(), RunnerError> { + use std::os::unix::fs::PermissionsExt as _; + + copy_file(src, dest)?; + let mut perms = std::fs::metadata(dest)?.permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(dest, perms)?; + Ok(()) +} + /// Write the init config for the VM. /// /// This creates `/etc/bencher/config.json` which is read by `bencher-init`. @@ -194,58 +294,61 @@ fn install_init_binary(rootfs: &Utf8Path) -> Result<(), RunnerError> { Ok(()) } -/// Find the bencher-init binary on disk (fallback when not bundled). -fn find_init_binary() -> Result { - // Look in these locations in order - let candidates = [ - // Next to the current executable +/// Where a bundled binary is looked for when it was not bundled in. +/// +/// Beside the runner first, so a self-contained install finds its own copy +/// before anything the host happens to have. +fn binary_candidates(name: &str) -> impl Iterator { + [ std::env::current_exe() .ok() - .and_then(|p| p.parent().map(|d| d.join("bencher-init"))) - .and_then(|p| Utf8PathBuf::try_from(p).ok()), - // Common installation paths - Some(Utf8PathBuf::from("/usr/local/bin/bencher-init")), - Some(Utf8PathBuf::from("/usr/bin/bencher-init")), - ]; + .and_then(|exe| exe.parent().map(|dir| dir.join(name))) + .and_then(|path| Utf8PathBuf::try_from(path).ok()), + Some(Utf8PathBuf::from(format!("/usr/local/bin/{name}"))), + Some(Utf8PathBuf::from(format!("/usr/bin/{name}"))), + ] + .into_iter() + .flatten() +} - for candidate in candidates.into_iter().flatten() { - if candidate.exists() { - return Ok(candidate); - } - } +/// Find a binary on disk, for when it was not bundled into the runner. +/// +/// One function for all three, because three copies of the same search differing +/// only in a name and a hint is three places for the search to drift. +/// +/// A candidate that cannot be stat'ed is passed over rather than reported, and +/// that is the whole of the failure handling this needs: the search is a list of +/// guesses, and the one thing it can conclude, that nothing was found, is +/// reported by name with what to do about it. +fn find_binary(name: &str, hint: &str) -> Result { + binary_candidates(name) + .find(|candidate| candidate.exists()) + .ok_or_else(|| { + crate::error::ConfigError::BinaryNotFound { + name: name.to_owned(), + hint: hint.to_owned(), + } + .into() + }) +} - Err(crate::error::ConfigError::BinaryNotFound { - name: "bencher-init".to_owned(), - hint: "Build with: cargo build -p bencher_init".to_owned(), - } - .into()) +/// Where to get Firecracker and its jailer, which ship together. +const FIRECRACKER_RELEASES: &str = + "Install from: https://github.com/firecracker-microvm/firecracker/releases"; + +/// Find the bencher-init binary on disk (fallback when not bundled). +fn find_init_binary() -> Result { + find_binary("bencher-init", "Build with: cargo build -p bencher_init") } /// Find the Firecracker binary on the system. fn find_firecracker_binary() -> Result { - let candidates = [ - // Next to the current executable - std::env::current_exe() - .ok() - .and_then(|p| p.parent().map(|d| d.join("firecracker"))) - .and_then(|p| Utf8PathBuf::try_from(p).ok()), - // Common installation paths - Some(Utf8PathBuf::from("/usr/local/bin/firecracker")), - Some(Utf8PathBuf::from("/usr/bin/firecracker")), - ]; - - for candidate in candidates.into_iter().flatten() { - if candidate.exists() { - return Ok(candidate); - } - } + find_binary("firecracker", FIRECRACKER_RELEASES) +} - Err(crate::error::ConfigError::BinaryNotFound { - name: "firecracker".to_owned(), - hint: "Install from: https://github.com/firecracker-microvm/firecracker/releases" - .to_owned(), - } - .into()) +/// Find the jailer binary on the system (fallback when not bundled). +fn find_jailer_binary() -> Result { + find_binary("jailer", FIRECRACKER_RELEASES) } /// Find the kernel image on the system. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/de/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/de/intro.mdx index 68736ef2d..2d212555d 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/de/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/de/intro.mdx @@ -14,7 +14,10 @@ das Verknüpfen beider und das Starten des `runner`-Binary. Einen konzeptionellen Überblick darüber, wie Runner, Specs, Sandboxes und Jobs zusammenpassen, finden Sie in der [Bare Metal-Übersicht][bare metal]. -> 🐰 Firecracker-Sandboxing erfordert Linux mit aktiviertem [KVM][kvm]. +> 🐰 Firecracker-Sandboxing erfordert Linux mit aktiviertem [KVM][kvm], +> und der Runner muss als `root` laufen. +> Jeder Sandbox-Job wird in einem Chroot als unprivilegierter Benutzer eingesperrt, +> und der Aufbau dieser Isolierung benötigt Rechte, die das Aktivieren von KVM allein nicht gewährt. > Ein Runner, der nur Specs ohne Sandbox bedient, kann auf jedem unterstützten Host laufen. [bare metal]: /de/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/de/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/de/run.mdx index fcdd95ad5..a5147f4c4 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/de/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/de/run.mdx @@ -29,6 +29,10 @@ Standardmäßig folgt der Runner dem `stable` [Update-Kanal][update channel] und aktualisiert nur auf versionierte Releases. Der `canary`-Kanal folgt stattdessen dem rollierenden Canary-Build von Bencher Cloud. +Ein Runner, der Sandbox-Jobs ausführt, muss als `root` laufen, +die obigen Befehle werden also so gezeigt, wie der Benutzer `root` sie ausführen würde, +oder unter einem Dienstmanager, der den Runner als `root` startet. + Standardmäßig lehnt ein Runner jeden Job ab, dessen Spec keine [Sandbox][sandbox] hat. Um einem Runner zu erlauben, Jobs ohne Sandbox direkt auf dem Host auszuführen, starten Sie ihn mit [dem `--danger-allow-no-sandbox`-Flag][danger allow no sandbox]. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/en/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/en/intro.mdx index 93fc96b08..cab2f6556 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/en/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/en/intro.mdx @@ -14,7 +14,10 @@ linking the two together, and starting the `runner` binary. For a conceptual overview of how Runners, Specs, Sandboxes, and Jobs fit together, see the [Bare Metal Overview][bare metal]. -> 🐰 Firecracker sandboxing requires Linux with [KVM][kvm] enabled. +> 🐰 Firecracker sandboxing requires Linux with [KVM][kvm] enabled, +> and the Runner must run as `root`. +> Each sandboxed Job is confined to a chroot as an unprivileged user, +> and building that confinement needs privileges that enabling KVM alone does not grant. > A Runner that only serves non-sandboxed Specs can run on any supported host. [bare metal]: /docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/en/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/en/run.mdx index 862400697..e3a87163f 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/en/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/en/run.mdx @@ -29,6 +29,10 @@ By default the Runner follows the `stable` [update channel][update channel], updating only to versioned releases. The `canary` channel instead tracks the rolling canary build deployed to Bencher Cloud. +A Runner that executes sandboxed Jobs must run as `root`, +so the commands above are shown as they would be run by the `root` user +or under a service manager that starts the Runner as `root`. + By default a Runner rejects any Job whose Spec has no [Sandbox][sandbox]. To let a Runner execute non-sandboxed Jobs directly on the host, start it with [the `--danger-allow-no-sandbox` flag][danger allow no sandbox]. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/es/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/es/intro.mdx index 68faedfc5..184a78e3c 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/es/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/es/intro.mdx @@ -14,7 +14,10 @@ la vinculación de ambos, y el inicio del binario `runner`. Para una visión general conceptual de cómo encajan Runners, Specs, Sandboxes y Jobs, consulta la [Visión General de Bare Metal][bare metal]. -> 🐰 El sandboxing con Firecracker requiere Linux con [KVM][kvm] habilitado. +> 🐰 El sandboxing con Firecracker requiere Linux con [KVM][kvm] habilitado, +> y el Runner debe ejecutarse como `root`. +> Cada Job con sandbox queda confinado en un chroot como usuario sin privilegios, +> y construir ese confinamiento necesita permisos que habilitar KVM por sí solo no concede. > Un Runner que solo sirve Specs sin sandbox puede ejecutarse en cualquier host compatible. [bare metal]: /es/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/es/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/es/run.mdx index 02ac1e068..dede154de 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/es/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/es/run.mdx @@ -29,6 +29,10 @@ Por defecto, el Runner sigue el [canal de actualización][update channel] `stabl y solo se actualiza a releases versionadas. El canal `canary` sigue en cambio la compilación canary continua desplegada en Bencher Cloud. +Un Runner que ejecuta Jobs con sandbox debe correr como `root`, +así que los comandos anteriores se muestran tal como los ejecutaría el usuario `root`, +o bajo un gestor de servicios que inicie el Runner como `root`. + Por defecto, un Runner rechaza cualquier Job cuyo Spec no tenga [Sandbox][sandbox]. Para permitir que un Runner ejecute Jobs sin sandbox directamente en el host, inícialo con [el flag `--danger-allow-no-sandbox`][danger allow no sandbox]. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/intro.mdx index 6f8812d28..5b61e4edf 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/intro.mdx @@ -14,7 +14,10 @@ la liaison des deux ensemble, et le démarrage du binaire `runner`. Pour un aperçu conceptuel de la façon dont les Runners, Specs, Sandboxes et Jobs s'articulent, consultez l'[Aperçu Bare Metal][bare metal]. -> 🐰 Le sandboxing Firecracker nécessite Linux avec [KVM][kvm] activé. +> 🐰 Le sandboxing Firecracker nécessite Linux avec [KVM][kvm] activé, +> et le Runner doit s'exécuter en tant que `root`. +> Chaque Job en bac à sable est confiné dans un chroot en tant qu'utilisateur non privilégié, +> et construire ce confinement demande des droits que l'activation de KVM seule n'accorde pas. > Un Runner qui ne sert que des Specs sans sandbox peut s'exécuter sur n'importe quel hôte pris en charge. [bare metal]: /fr/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/run.mdx index a73295cba..66cd3c677 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/fr/run.mdx @@ -29,6 +29,10 @@ Par défaut, le Runner suit le [canal de mise à jour][update channel] `stable` et ne se met à jour que vers les versions publiées. Le canal `canary` suit quant à lui le build canary continu déployé sur Bencher Cloud. +Un Runner qui exécute des Jobs en bac à sable doit s'exécuter en tant que `root`, +les commandes ci-dessus sont donc présentées telles que l'utilisateur `root` les lancerait, +ou sous un gestionnaire de services qui démarre le Runner en tant que `root`. + Par défaut, un Runner rejette tout Job dont le Spec n'a pas de [Sandbox][sandbox]. Pour permettre à un Runner d'exécuter directement des Jobs sans sandbox sur l'hôte, démarrez-le avec [le drapeau `--danger-allow-no-sandbox`][danger allow no sandbox]. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/intro.mdx index 282b8e682..4b47a69c0 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/intro.mdx @@ -14,7 +14,10 @@ Runner のセルフホストは、次のような場合に適しています。 Runner、Spec、Sandbox、Job がどのように組み合わさるかの概念的な概要については、 [Bare Metal の概要][bare metal] を参照してください。 -> 🐰 Firecracker のサンドボックス化には、[KVM][kvm] が有効になった Linux が必要です。 +> 🐰 Firecracker のサンドボックス化には、[KVM][kvm] が有効になった Linux が必要で、 +> Runner は `root` として実行する必要があります。 +> サンドボックス化された各 Job は非特権ユーザーとして chroot に閉じ込められ、 +> その閉じ込めの構築には、KVM を有効にするだけでは得られない権限が必要です。 > サンドボックスなしの Spec のみを提供する Runner は、サポートされている任意のホスト上で実行できます。 [bare metal]: /ja/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/run.mdx index 9b949a4e1..cab805713 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/ja/run.mdx @@ -29,6 +29,10 @@ runner up バージョン付きリリースへのみ更新します。 `canary` チャネルは、Bencher Cloud にデプロイされるローリング canary ビルドを追跡します。 +サンドボックス化された Job を実行する Runner は `root` として動かす必要があるため、 +上記のコマンドは `root` ユーザーが実行する場合、 +あるいは Runner を `root` で起動するサービスマネージャー配下での実行を想定しています。 + デフォルトでは、Runner は [Sandbox][sandbox] のない Spec の Job をすべて拒否します。 Runner にサンドボックスなしの Job をホスト上で直接実行させるには、 [`--danger-allow-no-sandbox` フラグ][danger allow no sandbox] を付けて起動します。 diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/intro.mdx index 61fb36c1d..90875fd97 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/intro.mdx @@ -14,7 +14,10 @@ Runner를 셀프 호스팅하는 것은 다음과 같은 경우에 적합합니 Runner, Spec, Sandbox, Job이 어떻게 맞물리는지에 대한 개념적 개요는 [Bare Metal 개요][bare metal]를 참고하세요. -> 🐰 Firecracker 샌드박싱에는 [KVM][kvm]이 활성화된 Linux가 필요합니다. +> 🐰 Firecracker 샌드박싱에는 [KVM][kvm]이 활성화된 Linux가 필요하며, +> Runner는 `root`로 실행해야 합니다. +> 샌드박스 Job은 각각 비특권 사용자로서 chroot에 가두어지며, +> 그 격리를 구성하려면 KVM 활성화만으로는 얻을 수 없는 권한이 필요합니다. > 샌드박스가 없는 Spec만 제공하는 Runner는 지원되는 모든 호스트에서 실행할 수 있습니다. [bare metal]: /ko/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/run.mdx index a43310e76..1ad2e5059 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/ko/run.mdx @@ -29,6 +29,10 @@ runner up 버전이 지정된 릴리스로만 업데이트합니다. `canary` 채널은 Bencher Cloud에 배포되는 롤링 canary 빌드를 추적합니다. +샌드박스 Job을 실행하는 Runner는 `root`로 실행해야 하므로, +위 명령은 `root` 사용자가 실행하거나 +Runner를 `root`로 시작하는 서비스 관리자 아래에서 실행하는 것을 전제로 합니다. + 기본적으로 Runner는 [Sandbox][sandbox]가 없는 Spec의 Job을 거부합니다. Runner가 샌드박스 없는 Job을 호스트에서 직접 실행하도록 하려면, [`--danger-allow-no-sandbox` 플래그][danger allow no sandbox]로 시작하세요. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/intro.mdx index 72f2be6c8..e15970b74 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/intro.mdx @@ -14,7 +14,10 @@ vincular os dois e iniciar o binário `runner`. Para uma visão conceitual de como Runners, Specs, Sandboxes e Jobs se encaixam, consulte a [Visão Geral de Bare Metal][bare metal]. -> 🐰 O sandboxing com Firecracker requer Linux com [KVM][kvm] habilitado. +> 🐰 O sandboxing com Firecracker requer Linux com [KVM][kvm] habilitado, +> e o Runner precisa ser executado como `root`. +> Cada Job com sandbox fica confinado em um chroot como usuário sem privilégios, +> e construir esse confinamento exige permissões que habilitar o KVM sozinho não concede. > Um Runner que atende apenas Specs sem sandbox pode rodar em qualquer host suportado. [bare metal]: /pt/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/run.mdx index 8c32f8924..b977148a6 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/pt/run.mdx @@ -29,6 +29,10 @@ Por padrão, o Runner segue o [canal de atualização][update channel] `stable` e só atualiza para releases versionadas. O canal `canary`, por sua vez, acompanha o build canary contínuo implantado no Bencher Cloud. +Um Runner que executa Jobs com sandbox precisa rodar como `root`, +então os comandos acima são mostrados como o usuário `root` os executaria, +ou sob um gerenciador de serviços que inicie o Runner como `root`. + Por padrão, um Runner rejeita qualquer Job cuja Spec não tenha [Sandbox][sandbox]. Para permitir que um Runner execute Jobs sem sandbox diretamente no host, inicie-o com [a flag `--danger-allow-no-sandbox`][danger allow no sandbox]. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/intro.mdx index c52993fcc..c8ea1a490 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/intro.mdx @@ -14,7 +14,10 @@ Self-Hosted Runner — это [Bare Metal Runner][bare metal], который в Концептуальный обзор того, как Runner, Spec, Sandbox и Job сочетаются друг с другом, смотрите в [обзоре Bare Metal][bare metal]. -> 🐰 Изоляция через Firecracker требует Linux с включённым [KVM][kvm]. +> 🐰 Изоляция через Firecracker требует Linux с включённым [KVM][kvm], +> а сам Runner должен работать от имени `root`. +> Каждый Job в песочнице запирается в chroot от имени непривилегированного пользователя, +> и построение этой изоляции требует прав, которых одно только включение KVM не даёт. > Runner, обслуживающий только не изолированные Spec, может работать на любом поддерживаемом хосте. [bare metal]: /ru/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/run.mdx index 9f5b27809..d083596e5 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/ru/run.mdx @@ -29,6 +29,10 @@ runner up и обновляется только до версионированных релизов. Канал `canary` вместо этого отслеживает скользящую canary-сборку, развёрнутую в Bencher Cloud. +Runner, выполняющий Job в песочнице, должен работать от имени `root`, +поэтому команды выше приведены так, как их запустил бы пользователь `root`, +или под менеджером служб, который запускает Runner от имени `root`. + По умолчанию Runner отклоняет любую Job, чей Spec не имеет [Sandbox][sandbox]. Чтобы разрешить Runner выполнять не изолированные Job напрямую на хосте, запустите его с [флагом `--danger-allow-no-sandbox`][danger allow no sandbox]. diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/intro.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/intro.mdx index 46a7a352b..15b06c418 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/intro.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/intro.mdx @@ -14,7 +14,10 @@ Self-Hosted Runner 是一个由你自己运行和管理的 [Bare Metal Runner][b 关于 Runner、Spec、Sandbox 和 Job 如何协同工作的概念性概述， 请参阅 [Bare Metal 概述][bare metal]。 -> 🐰 Firecracker 沙箱需要启用了 [KVM][kvm] 的 Linux。 +> 🐰 Firecracker 沙箱需要启用了 [KVM][kvm] 的 Linux， +> 并且 Runner 必须以 `root` 身份运行。 +> 每个沙箱 Job 都会以非特权用户的身份被限制在 chroot 中， +> 而构建这种限制所需的权限，仅仅启用 KVM 是无法获得的。 > 仅服务于非沙箱化 Spec 的 Runner 可以在任何受支持的主机上运行。 [bare metal]: /zh/docs/explanation/bare-metal/ diff --git a/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/run.mdx b/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/run.mdx index 067b89b3d..90451c8ff 100644 --- a/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/run.mdx +++ b/services/console/src/chunks/docs-explanation/self-hosted-runners/zh/run.mdx @@ -29,6 +29,10 @@ runner up 仅更新到版本化的发布版本。 `canary` 通道则跟踪部署到 Bencher Cloud 的滚动 canary 构建。 +执行沙箱 Job 的 Runner 必须以 `root` 身份运行， +因此上面的命令是按 `root` 用户执行的方式给出的， +或者在以 `root` 启动 Runner 的服务管理器下运行。 + 默认情况下，Runner 会拒绝任何其 Spec 没有 [Sandbox][sandbox] 的 Job。 要让 Runner 直接在主机上执行非沙箱化的 Job， 请使用 [`--danger-allow-no-sandbox` 标志][danger allow no sandbox] 启动它。 diff --git a/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx b/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx index 3bfc3a0e3..ef92101f6 100644 --- a/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx +++ b/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx @@ -1,4 +1,7 @@ ## `v0.6.11` +- **BREAKING CHANGE** A Bare Metal Runner that executes sandboxed Jobs must now run as `root`, where before it did not. Confining the Firecracker VMM requires privileges that enabling KVM does not grant: `mknod` to create the sandbox's `/dev/kvm`, `chown` to hand the guest images to the sandbox user, `pivot_root`, and `setns` to join an empty network namespace. A world-readable `/dev/kvm` is enough to *use* KVM unprivileged but not to build the sandbox around it. Start `runner up` as `root`, or as a service that starts it as `root`. If you cannot run as `root`, start the Runner with `--danger-allow-no-sandbox` and assign it only Specs with no Sandbox; be aware that this gives up the microVM itself and not just its confinement, so those Jobs execute directly on the host +- Confine the Firecracker VMM on Bare Metal Runners with the [Firecracker jailer](https://github.com/firecracker-microvm/firecracker/blob/main/docs/jailer.md): each sandboxed Job now runs in a chroot as a dedicated unprivileged user, in an empty network namespace, and is placed in its cgroup before it starts, so a VMM escape no longer lands as `root` on the Runner host holding the Runner key and every prior Job's work. The user defaults to `61016` and is settable with `--jail-uid`/`--jail-gid` +- Add `--state-dir` to the Runner (default `/var/lib/bencher-runner`), a persistent directory holding each sandboxed Job's chroot. It is swept of anything a previous Runner left behind, since a Runner killed mid-Job cannot clean up after itself - Fix the percent difference shown in the report table when a metric drops to zero; a value of `0.00` against a positive baseline now shows `-100.00%` instead of `0.00%` in both the Console and the CI comment (Thank you [@OmarTawfik](https://github.com/OmarTawfik)) - **BREAKING CHANGE** Name the GitHub Check after the Project (ie `Bencher Report ()`) so runs for different Projects on the same commit no longer overwrite each other; `--ci-id` takes the Project name's place when set, the in-progress Check falls back to `Bencher Report` until the results are posted when `--project` is not set, and the name is truncated to 255 bytes. Update any required status check named `Bencher Report` in branch protection to the new name, and note that renaming a Project changes the Check name too (Thank you [@OmarTawfik](https://github.com/OmarTawfik)) diff --git a/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx index e8ce92046..bfbccabee 100644 --- a/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/de/runner-run.mdx @@ -32,6 +32,34 @@ Lassen Sie es für die Ausführung auf dem Host ohne Sandbox weg. Das Ausführungs-Timeout in Sekunden. Standardmäßig wird `300` verwendet. +### `--state-dir ` + +Das persistente Zustandsverzeichnis für den Runner. +Es muss ein absoluter Pfad sein. +Das Jail, das die Sandbox einschließt, wird unter diesem Verzeichnis erstellt, +und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor dem ersten Sandbox-Job daraus entfernt. +Jedes Jail enthält ein vollständiges Gast-Root-Dateisystem und eine Kopie der microVM-Binärdatei, +daher braucht dieses Verzeichnis echte Kapazität für jeden gleichzeitig laufenden Job. +Ein eigenes Dateisystem oder ein `tmpfs`-Mount wird unterstützt und empfohlen: +es hält diese Schreiblast von der Systemplatte fern, +und ein neu erstelltes Dateisystem wird akzeptiert, auch wenn es ein `lost+found`-Verzeichnis mitbringt. +Standardmäßig wird `/var/lib/bencher-runner` verwendet. +Kann auch mit der Umgebungsvariable `BENCHER_STATE_DIR` gesetzt werden. + +### `--jail-uid ` + +Die unprivilegierte Benutzer-ID, auf die der Sandbox-Prozess wechselt. +Standardmäßig wird `61016` verwendet. +Ein Prozess auf dem Host, dem diese Benutzer-ID gehört, kann Signale an die Sandbox senden, +setzen Sie sie also auf eine ID, die der Host nichts anderem zuweist. +Kann auch mit der Umgebungsvariable `BENCHER_JAIL_UID` gesetzt werden. + +### `--jail-gid ` + +Die unprivilegierte Gruppen-ID, auf die der Sandbox-Prozess wechselt. +Standardmäßig wird `61016` verwendet. +Kann auch mit der Umgebungsvariable `BENCHER_JAIL_GID` gesetzt werden. + ### `--iter ` Die Anzahl der auszuführenden Benchmark-Iterationen. diff --git a/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx index a27169f34..94113edc3 100644 --- a/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/de/runner-up.mdx @@ -35,6 +35,34 @@ Kann auch mit der Umgebungsvariable `BENCHER_RUNNER_KEY` gesetzt werden. Das Long-Poll-Timeout in Sekunden beim Warten auf einen Job, zwischen `1` und `900`. Standardmäßig wird `55` verwendet. +### `--state-dir ` + +Das persistente Zustandsverzeichnis für den Runner. +Es muss ein absoluter Pfad sein. +Das Jail, das die Sandbox einschließt, wird unter diesem Verzeichnis erstellt, +und Jails, die durch eine unsaubere Beendigung zurückgeblieben sind, werden vor dem ersten Sandbox-Job daraus entfernt. +Jedes Jail enthält ein vollständiges Gast-Root-Dateisystem und eine Kopie der microVM-Binärdatei, +daher braucht dieses Verzeichnis echte Kapazität für jeden gleichzeitig laufenden Job. +Ein eigenes Dateisystem oder ein `tmpfs`-Mount wird unterstützt und empfohlen: +es hält diese Schreiblast von der Systemplatte fern, +und ein neu erstelltes Dateisystem wird akzeptiert, auch wenn es ein `lost+found`-Verzeichnis mitbringt. +Standardmäßig wird `/var/lib/bencher-runner` verwendet. +Kann auch mit der Umgebungsvariable `BENCHER_STATE_DIR` gesetzt werden. + +### `--jail-uid ` + +Die unprivilegierte Benutzer-ID, auf die der Sandbox-Prozess wechselt. +Standardmäßig wird `61016` verwendet. +Ein Prozess auf dem Host, dem diese Benutzer-ID gehört, kann Signale an die Sandbox senden, +setzen Sie sie also auf eine ID, die der Host nichts anderem zuweist. +Kann auch mit der Umgebungsvariable `BENCHER_JAIL_UID` gesetzt werden. + +### `--jail-gid ` + +Die unprivilegierte Gruppen-ID, auf die der Sandbox-Prozess wechselt. +Standardmäßig wird `61016` verwendet. +Kann auch mit der Umgebungsvariable `BENCHER_JAIL_GID` gesetzt werden. + ### `--danger-allow-no-sandbox` Erlaubt das Ausführen von Jobs ohne [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx index 403d8c060..4f186f7a5 100644 --- a/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/en/runner-run.mdx @@ -32,6 +32,34 @@ Omit for non-sandboxed host execution. The execution timeout in seconds. By default, `300` is used. +### `--state-dir ` + +The persistent state directory for the Runner. +It must be an absolute path. +The jail that confines the sandbox is built under this directory, +and jails left behind by an unclean exit are swept from it before the first sandboxed Job runs. +Each jail holds a full guest root filesystem and a copy of the microVM binary, +so this directory needs real capacity for every Job that runs at once. +A dedicated filesystem or a `tmpfs` mount is supported and recommended: +it keeps that write traffic off the system disk, +and a freshly created filesystem is accepted even though it arrives with a `lost+found` directory. +By default, `/var/lib/bencher-runner` is used. +Can also be set with the `BENCHER_STATE_DIR` environment variable. + +### `--jail-uid ` + +The unprivileged user ID the sandbox process drops to. +By default, `61016` is used. +A process on the host that owns this user ID can signal the sandbox, +so set it to an ID the host does not allocate to anything else. +Can also be set with the `BENCHER_JAIL_UID` environment variable. + +### `--jail-gid ` + +The unprivileged group ID the sandbox process drops to. +By default, `61016` is used. +Can also be set with the `BENCHER_JAIL_GID` environment variable. + ### `--iter ` The number of benchmark iterations to execute. diff --git a/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx index 21455118c..603c98339 100644 --- a/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/en/runner-up.mdx @@ -35,6 +35,34 @@ Can also be set with the `BENCHER_RUNNER_KEY` environment variable. The long-poll timeout in seconds while waiting for a Job, between `1` and `900`. By default, `55` is used. +### `--state-dir ` + +The persistent state directory for the Runner. +It must be an absolute path. +The jail that confines the sandbox is built under this directory, +and jails left behind by an unclean exit are swept from it before the first sandboxed Job runs. +Each jail holds a full guest root filesystem and a copy of the microVM binary, +so this directory needs real capacity for every Job that runs at once. +A dedicated filesystem or a `tmpfs` mount is supported and recommended: +it keeps that write traffic off the system disk, +and a freshly created filesystem is accepted even though it arrives with a `lost+found` directory. +By default, `/var/lib/bencher-runner` is used. +Can also be set with the `BENCHER_STATE_DIR` environment variable. + +### `--jail-uid ` + +The unprivileged user ID the sandbox process drops to. +By default, `61016` is used. +A process on the host that owns this user ID can signal the sandbox, +so set it to an ID the host does not allocate to anything else. +Can also be set with the `BENCHER_JAIL_UID` environment variable. + +### `--jail-gid ` + +The unprivileged group ID the sandbox process drops to. +By default, `61016` is used. +Can also be set with the `BENCHER_JAIL_GID` environment variable. + ### `--danger-allow-no-sandbox` Allow executing Jobs without a [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx index 757bf8378..da8701683 100644 --- a/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/es/runner-run.mdx @@ -32,6 +32,34 @@ Omítelo para ejecución en el host sin sandbox. El tiempo de espera de ejecución en segundos. Por defecto, se usa `300`. +### `--state-dir ` + +El directorio de estado persistente del Runner. +Debe ser una ruta absoluta. +La jaula que confina el sandbox se crea dentro de este directorio, +y las jaulas que quedaron tras una salida no limpia se eliminan de él antes del primer Job con sandbox. +Cada jaula contiene un sistema de archivos raíz completo del huésped y una copia del binario de la microVM, +por lo que este directorio necesita capacidad real para cada Job que se ejecute a la vez. +Se admite y se recomienda un sistema de archivos dedicado o un montaje `tmpfs`: +mantiene esa carga de escritura fuera del disco del sistema, +y se acepta un sistema de archivos recién creado aunque llegue con un directorio `lost+found`. +Por defecto, se usa `/var/lib/bencher-runner`. +También se puede establecer con la variable de entorno `BENCHER_STATE_DIR`. + +### `--jail-uid ` + +El ID de usuario sin privilegios al que cambia el proceso del sandbox. +Por defecto, se usa `61016`. +Un proceso del host que sea propietario de ese ID de usuario puede enviar señales al sandbox, +así que asígnele un ID que el host no destine a ninguna otra cosa. +También se puede establecer con la variable de entorno `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +El ID de grupo sin privilegios al que cambia el proceso del sandbox. +Por defecto, se usa `61016`. +También se puede establecer con la variable de entorno `BENCHER_JAIL_GID`. + ### `--iter ` El número de iteraciones de benchmark a ejecutar. diff --git a/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx index 503e46964..3d7d7f465 100644 --- a/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/es/runner-up.mdx @@ -35,6 +35,34 @@ También se puede establecer con la variable de entorno `BENCHER_RUNNER_KEY`. El tiempo de espera del long-poll en segundos mientras se espera un Job, entre `1` y `900`. Por defecto, se usa `55`. +### `--state-dir ` + +El directorio de estado persistente del Runner. +Debe ser una ruta absoluta. +La jaula que confina el sandbox se crea dentro de este directorio, +y las jaulas que quedaron tras una salida no limpia se eliminan de él antes del primer Job con sandbox. +Cada jaula contiene un sistema de archivos raíz completo del huésped y una copia del binario de la microVM, +por lo que este directorio necesita capacidad real para cada Job que se ejecute a la vez. +Se admite y se recomienda un sistema de archivos dedicado o un montaje `tmpfs`: +mantiene esa carga de escritura fuera del disco del sistema, +y se acepta un sistema de archivos recién creado aunque llegue con un directorio `lost+found`. +Por defecto, se usa `/var/lib/bencher-runner`. +También se puede establecer con la variable de entorno `BENCHER_STATE_DIR`. + +### `--jail-uid ` + +El ID de usuario sin privilegios al que cambia el proceso del sandbox. +Por defecto, se usa `61016`. +Un proceso del host que sea propietario de ese ID de usuario puede enviar señales al sandbox, +así que asígnele un ID que el host no destine a ninguna otra cosa. +También se puede establecer con la variable de entorno `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +El ID de grupo sin privilegios al que cambia el proceso del sandbox. +Por defecto, se usa `61016`. +También se puede establecer con la variable de entorno `BENCHER_JAIL_GID`. + ### `--danger-allow-no-sandbox` Permite ejecutar Jobs sin un [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx index d0d4e1c6f..2b694e520 100644 --- a/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/fr/runner-run.mdx @@ -32,6 +32,34 @@ Omettez pour une exécution sur l'hôte sans sandbox. Le délai d'attente d'exécution en secondes. Par défaut, `300` est utilisé. +### `--state-dir ` + +Le répertoire d'état persistant du Runner. +Il doit s'agir d'un chemin absolu. +La prison qui confine le bac à sable est créée dans ce répertoire, +et les prisons laissées par un arrêt brutal en sont supprimées avant le premier Job avec bac à sable. +Chaque prison contient un système de fichiers racine invité complet et une copie du binaire de la microVM, +ce répertoire a donc besoin d'une capacité réelle pour chaque Job exécuté simultanément. +Un système de fichiers dédié ou un montage `tmpfs` est pris en charge et recommandé : +il maintient cette charge d'écriture hors du disque système, +et un système de fichiers fraîchement créé est accepté même s'il arrive avec un répertoire `lost+found`. +Par défaut, `/var/lib/bencher-runner` est utilisé. +Peut également être défini avec la variable d'environnement `BENCHER_STATE_DIR`. + +### `--jail-uid ` + +L'identifiant d'utilisateur non privilégié que le processus du bac à sable adopte. +Par défaut, `61016` est utilisé. +Un processus de l'hôte possédant cet identifiant peut envoyer des signaux au bac à sable, +choisissez donc un identifiant que l'hôte n'attribue à rien d'autre. +Peut également être défini avec la variable d'environnement `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +L'identifiant de groupe non privilégié que le processus du bac à sable adopte. +Par défaut, `61016` est utilisé. +Peut également être défini avec la variable d'environnement `BENCHER_JAIL_GID`. + ### `--iter ` Le nombre d'itérations de benchmark à exécuter. diff --git a/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx index bd5e6b4e3..f1b767018 100644 --- a/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/fr/runner-up.mdx @@ -35,6 +35,34 @@ Peut aussi être défini avec la variable d'environnement `BENCHER_RUNNER_KEY`. Le délai d'attente du long-poll en secondes pendant l'attente d'un Job, entre `1` et `900`. Par défaut, `55` est utilisé. +### `--state-dir ` + +Le répertoire d'état persistant du Runner. +Il doit s'agir d'un chemin absolu. +La prison qui confine le bac à sable est créée dans ce répertoire, +et les prisons laissées par un arrêt brutal en sont supprimées avant le premier Job avec bac à sable. +Chaque prison contient un système de fichiers racine invité complet et une copie du binaire de la microVM, +ce répertoire a donc besoin d'une capacité réelle pour chaque Job exécuté simultanément. +Un système de fichiers dédié ou un montage `tmpfs` est pris en charge et recommandé : +il maintient cette charge d'écriture hors du disque système, +et un système de fichiers fraîchement créé est accepté même s'il arrive avec un répertoire `lost+found`. +Par défaut, `/var/lib/bencher-runner` est utilisé. +Peut également être défini avec la variable d'environnement `BENCHER_STATE_DIR`. + +### `--jail-uid ` + +L'identifiant d'utilisateur non privilégié que le processus du bac à sable adopte. +Par défaut, `61016` est utilisé. +Un processus de l'hôte possédant cet identifiant peut envoyer des signaux au bac à sable, +choisissez donc un identifiant que l'hôte n'attribue à rien d'autre. +Peut également être défini avec la variable d'environnement `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +L'identifiant de groupe non privilégié que le processus du bac à sable adopte. +Par défaut, `61016` est utilisé. +Peut également être défini avec la variable d'environnement `BENCHER_JAIL_GID`. + ### `--danger-allow-no-sandbox` Autorise l'exécution de Jobs sans [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx index dbc8cc51e..13e363360 100644 --- a/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ja/runner-run.mdx @@ -32,6 +32,34 @@ Firecracker microVM には `firecracker` を使用します (Linux のみ)。 実行タイムアウト (秒)。 デフォルトでは `300` が使用されます。 +### `--state-dir ` + +Runner の永続的な状態ディレクトリ。 +絶対パスである必要があります。 +サンドボックスを閉じ込める jail はこのディレクトリの下に作成され、 +異常終了で残った jail は、最初のサンドボックス Job の実行前にここから一掃されます。 +各 jail はゲストのルートファイルシステム全体と microVM バイナリのコピーを保持するため、 +このディレクトリには同時に実行される Job の分だけの実容量が必要です。 +専用のファイルシステムまたは `tmpfs` マウントがサポートされており、推奨されます。 +書き込み負荷をシステムディスクから切り離せます。 +作成したばかりのファイルシステムは `lost+found` ディレクトリを含んでいても受け付けられます。 +デフォルトでは `/var/lib/bencher-runner` が使用されます。 +環境変数 `BENCHER_STATE_DIR` でも設定できます。 + +### `--jail-uid ` + +サンドボックスプロセスが移行する非特権ユーザー ID。 +デフォルトでは `61016` が使用されます。 +このユーザー ID を所有するホスト上のプロセスはサンドボックスにシグナルを送れるため、 +ホストが他の用途に割り当てていない ID を指定してください。 +環境変数 `BENCHER_JAIL_UID` でも設定できます。 + +### `--jail-gid ` + +サンドボックスプロセスが移行する非特権グループ ID。 +デフォルトでは `61016` が使用されます。 +環境変数 `BENCHER_JAIL_GID` でも設定できます。 + ### `--iter ` 実行するベンチマークの反復回数。 diff --git a/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx index 37619064b..66c80d616 100644 --- a/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ja/runner-up.mdx @@ -34,6 +34,34 @@ runner up [OPTIONS] Job を待機する際のロングポーリングのタイムアウト (秒)。`1` から `900` の範囲です。 デフォルトでは `55` が使用されます。 +### `--state-dir ` + +Runner の永続的な状態ディレクトリ。 +絶対パスである必要があります。 +サンドボックスを閉じ込める jail はこのディレクトリの下に作成され、 +異常終了で残った jail は、最初のサンドボックス Job の実行前にここから一掃されます。 +各 jail はゲストのルートファイルシステム全体と microVM バイナリのコピーを保持するため、 +このディレクトリには同時に実行される Job の分だけの実容量が必要です。 +専用のファイルシステムまたは `tmpfs` マウントがサポートされており、推奨されます。 +書き込み負荷をシステムディスクから切り離せます。 +作成したばかりのファイルシステムは `lost+found` ディレクトリを含んでいても受け付けられます。 +デフォルトでは `/var/lib/bencher-runner` が使用されます。 +環境変数 `BENCHER_STATE_DIR` でも設定できます。 + +### `--jail-uid ` + +サンドボックスプロセスが移行する非特権ユーザー ID。 +デフォルトでは `61016` が使用されます。 +このユーザー ID を所有するホスト上のプロセスはサンドボックスにシグナルを送れるため、 +ホストが他の用途に割り当てていない ID を指定してください。 +環境変数 `BENCHER_JAIL_UID` でも設定できます。 + +### `--jail-gid ` + +サンドボックスプロセスが移行する非特権グループ ID。 +デフォルトでは `61016` が使用されます。 +環境変数 `BENCHER_JAIL_GID` でも設定できます。 + ### `--danger-allow-no-sandbox` [Sandbox][sandbox] なしでの Job の実行を許可します。 diff --git a/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx index edd05be9c..dbe0e856d 100644 --- a/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ko/runner-run.mdx @@ -32,6 +32,34 @@ Firecracker microVM(Linux 전용)을 사용하려면 `firecracker`를 사용하 실행 타임아웃(초)입니다. 기본적으로 `300`이 사용됩니다. +### `--state-dir ` + +Runner의 영구 상태 디렉터리입니다. +절대 경로여야 합니다. +샌드박스를 가두는 jail이 이 디렉터리 아래에 생성되며, +비정상 종료로 남겨진 jail은 첫 번째 샌드박스 Job 실행 전에 이곳에서 정리됩니다. +각 jail은 게스트의 전체 루트 파일 시스템과 microVM 바이너리 사본을 보관하므로, +이 디렉터리에는 동시에 실행되는 Job 수만큼의 실제 용량이 필요합니다. +전용 파일 시스템이나 `tmpfs` 마운트가 지원되며 권장됩니다. +쓰기 부하를 시스템 디스크에서 분리할 수 있습니다. +새로 만든 파일 시스템은 `lost+found` 디렉터리가 있어도 허용됩니다. +기본적으로 `/var/lib/bencher-runner`가 사용됩니다. +`BENCHER_STATE_DIR` 환경 변수로도 설정할 수 있습니다. + +### `--jail-uid ` + +샌드박스 프로세스가 전환할 비특권 사용자 ID입니다. +기본적으로 `61016`이 사용됩니다. +이 사용자 ID를 소유한 호스트의 프로세스는 샌드박스에 시그널을 보낼 수 있으므로, +호스트가 다른 용도로 할당하지 않는 ID를 지정하십시오. +`BENCHER_JAIL_UID` 환경 변수로도 설정할 수 있습니다. + +### `--jail-gid ` + +샌드박스 프로세스가 전환할 비특권 그룹 ID입니다. +기본적으로 `61016`이 사용됩니다. +`BENCHER_JAIL_GID` 환경 변수로도 설정할 수 있습니다. + ### `--iter ` 실행할 벤치마크 반복 횟수입니다. diff --git a/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx index 1b6635605..7a347ab40 100644 --- a/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ko/runner-up.mdx @@ -35,6 +35,34 @@ Runner 인증 키(`bencher_runner_...`)입니다. Job을 기다리는 동안의 롱 폴(long-poll) 타임아웃(초)으로, `1`에서 `900` 사이입니다. 기본적으로 `55`가 사용됩니다. +### `--state-dir ` + +Runner의 영구 상태 디렉터리입니다. +절대 경로여야 합니다. +샌드박스를 가두는 jail이 이 디렉터리 아래에 생성되며, +비정상 종료로 남겨진 jail은 첫 번째 샌드박스 Job 실행 전에 이곳에서 정리됩니다. +각 jail은 게스트의 전체 루트 파일 시스템과 microVM 바이너리 사본을 보관하므로, +이 디렉터리에는 동시에 실행되는 Job 수만큼의 실제 용량이 필요합니다. +전용 파일 시스템이나 `tmpfs` 마운트가 지원되며 권장됩니다. +쓰기 부하를 시스템 디스크에서 분리할 수 있습니다. +새로 만든 파일 시스템은 `lost+found` 디렉터리가 있어도 허용됩니다. +기본적으로 `/var/lib/bencher-runner`가 사용됩니다. +`BENCHER_STATE_DIR` 환경 변수로도 설정할 수 있습니다. + +### `--jail-uid ` + +샌드박스 프로세스가 전환할 비특권 사용자 ID입니다. +기본적으로 `61016`이 사용됩니다. +이 사용자 ID를 소유한 호스트의 프로세스는 샌드박스에 시그널을 보낼 수 있으므로, +호스트가 다른 용도로 할당하지 않는 ID를 지정하십시오. +`BENCHER_JAIL_UID` 환경 변수로도 설정할 수 있습니다. + +### `--jail-gid ` + +샌드박스 프로세스가 전환할 비특권 그룹 ID입니다. +기본적으로 `61016`이 사용됩니다. +`BENCHER_JAIL_GID` 환경 변수로도 설정할 수 있습니다. + ### `--danger-allow-no-sandbox` [Sandbox][sandbox] 없이 Job 실행을 허용합니다. diff --git a/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx index adb578127..4da00f0e3 100644 --- a/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/pt/runner-run.mdx @@ -32,6 +32,34 @@ Omita para execução no host sem sandbox. O timeout de execução em segundos. Por padrão, `300` é usado. +### `--state-dir ` + +O diretório de estado persistente do Runner. +Deve ser um caminho absoluto. +A jail que confina o sandbox é criada sob este diretório, +e as jails deixadas por um encerramento não limpo são removidas dele antes do primeiro Job com sandbox. +Cada jail contém um sistema de arquivos raiz completo do convidado e uma cópia do binário da microVM, +portanto este diretório precisa de capacidade real para cada Job executado ao mesmo tempo. +Um sistema de arquivos dedicado ou uma montagem `tmpfs` é suportado e recomendado: +mantém essa carga de escrita fora do disco do sistema, +e um sistema de arquivos recém-criado é aceito mesmo que venha com um diretório `lost+found`. +Por padrão, `/var/lib/bencher-runner` é usado. +Também pode ser definido com a variável de ambiente `BENCHER_STATE_DIR`. + +### `--jail-uid ` + +O ID de usuário sem privilégios para o qual o processo do sandbox muda. +Por padrão, `61016` é usado. +Um processo no host que seja dono desse ID de usuário pode enviar sinais ao sandbox, +portanto defina um ID que o host não destine a mais nada. +Também pode ser definido com a variável de ambiente `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +O ID de grupo sem privilégios para o qual o processo do sandbox muda. +Por padrão, `61016` é usado. +Também pode ser definido com a variável de ambiente `BENCHER_JAIL_GID`. + ### `--iter ` O número de iterações de benchmark a executar. diff --git a/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx index df5d1fe17..076a27b6d 100644 --- a/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/pt/runner-up.mdx @@ -35,6 +35,34 @@ Também pode ser definida com a variável de ambiente `BENCHER_RUNNER_KEY`. O timeout de long-poll em segundos enquanto aguarda um Job, entre `1` e `900`. Por padrão, `55` é usado. +### `--state-dir ` + +O diretório de estado persistente do Runner. +Deve ser um caminho absoluto. +A jail que confina o sandbox é criada sob este diretório, +e as jails deixadas por um encerramento não limpo são removidas dele antes do primeiro Job com sandbox. +Cada jail contém um sistema de arquivos raiz completo do convidado e uma cópia do binário da microVM, +portanto este diretório precisa de capacidade real para cada Job executado ao mesmo tempo. +Um sistema de arquivos dedicado ou uma montagem `tmpfs` é suportado e recomendado: +mantém essa carga de escrita fora do disco do sistema, +e um sistema de arquivos recém-criado é aceito mesmo que venha com um diretório `lost+found`. +Por padrão, `/var/lib/bencher-runner` é usado. +Também pode ser definido com a variável de ambiente `BENCHER_STATE_DIR`. + +### `--jail-uid ` + +O ID de usuário sem privilégios para o qual o processo do sandbox muda. +Por padrão, `61016` é usado. +Um processo no host que seja dono desse ID de usuário pode enviar sinais ao sandbox, +portanto defina um ID que o host não destine a mais nada. +Também pode ser definido com a variável de ambiente `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +O ID de grupo sem privilégios para o qual o processo do sandbox muda. +Por padrão, `61016` é usado. +Também pode ser definido com a variável de ambiente `BENCHER_JAIL_GID`. + ### `--danger-allow-no-sandbox` Permite executar Jobs sem um [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx index 6caceb1ba..3debccf53 100644 --- a/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/ru/runner-run.mdx @@ -32,6 +32,34 @@ JWT-токен для аутентификации в реестре при за Тайм-аут выполнения в секундах. По умолчанию используется `300`. +### `--state-dir ` + +Постоянный каталог состояния Runner. +Путь должен быть абсолютным. +Изолирующая среда (jail), ограничивающая песочницу, создаётся внутри этого каталога, +а среды, оставшиеся после аварийного завершения, удаляются из него перед запуском первого Job с песочницей. +Каждая среда содержит полную корневую файловую систему гостя и копию двоичного файла microVM, +поэтому этому каталогу нужен реальный объём для каждого Job, работающего одновременно. +Выделенная файловая система или монтирование `tmpfs` поддерживается и рекомендуется: +это уводит нагрузку на запись с системного диска, +а только что созданная файловая система принимается, даже если в ней есть каталог `lost+found`. +По умолчанию используется `/var/lib/bencher-runner`. +Также может быть задан переменной окружения `BENCHER_STATE_DIR`. + +### `--jail-uid ` + +Непривилегированный идентификатор пользователя, на который переходит процесс песочницы. +По умолчанию используется `61016`. +Процесс на хосте, которому принадлежит этот идентификатор, может посылать сигналы песочнице, +поэтому выберите идентификатор, который хост ни на что другое не выделяет. +Также может быть задан переменной окружения `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +Непривилегированный идентификатор группы, на который переходит процесс песочницы. +По умолчанию используется `61016`. +Также может быть задан переменной окружения `BENCHER_JAIL_GID`. + ### `--iter ` Количество итераций бенчмарка для выполнения. diff --git a/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx index 90f62beb0..251828ab5 100644 --- a/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/ru/runner-up.mdx @@ -35,6 +35,34 @@ UUID или slug Runner, от имени которого работать. Тайм-аут длинного опроса в секундах при ожидании Job, от `1` до `900`. По умолчанию используется `55`. +### `--state-dir ` + +Постоянный каталог состояния Runner. +Путь должен быть абсолютным. +Изолирующая среда (jail), ограничивающая песочницу, создаётся внутри этого каталога, +а среды, оставшиеся после аварийного завершения, удаляются из него перед запуском первого Job с песочницей. +Каждая среда содержит полную корневую файловую систему гостя и копию двоичного файла microVM, +поэтому этому каталогу нужен реальный объём для каждого Job, работающего одновременно. +Выделенная файловая система или монтирование `tmpfs` поддерживается и рекомендуется: +это уводит нагрузку на запись с системного диска, +а только что созданная файловая система принимается, даже если в ней есть каталог `lost+found`. +По умолчанию используется `/var/lib/bencher-runner`. +Также может быть задан переменной окружения `BENCHER_STATE_DIR`. + +### `--jail-uid ` + +Непривилегированный идентификатор пользователя, на который переходит процесс песочницы. +По умолчанию используется `61016`. +Процесс на хосте, которому принадлежит этот идентификатор, может посылать сигналы песочнице, +поэтому выберите идентификатор, который хост ни на что другое не выделяет. +Также может быть задан переменной окружения `BENCHER_JAIL_UID`. + +### `--jail-gid ` + +Непривилегированный идентификатор группы, на который переходит процесс песочницы. +По умолчанию используется `61016`. +Также может быть задан переменной окружения `BENCHER_JAIL_GID`. + ### `--danger-allow-no-sandbox` Разрешить выполнение Job без [Sandbox][sandbox]. diff --git a/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx b/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx index 445d703f2..9273af172 100644 --- a/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx +++ b/services/console/src/chunks/docs-reference/runner/zh/runner-run.mdx @@ -32,6 +32,34 @@ runner run --image [OPTIONS] 执行超时（秒）。 默认使用 `300`。 +### `--state-dir ` + +Runner 的持久状态目录。 +必须是绝对路径。 +限制沙箱的 jail 在该目录下创建， +非正常退出遗留的 jail 会在第一个沙箱 Job 运行前从中清除。 +每个 jail 都保存着完整的客户机根文件系统和一份 microVM 二进制文件副本， +因此该目录需要为每个同时运行的 Job 准备真实的容量。 +支持并推荐使用专用文件系统或 `tmpfs` 挂载： +这能让写入负载远离系统磁盘， +并且刚创建的文件系统即使带有 `lost+found` 目录也会被接受。 +默认使用 `/var/lib/bencher-runner`。 +也可以通过 `BENCHER_STATE_DIR` 环境变量设置。 + +### `--jail-uid ` + +沙箱进程切换到的非特权用户 ID。 +默认使用 `61016`。 +宿主机上拥有该用户 ID 的进程可以向沙箱发送信号， +因此请设置为宿主机未分配给其他用途的 ID。 +也可以通过 `BENCHER_JAIL_UID` 环境变量设置。 + +### `--jail-gid ` + +沙箱进程切换到的非特权组 ID。 +默认使用 `61016`。 +也可以通过 `BENCHER_JAIL_GID` 环境变量设置。 + ### `--iter ` 要执行的基准测试迭代次数。 diff --git a/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx b/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx index cfb283b8f..08c232a68 100644 --- a/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx +++ b/services/console/src/chunks/docs-reference/runner/zh/runner-up.mdx @@ -34,6 +34,34 @@ runner up [OPTIONS] 等待 Job 时的长轮询超时（秒），介于 `1` 和 `900` 之间。 默认使用 `55`。 +### `--state-dir ` + +Runner 的持久状态目录。 +必须是绝对路径。 +限制沙箱的 jail 在该目录下创建， +非正常退出遗留的 jail 会在第一个沙箱 Job 运行前从中清除。 +每个 jail 都保存着完整的客户机根文件系统和一份 microVM 二进制文件副本， +因此该目录需要为每个同时运行的 Job 准备真实的容量。 +支持并推荐使用专用文件系统或 `tmpfs` 挂载： +这能让写入负载远离系统磁盘， +并且刚创建的文件系统即使带有 `lost+found` 目录也会被接受。 +默认使用 `/var/lib/bencher-runner`。 +也可以通过 `BENCHER_STATE_DIR` 环境变量设置。 + +### `--jail-uid ` + +沙箱进程切换到的非特权用户 ID。 +默认使用 `61016`。 +宿主机上拥有该用户 ID 的进程可以向沙箱发送信号， +因此请设置为宿主机未分配给其他用途的 ID。 +也可以通过 `BENCHER_JAIL_UID` 环境变量设置。 + +### `--jail-gid ` + +沙箱进程切换到的非特权组 ID。 +默认使用 `61016`。 +也可以通过 `BENCHER_JAIL_GID` 环境变量设置。 + ### `--danger-allow-no-sandbox` 允许在没有 [Sandbox][sandbox] 的情况下执行 Job。 diff --git a/services/runner/TEST.md b/services/runner/TEST.md index ec34ec993..b1610f609 100644 --- a/services/runner/TEST.md +++ b/services/runner/TEST.md @@ -1,6 +1,15 @@ # Runner Integration Tests -The runner integration tests (`cargo test-runner scenarios`) require Linux with KVM. They cannot run on macOS directly. A GCP VM is available for running these tests remotely. +The runner integration tests require Linux with KVM **and root**. They cannot run on macOS directly. A GCP VM is available for running these tests remotely. + +`cargo test-runner scenarios` on its own always fails now: the sandbox is built by dropping privilege rather than by starting without it, so the scenarios refuse to run unelevated. Build unprivileged, then run elevated: + +```bash +cargo test-runner scenarios --build-only +sudo BENCHER_RUNNER_BIN=./target/debug/runner ./target/debug/test_runner scenarios +``` + +Keeping the build out of the elevated half is what stops `cargo` leaving root-owned artifacts in the target directory, which is why `BENCHER_RUNNER_BIN` points the elevated binary at the one already built. ## Prerequisites @@ -55,18 +64,28 @@ All `cargo test-runner` commands must be run on the VM (they require Linux + KVM ### Run all scenarios +Two commands, because only the second may be elevated: + ```bash gcloud compute ssh bencher-vmm-test --zone=us-central1-a --project=bencher-411313 \ - --command="export PATH=\$HOME/.cargo/bin:\$PATH && cd bencher && cargo test-runner scenarios 2>&1" + --command="export PATH=\$HOME/.cargo/bin:\$PATH && cd bencher && cargo test-runner scenarios --build-only 2>&1" +gcloud compute ssh bencher-vmm-test --zone=us-central1-a --project=bencher-411313 \ + --command="cd bencher && sudo BENCHER_RUNNER_BIN=./target/debug/runner ./target/debug/test_runner scenarios 2>&1" ``` -This builds `bencher-init` (musl, statically linked) and the runner CLI, then runs all ~58 integration scenarios. Each scenario builds a Docker image, converts it to OCI format, and runs it inside a Firecracker microVM. Expect this to take several minutes. +The first builds `bencher-init` (musl, statically linked) and the runner CLI; the second runs all integration scenarios as root. Each scenario builds a Docker image, converts it to OCI format, and runs it inside a Firecracker microVM. Expect this to take several minutes. + +### The tuning scenario + +`host_tuning` is the only scenario that runs with host tuning enabled, and it runs last. It asserts that each setting the host lets the runner change is changed while the Job runs and restored when it exits. It excludes two knobs deliberately: SMT stays on, because offlining a sibling changes the core count for everything that follows and no harness can put a CPU back, and IRQ steering is skipped, because an unmovable IRQ rejects the restoring write with `EIO` so the harness cannot promise to undo it. Governor, turbo, and deep C-states are not asserted either: the first two have no sysfs to write inside a VM, and the third is a held file descriptor rather than a value anything can read back. ### Run a single scenario ```bash gcloud compute ssh bencher-vmm-test --zone=us-central1-a --project=bencher-411313 \ - --command="export PATH=\$HOME/.cargo/bin:\$PATH && cd bencher && cargo test-runner scenarios --scenario basic_execution 2>&1" + --command="export PATH=\$HOME/.cargo/bin:\$PATH && cd bencher && cargo test-runner scenarios --build-only 2>&1" +gcloud compute ssh bencher-vmm-test --zone=us-central1-a --project=bencher-411313 \ + --command="cd bencher && sudo BENCHER_RUNNER_BIN=./target/debug/runner ./target/debug/test_runner scenarios --scenario basic_execution 2>&1" ``` ### List all scenarios @@ -111,4 +130,7 @@ gcloud compute ssh bencher-vmm-test --zone=us-central1-a --project=bencher-41131 - **Compilation errors about private imports**: The `plus` feature gates most code. Check `pub use` re-exports if a type is accessible within a crate but not from outside its module. - **"KVM is not available"**: The scenarios require `/dev/kvm` on the host. This is why they must run on the GCP VM, not macOS. - **Timeout scenarios take wall-clock time**: Scenarios like `timeout_handling`, `timeout_enforced`, and `minimum_timeout` intentionally wait for the VM to time out (1-10 seconds each). This is expected. -- **Tuning permission errors in stderr**: Messages like `Tuning: ASLR — skipped (write failed: Permission denied)` are expected when not running as root. They do not cause test failures. +- **"The scenarios must run as root"**: `cargo test-runner scenarios` was run directly. Use the two-step invocation at the top of this file; the message itself repeats it. +- **No tuning output at all**: expected for every scenario except `host_tuning`. The rest pass `--no-tuning`, because they run as root and would otherwise apply real host tuning to the machine running them, including offlining SMT siblings mid-suite. +- **`host_tuning` fails with "No tuning knob on this host can be exercised"**: the host offers nothing the runner can change, so the scenario refuses to pass vacuously. The line above it lists what it considered and why each was skipped: absent, already at the target, or not writable. +- **`host_tuning` left the machine tuned**: it should not be possible. The harness snapshots every setting before the runner starts and restores from its own copy afterwards, independently of the runner's guard, and prints `harness restored ...` for anything it had to put back. Anything it reports there is a real failure of `TuningGuard`, not of the scenario. diff --git a/services/runner/src/parser/mod.rs b/services/runner/src/parser/mod.rs index 4641f88c9..e2ed8af09 100644 --- a/services/runner/src/parser/mod.rs +++ b/services/runner/src/parser/mod.rs @@ -62,6 +62,33 @@ pub struct CliRun { #[arg(long, default_value = "300")] pub timeout: u64, + /// Persistent state directory for the runner (absolute path). + #[arg( + long, + env = "BENCHER_STATE_DIR", + default_value = bencher_runner::DEFAULT_STATE_DIR, + value_parser = absolute_state_dir, + )] + pub state_dir: Utf8PathBuf, + + /// Unprivileged uid the jailed sandbox process drops to. + #[arg( + long, + env = "BENCHER_JAIL_UID", + default_value_t = bencher_runner::DEFAULT_JAIL_UID, + value_parser = clap::value_parser!(u32).range(1..), + )] + pub jail_uid: u32, + + /// Unprivileged gid the jailed sandbox process drops to. + #[arg( + long, + env = "BENCHER_JAIL_GID", + default_value_t = bencher_runner::DEFAULT_JAIL_GID, + value_parser = clap::value_parser!(u32).range(1..), + )] + pub jail_gid: u32, + /// Output file paths inside guest (may be repeated). #[arg(long)] pub output: Vec, @@ -120,3 +147,34 @@ pub struct CliRun { #[arg(long, default_value = "warning", requires = "sandbox")] pub sandbox_log_level: bencher_runner::SandboxLogLevel, } + +/// Require an absolute state directory, before the runner starts. +/// +/// The rule itself lives in the library, on the type that holds the path, since +/// every caller that hands it a state directory is exposed to the same thing. +/// This is the same check run early, so an operator hears about it at the +/// command line rather than when the first sandboxed Job builds its jail. +#[cfg(feature = "plus")] +fn absolute_state_dir(arg: &str) -> Result { + let path = Utf8PathBuf::from(arg); + bencher_runner::check_absolute_state_dir(&path).map_err(|e| e.to_string())?; + Ok(path) +} + +#[cfg(all(test, feature = "plus"))] +mod tests { + use super::absolute_state_dir; + + #[test] + fn a_relative_state_dir_is_refused() { + // A relative path resolves against the jailer's working directory, not + // the runner's, so it has to be caught before it can be handed over. + assert_eq!( + absolute_state_dir("/var/lib/bencher-runner").unwrap(), + "/var/lib/bencher-runner" + ); + absolute_state_dir("bencher-runner").unwrap_err(); + absolute_state_dir("./bencher-runner").unwrap_err(); + absolute_state_dir("").unwrap_err(); + } +} diff --git a/services/runner/src/parser/up.rs b/services/runner/src/parser/up.rs index 8422adf47..9be3daad6 100644 --- a/services/runner/src/parser/up.rs +++ b/services/runner/src/parser/up.rs @@ -1,4 +1,5 @@ use bencher_json::RunnerResourceId; +use camino::Utf8PathBuf; use clap::Parser; use super::CliTuning; @@ -22,6 +23,33 @@ pub struct CliUp { #[arg(long, default_value = "55", value_parser = clap::value_parser!(u32).range(1..=900))] pub poll_timeout: u32, + /// Persistent state directory for the runner (absolute path). + #[arg( + long, + env = "BENCHER_STATE_DIR", + default_value = bencher_runner::DEFAULT_STATE_DIR, + value_parser = super::absolute_state_dir, + )] + pub state_dir: Utf8PathBuf, + + /// Unprivileged uid the jailed sandbox process drops to. + #[arg( + long, + env = "BENCHER_JAIL_UID", + default_value_t = bencher_runner::DEFAULT_JAIL_UID, + value_parser = clap::value_parser!(u32).range(1..), + )] + pub jail_uid: u32, + + /// Unprivileged gid the jailed sandbox process drops to. + #[arg( + long, + env = "BENCHER_JAIL_GID", + default_value_t = bencher_runner::DEFAULT_JAIL_GID, + value_parser = clap::value_parser!(u32).range(1..), + )] + pub jail_gid: u32, + #[command(flatten)] pub tuning: CliTuning, diff --git a/services/runner/src/runner/run.rs b/services/runner/src/runner/run.rs index 509865c6a..38cc950e7 100644 --- a/services/runner/src/runner/run.rs +++ b/services/runner/src/runner/run.rs @@ -13,6 +13,8 @@ impl TryFrom for Run { fn try_from(task: CliRun) -> Result { let tuning = task.tuning.try_into()?; + let jail_user = bencher_runner::JailUser::new(task.jail_uid, task.jail_gid) + .map_err(bencher_runner::RunnerError::from)?; let vcpus = task.vcpus.map(bencher_runner::Cpu::try_from).transpose()?; let memory = task @@ -54,6 +56,8 @@ impl TryFrom for Run { grace_period: task.grace_period, sandbox_log_level: task.sandbox_log_level, sandbox: task.sandbox, + state_dir: task.state_dir, + jail_user, }, }) } diff --git a/services/runner/src/runner/up.rs b/services/runner/src/runner/up.rs index 0407dfd51..eafdea8fa 100644 --- a/services/runner/src/runner/up.rs +++ b/services/runner/src/runner/up.rs @@ -13,6 +13,8 @@ impl TryFrom for Up { fn try_from(task: CliUp) -> Result { let tuning = task.tuning.try_into()?; + let jail_user = bencher_runner::JailUser::new(task.jail_uid, task.jail_gid) + .map_err(bencher_runner::RunnerError::from)?; Ok(Self { config: UpConfig { @@ -31,6 +33,8 @@ impl TryFrom for Up { no_auto_update: task.no_auto_update, update_channel: task.update_channel, max_download_size: task.max_download_size, + state_dir: task.state_dir, + jail_user, }, }) } diff --git a/tasks/test_api/src/task/plus/runner.rs b/tasks/test_api/src/task/plus/runner.rs index 3961f4907..8f0e357ca 100644 --- a/tasks/test_api/src/task/plus/runner.rs +++ b/tasks/test_api/src/task/plus/runner.rs @@ -3,6 +3,8 @@ use std::process::Command; use assert_cmd::cargo::CommandCargoExt as _; use bencher_json::{JsonProjectKeyCreated, JsonUserKeyCreated, Jwt, Url}; use pretty_assertions::assert_eq; +#[cfg(unix)] +use std::os::unix::process::CommandExt as _; use crate::parser::TaskRunner; use crate::task::test::seed_test::{ @@ -174,11 +176,38 @@ impl RunnerTest { anyhow::ensure!(build_status.success(), "Failed to build bencher CLI"); } - // Start the Firecracker runner daemon only when KVM is available + // Start the Firecracker runner daemon only when KVM is available. + // + // Elevated, because a sandboxed Job is a jailed Job: the jailer creates + // the chroot's device nodes with mknod, chowns the tree to the jail + // user, pivot_roots, and joins a network namespace. Only this one + // process is elevated. Everything else in this test, cargo included, + // stays as the invoking user, and the no-sandbox runner below stays + // unprivileged, which proves the coupling holds in both directions in + // the same run. let runner_child_and_handle = if has_kvm { - println!("Starting runner daemon..."); - let mut runner_child = Command::cargo_bin("runner")?; - let mut runner_child = runner_child + println!("Starting runner daemon (elevated)..."); + ensure_passwordless_sudo()?; + + // Resolve the already-built binary and run it under sudo directly, + // so cargo is never invoked as root and cannot leave root-owned + // artifacts in the target directory. + let runner_cmd = Command::cargo_bin("runner")?; + let runner_bin = runner_cmd.get_program().to_owned(); + + // Never the default state directory. This runner is root, and + // preparing `/var/lib/bencher-runner` chmods it to 0700, sweeps + // every jail in it, and leaves it root-owned on the machine of + // whoever ran this harness, which on a developer's box is a real + // directory a real runner may own. The scenarios harness keeps its + // state under the target directory for the same reason. + let state_dir = elevated_state_dir(&runner_bin); + println!(" Runner state directory: {}", state_dir.display()); + + let mut runner_child = Command::new("sudo"); + let runner_child = runner_child + .args(["-n", "--"]) + .arg(&runner_bin) .args([ "up", HOST_ARG, @@ -188,9 +217,16 @@ impl RunnerTest { "--runner", "test-runner", ]) + .arg("--state-dir") + .arg(&state_dir) .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::inherit()) - .spawn()?; + .stderr(std::process::Stdio::inherit()); + // Its own process group, so teardown can signal the runner even + // though the child handle is sudo, which may exec the runner in + // place or fork it depending on version and configuration. + #[cfg(unix)] + runner_child.process_group(0); + let mut runner_child = runner_child.spawn()?; let reader_handle = wait_for_stdout_ready( &mut runner_child, @@ -198,7 +234,7 @@ impl RunnerTest { "runner", std::time::Duration::from_secs(30), ); - Some((runner_child, reader_handle)) + Some((runner_child, reader_handle, state_dir)) } else { println!("Skipping Firecracker runner daemon (no KVM)"); None @@ -287,10 +323,10 @@ impl RunnerTest { }; // Always kill runner daemons, even if the test failed - if let Some((mut runner_child, reader_handle)) = runner_child_and_handle { - let _kill = runner_child.kill(); - let _wait = runner_child.wait(); + if let Some((mut runner_child, reader_handle, state_dir)) = runner_child_and_handle { + kill_elevated_runner(&mut runner_child); let _join = reader_handle.join(); + remove_elevated_state_dir(&state_dir); } let _kill = no_sandbox_child.kill(); let _wait = no_sandbox_child.wait(); @@ -309,6 +345,82 @@ impl RunnerTest { } } +/// Fail early when the sandboxed runner cannot be elevated. +/// +/// Without this the runner would start, fail to build its first jail, and the +/// readiness wait would time out after thirty seconds with nothing pointing at +/// the cause. +fn ensure_passwordless_sudo() -> anyhow::Result<()> { + let status = Command::new("sudo") + .args(["-n", "true"]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status()?; + anyhow::ensure!( + status.success(), + "A Runner executing sandboxed Jobs must run as root, and passwordless sudo is not available. \ + The jailer creates the chroot's device nodes with mknod, chowns the tree to the jail user, \ + pivot_roots into it, and joins a network namespace, none of which an unprivileged process can do." + ); + Ok(()) +} + +/// The state directory the elevated runner is pointed at. +/// +/// Beside the runner binary, so it lands in the target directory the harness +/// already owns and is thrown away with it. +fn elevated_state_dir(runner_bin: &std::ffi::OsStr) -> std::path::PathBuf { + std::path::Path::new(runner_bin) + .parent() + .unwrap_or(std::path::Path::new(".")) + .join("test-api-runner-state") +} + +/// Remove the state directory the elevated runner left behind. +/// +/// Root-owned, because the runner that created it was, so the unprivileged +/// harness cannot remove it itself. Passwordless sudo was already established +/// before the daemon started. A failure here is not a test failure, but it is +/// said out loud with the command that finishes the job, because what is left is +/// a root-owned directory inside the developer's own tree. +fn remove_elevated_state_dir(state_dir: &std::path::Path) { + if !state_dir.exists() { + return; + } + let removed = Command::new("sudo") + .args(["-n", "rm", "-rf"]) + .arg(state_dir) + .status() + .is_ok_and(|status| status.success()); + if !removed { + println!( + "Note: {} is left owned by root. Remove it with: sudo rm -rf {}", + state_dir.display(), + state_dir.display() + ); + } +} + +/// Stop the elevated runner daemon and anything it spawned. +/// +/// The runner runs as root, so the unprivileged test process cannot signal it, +/// and the handle is sudo rather than the runner itself. Signalling the whole +/// process group covers both cases; the group exists because the spawn put the +/// child in its own. Killing sudo alone would leave a root runner daemon +/// holding the jail lock for the rest of the run. +fn kill_elevated_runner(child: &mut std::process::Child) { + let pid = child.id(); + let _status = Command::new("sudo") + .args(["-n", "sh", "-c"]) + .arg(format!( + "kill -KILL -{pid} 2>/dev/null || kill -KILL {pid} 2>/dev/null || true" + )) + .status(); + // Reap the handle whatever the signal did. + let _kill = child.kill(); + let _wait = child.wait(); +} + /// Check whether Docker is available. pub fn docker_available() -> bool { Command::new("docker") diff --git a/tasks/test_runner/Cargo.toml b/tasks/test_runner/Cargo.toml index 59899b24b..7a0a9d47b 100644 --- a/tasks/test_runner/Cargo.toml +++ b/tasks/test_runner/Cargo.toml @@ -19,5 +19,8 @@ camino.workspace = true clap.workspace = true serde_json.workspace = true +[dev-dependencies] +tempfile.workspace = true + [lints] workspace = true diff --git a/tasks/test_runner/src/parser/mod.rs b/tasks/test_runner/src/parser/mod.rs index 8ad029769..12ef33720 100644 --- a/tasks/test_runner/src/parser/mod.rs +++ b/tasks/test_runner/src/parser/mod.rs @@ -21,4 +21,12 @@ pub struct TaskScenarios { /// List all available scenarios #[clap(long, short)] pub list: bool, + + /// Build the binaries the scenarios need, then exit. + /// + /// The scenarios themselves need root, and building as root would leave + /// cargo's cache and target directory root-owned. This splits the build + /// out so it can run unprivileged before the elevated run. + #[clap(long, conflicts_with_all = ["scenario", "list"])] + pub build_only: bool, } diff --git a/tasks/test_runner/src/task/scenarios.rs b/tasks/test_runner/src/task/scenarios.rs index 3ee044963..74dba09d7 100644 --- a/tasks/test_runner/src/task/scenarios.rs +++ b/tasks/test_runner/src/task/scenarios.rs @@ -37,7 +37,20 @@ fn extract_json_substr(line: &str) -> &str { &line[start..end] } +/// A host-side check run while the runner is still executing. +/// +/// Some confinement invariants only exist while the VMM is alive and cannot +/// be recovered from the runner's output afterwards. The probe receives the +/// runner's state directory and returns `Ok(false)` while the VMM has not +/// appeared yet, `Ok(true)` once the invariant has been observed to hold, and +/// `Err` once it has been observed to be violated. +type Probe = fn(&Utf8Path) -> Result; + /// Test scenario definition. +/// +/// Build one with `..Scenario::default()` so a scenario names only what it +/// actually varies, and so adding a field does not have to be written out +/// across every scenario in this file. struct Scenario { name: &'static str, description: &'static str, @@ -47,9 +60,45 @@ struct Scenario { cancel_after_secs: Option, /// Whether to use `--sandbox firecracker` (default: true). sandboxed: bool, + /// If set, run before the runner starts, to put the host in some state. + setup: Option Result<()>>, + /// If set, a host-side check run while the runner is executing. + probe: Option, + /// Run with host tuning enabled and assert it applies and is restored. + /// + /// Every other scenario passes `--no-tuning`, so this is the only one that + /// can leave the machine changed, and the harness undoes it itself. + tuning: bool, + /// Kill the runner once its VMM is up so nothing unwinds, then run the + /// image again and report the second run. + /// + /// SIGKILL is the point: it is the exit that never unwinds, so `Drop` + /// cannot reclaim the chroot and only the sweep can. + orphan_then_rerun: bool, validate: fn(&ScenarioOutput) -> Result<()>, } +impl Default for Scenario { + fn default() -> Self { + Self { + name: "", + description: "", + dockerfile: "", + extra_args: &[], + cancel_after_secs: None, + setup: None, + probe: None, + tuning: false, + orphan_then_rerun: false, + // Sandboxed is the interesting case and the overwhelming majority, + // so the handful of non-sandboxed scenarios opt out rather than + // every other scenario opting in. + sandboxed: true, + validate: |_output| Ok(()), + } + } +} + /// Output from running a scenario. #[derive(Debug)] struct ScenarioOutput { @@ -62,6 +111,7 @@ struct ScenarioOutput { pub struct Scenarios { scenario: Option, list: bool, + build_only: bool, } impl TryFrom for Scenarios { @@ -71,6 +121,7 @@ impl TryFrom for Scenarios { Ok(Self { scenario: task.scenario, list: task.list, + build_only: task.build_only, }) } } @@ -82,7 +133,29 @@ impl Scenarios { return Ok(()); } - // Check prerequisites + if self.build_only { + let runner_bin = ensure_runner_bin()?; + println!("Built runner: {runner_bin}"); + println!("Run the scenarios with:"); + println!(" sudo {RUNNER_BIN_ENV}={runner_bin} scenarios"); + return Ok(()); + } + + // Check prerequisites. + // + // Root is one of them. The jailer creates the chroot's device nodes + // with mknod, chowns the tree to the jail user, pivot_roots, and joins + // a network namespace, none of which an unprivileged process can do. + // A udev rule that makes /dev/kvm world accessible is enough to *use* + // KVM without root but not to build the jail around it. + if !is_root() { + bail!( + "The scenarios must run as root: the sandbox is built by dropping privilege, not by starting without it.\n\ + Build unprivileged first, then run elevated:\n\ + \x20 cargo test-runner scenarios --build-only\n\ + \x20 sudo {RUNNER_BIN_ENV}=./target/debug/runner ./target/debug/test_runner scenarios" + ); + } if !kvm_available() { bail!("KVM is not available (/dev/kvm not found)"); } @@ -105,27 +178,74 @@ impl Scenarios { let runner_bin = ensure_runner_bin()?; let mut scenarios = all_scenarios(); + scenarios.extend(jail_scenarios()); scenarios.extend(nosandbox_scenarios()); + // Last, always. It is the only scenario that tunes the machine, so + // nothing it leaves behind can reach the others, and if the suite is + // killed part way through it is the least likely to have started. + scenarios.extend(tuning_scenarios()); - if let Some(name) = &self.scenario { + let result = if let Some(name) = &self.scenario { // Run a single scenario - let scenario = scenarios + scenarios .iter() .find(|s| s.name == name) - .with_context(|| format!("Unknown scenario: {name}"))?; - - run_scenario(scenario, &runner_bin) + .with_context(|| format!("Unknown scenario: {name}")) + .and_then(|scenario| run_scenario(scenario, &runner_bin)) } else { // Run all scenarios run_all_scenarios(&scenarios, &runner_bin) - } + }; + + // Whatever the outcome. The state directory is only root-owned because + // the scenarios had to be, and it sits inside the repo tree. + return_state_dir_to_invoker(); + + result + } +} + +/// Hand the state directory back to whoever invoked `sudo`. +/// +/// The scenarios must run as root, and the runner creates its state directory at +/// 0700 owned by root under the repo's target tree. CI throws that tree away, so +/// it costs nothing there, but on a developer's machine the next unprivileged +/// `cargo` invocation trips over a directory it can neither read nor remove. +/// `SUDO_UID` names who to give it back to; with no one to give it back to, or a +/// `chown` that will not run, the path is printed with the command that clears +/// it rather than left to be discovered. +fn return_state_dir_to_invoker() { + let state_dir = scenario_state_dir(); + if !state_dir.exists() { + return; + } + + if let Some((uid, gid)) = invoking_user() + && Command::new("chown") + .args(["-R", &format!("{uid}:{gid}"), state_dir.as_str()]) + .status() + .is_ok_and(|status| status.success()) + { + println!("Returned {state_dir} to uid {uid}"); + return; } + + println!("Note: {state_dir} is left owned by root. Remove it with: sudo rm -rf {state_dir}"); +} + +/// The uid and gid that invoked `sudo`, when one did. +fn invoking_user() -> Option<(u32, u32)> { + let uid = std::env::var("SUDO_UID").ok()?.parse().ok()?; + let gid = std::env::var("SUDO_GID").ok()?.parse().ok()?; + Some((uid, gid)) } /// List all available scenarios. fn list_scenarios() { let mut scenarios = all_scenarios(); + scenarios.extend(jail_scenarios()); scenarios.extend(nosandbox_scenarios()); + scenarios.extend(tuning_scenarios()); println!("Available scenarios:"); println!(); for scenario in &scenarios { @@ -179,16 +299,54 @@ fn run_scenario(scenario: &Scenario, runner_bin: &Utf8Path) -> Result<()> { let image_path = build_test_image(scenario.name, scenario.dockerfile) .with_context(|| format!("Failed to build image for {}", scenario.name))?; - // Prepend --sandbox firecracker for sandboxed scenarios - let mut args: Vec<&str> = Vec::new(); + if let Some(setup) = scenario.setup { + setup().with_context(|| format!("Setup failed for {}", scenario.name))?; + } + + // One state directory for the suite, wiped before each scenario, so jail + // assertions see only this scenario's jails and never touch a real runner's + // state. Reclaimed before the wipe, because the wipe is the one thing the + // runner's own sweep refuses to do. + let state_dir = scenario_state_dir(); + reclaim_stranded_jails(&state_dir) + .with_context(|| format!("Failed to reclaim jails stranded before {}", scenario.name))?; + drop(fs::remove_dir_all(&state_dir)); + + // Prepend --sandbox firecracker for sandboxed scenarios. + // + // `--no-tuning` everywhere except the one scenario that exists to test + // tuning. Elevated, the knobs really apply, and a suite that tuned the + // machine twenty-five times would offline SMT siblings on a two-vCPU hosted + // runner, changing the core count under itself. The tuning scenario turns + // them back on for one job, keeps SMT and IRQ steering out of it, and the + // harness undoes everything itself afterwards. + let mut args: Vec<&str> = vec!["--state-dir", state_dir.as_str()]; + if scenario.tuning { + // The two knobs this scenario deliberately does not exercise. `--smt` + // keeps hyper-threading on: offlining a sibling changes `nproc` for + // everything that follows in the CI job, and the harness cannot put a + // CPU back if the runner is killed before its guard runs. IRQ steering + // is skipped because a hand-restore of it is unavoidably partial, since + // an unmovable IRQ rejects the write with EIO, and the rule for this + // scenario is that the harness can undo anything it turned on. + args.extend(["--smt", "--no-irq-steering"]); + } else { + args.push("--no-tuning"); + } if scenario.sandboxed { args.extend(["--sandbox", "firecracker"]); } args.extend(scenario.extra_args); - // Run the runner (with optional cancellation) + // Run the runner (with optional cancellation or host-side probe) let output = if let Some(secs) = scenario.cancel_after_secs { run_runner_with_cancel(&image_path, &args, Duration::from_secs(secs), runner_bin) + } else if scenario.orphan_then_rerun { + run_runner_after_orphan(&image_path, &args, &state_dir, runner_bin) + } else if let Some(probe) = scenario.probe { + run_runner_with_probe(&image_path, &args, probe, &state_dir, runner_bin) + } else if scenario.tuning { + run_runner_with_tuning(&image_path, &args, runner_bin) } else { run_runner(&image_path, &args, runner_bin) } @@ -218,8 +376,6 @@ fn all_scenarios() -> Vec { description: "Simple echo command", dockerfile: r#"FROM busybox CMD ["echo", "hello from vm"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("hello from vm") { @@ -228,6 +384,7 @@ CMD ["echo", "hello from vm"]"#, bail!("Expected 'hello from vm' in output, got: {}", output.stdout) } }, + ..Scenario::default() }, Scenario { name: "environment_variables", @@ -235,8 +392,6 @@ CMD ["echo", "hello from vm"]"#, dockerfile: r#"FROM busybox ENV MY_VAR=test_value CMD ["sh", "-c", "echo $MY_VAR"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("test_value") { @@ -250,6 +405,7 @@ CMD ["sh", "-c", "echo $MY_VAR"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "working_directory", @@ -257,8 +413,6 @@ CMD ["sh", "-c", "echo $MY_VAR"]"#, dockerfile: r#"FROM busybox WORKDIR /myapp CMD ["pwd"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("/myapp") { @@ -267,14 +421,13 @@ CMD ["pwd"]"#, bail!("Expected '/myapp' in output, got: {}", output.stdout) } }, + ..Scenario::default() }, Scenario { name: "file_output", description: "Output file collection via vsock", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo '{\"result\": 42}' > /tmp/output.json && cat /tmp/output.json"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--output", "/tmp/output.json"], validate: |output| { if output.stdout.contains("\"result\"") || output.stdout.contains("42") { @@ -283,14 +436,13 @@ CMD ["sh", "-c", "echo '{\"result\": 42}' > /tmp/output.json && cat /tmp/output. bail!("Expected JSON output, got: {}", output.stdout) } }, + ..Scenario::default() }, Scenario { name: "exit_code", description: "Non-zero exit codes captured", dockerfile: r#"FROM busybox CMD ["sh", "-c", "exit 42"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -300,14 +452,13 @@ CMD ["sh", "-c", "exit 42"]"#, bail!("Expected exit code 42 in output") } }, + ..Scenario::default() }, Scenario { name: "timeout_handling", description: "VM killed after timeout", dockerfile: r#"FROM busybox CMD ["sleep", "3600"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr).to_lowercase(); @@ -317,14 +468,13 @@ CMD ["sleep", "3600"]"#, bail!("Expected timeout error") } }, + ..Scenario::default() }, Scenario { name: "writable_filesystem", description: "Guest can write to ext4 rootfs", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo test > /data.txt && cat /data.txt"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("test") { @@ -336,14 +486,13 @@ CMD ["sh", "-c", "echo test > /data.txt && cat /data.txt"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "stderr_capture", description: "Stderr captured separately", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo stdout && echo stderr >&2"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -353,14 +502,13 @@ CMD ["sh", "-c", "echo stdout && echo stderr >&2"]"#, bail!("Expected 'stdout' in output") } }, + ..Scenario::default() }, Scenario { name: "multi_cpu", description: "Multiple vCPUs work (expected: timeout, SMP boot unsupported)", dockerfile: r#"FROM busybox CMD ["sh", "-c", "cat /proc/cpuinfo | grep processor | wc -l"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "10", "--vcpus", "4"], validate: |output| { // SMP boot is not yet supported (requires LAPIC/APIC emulation). @@ -379,6 +527,7 @@ CMD ["sh", "-c", "cat /proc/cpuinfo | grep processor | wc -l"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "entrypoint_with_args", @@ -386,8 +535,6 @@ CMD ["sh", "-c", "cat /proc/cpuinfo | grep processor | wc -l"]"#, dockerfile: r#"FROM busybox ENTRYPOINT ["echo"] CMD ["hello", "world"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("hello world") { @@ -396,14 +543,13 @@ CMD ["hello", "world"]"#, bail!("Expected 'hello world' in output, got: {}", output.stdout) } }, + ..Scenario::default() }, Scenario { name: "no_network_access", description: "Guest has no network", dockerfile: r#"FROM busybox CMD ["sh", "-c", "ping -c 1 -W 1 8.8.8.8 2>&1 || echo no_network"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -416,6 +562,7 @@ CMD ["sh", "-c", "ping -c 1 -W 1 8.8.8.8 2>&1 || echo no_network"]"#, bail!("Expected network failure, got: {combined}") } }, + ..Scenario::default() }, // ======================================================================= // Security hardening scenarios @@ -426,8 +573,6 @@ CMD ["sh", "-c", "ping -c 1 -W 1 8.8.8.8 2>&1 || echo no_network"]"#, // Generate ~20MB of output - should be truncated to the 10MB limit dockerfile: r#"FROM busybox CMD ["sh", "-c", "dd if=/dev/zero bs=1M count=20 2>/dev/null | tr '\\0' 'A' && echo DONE"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "120", "--max-output-size", "10485760"], validate: |output| { // The key test: the runner completes without OOM and output is bounded. @@ -441,6 +586,7 @@ CMD ["sh", "-c", "dd if=/dev/zero bs=1M count=20 2>/dev/null | tr '\\0' 'A' && e // Runner completed (didn't hang or OOM) - that's a pass Ok(()) }, + ..Scenario::default() }, Scenario { name: "timeout_enforced", @@ -448,8 +594,6 @@ CMD ["sh", "-c", "dd if=/dev/zero bs=1M count=20 2>/dev/null | tr '\\0' 'A' && e // This process ignores signals and runs forever dockerfile: r#"FROM busybox CMD ["sh", "-c", "trap '' TERM INT; echo started; while true; do sleep 1; done"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { // The VM should be killed after 5 seconds due to timeout @@ -465,6 +609,7 @@ CMD ["sh", "-c", "trap '' TERM INT; echo started; while true; do sleep 1; done"] ) } }, + ..Scenario::default() }, // ======================================================================= // Error regression scenarios @@ -479,8 +624,6 @@ CMD ["sh", "-c", "trap '' TERM INT; echo started; while true; do sleep 1; done"] // causing uid_map writes to fail with EPERM. dockerfile: r#"FROM busybox CMD ["id"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // The runner should not fail with uid_map errors. @@ -496,6 +639,7 @@ CMD ["id"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "dev_kvm_available", @@ -505,8 +649,6 @@ CMD ["id"]"#, // the bind-mounted /dev/kvm. dockerfile: r#"FROM busybox CMD ["echo", "kvm_test_ok"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -518,6 +660,7 @@ CMD ["echo", "kvm_test_ok"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "proc_mount_works", @@ -527,8 +670,6 @@ CMD ["echo", "kvm_test_ok"]"#, // which we fixed by bind-mounting the host's /proc instead. dockerfile: r#"FROM busybox CMD ["cat", "/proc/version"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -542,6 +683,7 @@ CMD ["cat", "/proc/version"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "rootfs_writable", @@ -551,8 +693,6 @@ CMD ["cat", "/proc/version"]"#, // when trying to write to the filesystem. dockerfile: r#"FROM busybox CMD ["sh", "-c", "touch /tmp/write_test && echo write_ok"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("write_ok") { @@ -567,6 +707,7 @@ CMD ["sh", "-c", "touch /tmp/write_test && echo write_ok"]"#, bail!("Expected 'write_ok' in output, got: {combined}") } }, + ..Scenario::default() }, Scenario { name: "timeout_includes_partial_output", @@ -576,8 +717,6 @@ CMD ["sh", "-c", "touch /tmp/write_test && echo write_ok"]"#, // short-circuited before serial output extraction. dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo partial_output_marker && sleep 3600"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "10"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -592,6 +731,7 @@ CMD ["sh", "-c", "echo partial_output_marker && sleep 3600"]"#, bail!("Expected timeout error in output, got: {combined}") } }, + ..Scenario::default() }, Scenario { name: "no_seccomp_sigsys", @@ -602,8 +742,6 @@ CMD ["sh", "-c", "echo partial_output_marker && sleep 3600"]"#, // This scenario exercises the timeout path which requires kill(). dockerfile: r#"FROM busybox CMD ["sleep", "3600"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { // SIGSYS from seccomp violation produces exit code 159 (128 + 31) @@ -621,6 +759,7 @@ CMD ["sleep", "3600"]"#, bail!("Expected timeout exit, got exit_code={}", output.exit_code) } }, + ..Scenario::default() }, Scenario { name: "iopl_dropped_before_exec", @@ -635,8 +774,6 @@ RUN printf '#include \n#include \n#include \nstatic FROM busybox COPY --from=build /test_iopl /test_iopl CMD ["/test_iopl"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("IOPL_DROPPED") { @@ -655,6 +792,7 @@ CMD ["/test_iopl"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "unique_output_validation", @@ -664,8 +802,6 @@ CMD ["/test_iopl"]"#, // never appear in runner logs. dockerfile: r#"FROM busybox CMD ["echo", "UNIQUE_VM_OUTPUT_a7f3b2c9"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // This unique string should only appear if the VM actually ran @@ -681,6 +817,7 @@ CMD ["echo", "UNIQUE_VM_OUTPUT_a7f3b2c9"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // PID namespace isolation scenarios (Item 9) @@ -692,8 +829,6 @@ CMD ["echo", "UNIQUE_VM_OUTPUT_a7f3b2c9"]"#, // The init process should be PID 1, and there should be very few processes. dockerfile: r#"FROM busybox CMD ["sh", "-c", "ls /proc | grep -E '^[0-9]+$' | wc -l"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // The guest should see a small number of PIDs (1-5), not hundreds @@ -715,6 +850,7 @@ CMD ["sh", "-c", "ls /proc | grep -E '^[0-9]+$' | wc -l"]"#, Ok(()) } }, + ..Scenario::default() }, Scenario { name: "pid_namespace_procfs", @@ -724,8 +860,6 @@ CMD ["sh", "-c", "ls /proc | grep -E '^[0-9]+$' | wc -l"]"#, // should be accessible and /proc/1/cmdline should show the init process. dockerfile: r#"FROM busybox CMD ["sh", "-c", "cat /proc/version && echo PID1=$(cat /proc/1/cmdline | tr '\\0' ' ')"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -741,6 +875,7 @@ CMD ["sh", "-c", "cat /proc/version && echo PID1=$(cat /proc/1/cmdline | tr '\\0 ) } }, + ..Scenario::default() }, // ======================================================================= // Telemetry/Metrics scenarios (Item 10) @@ -751,8 +886,6 @@ CMD ["sh", "-c", "cat /proc/version && echo PID1=$(cat /proc/1/cmdline | tr '\\0 // Verifies the runner outputs ---BENCHER_METRICS:{json}--- on stderr. dockerfile: r#"FROM busybox CMD ["echo", "metrics_test"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stderr.contains("---BENCHER_METRICS:") && output.stderr.contains("---") { @@ -765,6 +898,7 @@ CMD ["echo", "metrics_test"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "metrics_wall_clock_reasonable", @@ -773,8 +907,6 @@ CMD ["echo", "metrics_test"]"#, // This catches cases where timing is broken (e.g., always 0 or absurdly large). dockerfile: r#"FROM busybox CMD ["echo", "fast_benchmark"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // Parse metrics from stderr @@ -803,6 +935,7 @@ CMD ["echo", "fast_benchmark"]"#, } bail!("Could not parse wall_clock_ms from metrics: {json_str}") }, + ..Scenario::default() }, Scenario { name: "metrics_timeout_flag", @@ -810,8 +943,6 @@ CMD ["echo", "fast_benchmark"]"#, // When a VM times out, the metrics should include timed_out: true. dockerfile: r#"FROM busybox CMD ["sleep", "3600"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "5"], validate: |output| { // The stderr should contain metrics with timed_out: true @@ -837,6 +968,7 @@ CMD ["sleep", "3600"]"#, } bail!("Expected timed_out: true in metrics: {json_str}") }, + ..Scenario::default() }, // ======================================================================= // HMAC Result Integrity scenarios (Item 11) @@ -848,8 +980,6 @@ CMD ["sleep", "3600"]"#, // The vmm child process should log [HMAC] status on stderr. dockerfile: r#"FROM busybox CMD ["echo", "hmac_test_output"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -873,6 +1003,7 @@ CMD ["echo", "hmac_test_output"]"#, } } }, + ..Scenario::default() }, Scenario { name: "metrics_transport_type", @@ -880,8 +1011,6 @@ CMD ["echo", "hmac_test_output"]"#, // Verifies the metrics include the transport type (vsock or serial). dockerfile: r#"FROM busybox CMD ["echo", "transport_test"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { let metrics_line = output @@ -903,6 +1032,7 @@ CMD ["echo", "transport_test"]"#, } bail!("Could not find transport in metrics: {json_str}") }, + ..Scenario::default() }, // ======================================================================= // Cancellation scenarios @@ -915,7 +1045,6 @@ CMD ["echo", "transport_test"]"#, dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo started && sleep 3600"]"#, cancel_after_secs: Some(5), - sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { // The runner should exit with a non-zero code (killed by signal) @@ -931,6 +1060,7 @@ CMD ["sh", "-c", "echo started && sleep 3600"]"#, } Ok(()) }, + ..Scenario::default() }, // ======================================================================= // Output edge-case scenarios @@ -940,8 +1070,6 @@ CMD ["sh", "-c", "echo started && sleep 3600"]"#, description: "Stderr captured when stdout is empty", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo error_output >&2"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stderr.contains("error_output") { @@ -954,14 +1082,13 @@ CMD ["sh", "-c", "echo error_output >&2"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "empty_output", description: "Process exits 0 with no output", dockerfile: r#"FROM busybox CMD ["true"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -974,6 +1101,7 @@ CMD ["true"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "binary_output", @@ -982,8 +1110,6 @@ CMD ["true"]"#, // The runner should not panic — it should lossy-convert or pass through. dockerfile: r#"FROM busybox CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // The runner must not crash. Exit code 0 and "done" somewhere @@ -1005,6 +1131,7 @@ CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // OCI config parsing scenarios @@ -1016,8 +1143,6 @@ CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, // OCI config stores this as ["/bin/sh", "-c", "echo shell_form_works"] // which differs from exec form ["echo", "shell_form_works"]. dockerfile: "FROM busybox\nCMD echo shell_form_works", - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("shell_form_works") { @@ -1031,6 +1156,7 @@ CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "entrypoint_only", @@ -1039,8 +1165,6 @@ CMD ["sh", "-c", "printf '\\x80\\x81\\xFE\\xFF' && echo done"]"#, // CMD args appended. The runner must not fail when Cmd is null/empty. dockerfile: r#"FROM busybox ENTRYPOINT ["echo", "entrypoint_only_works"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("entrypoint_only_works") { @@ -1054,6 +1178,7 @@ ENTRYPOINT ["echo", "entrypoint_only_works"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "shell_form_entrypoint", @@ -1061,8 +1186,6 @@ ENTRYPOINT ["echo", "entrypoint_only_works"]"#, // Shell form ENTRYPOINT: stored as ["/bin/sh", "-c", "echo ..."] // in OCI config. CMD is ignored when ENTRYPOINT uses shell form. dockerfile: "FROM busybox\nENTRYPOINT echo shell_entrypoint_works", - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.stdout.contains("shell_entrypoint_works") { @@ -1076,6 +1199,7 @@ ENTRYPOINT ["echo", "entrypoint_only_works"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "entrypoint_shell_with_cmd", @@ -1090,8 +1214,6 @@ ENTRYPOINT ["echo", "entrypoint_only_works"]"#, dockerfile: r#"FROM busybox ENTRYPOINT echo ep_marker CMD ["cmd_arg"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1111,6 +1233,7 @@ CMD ["cmd_arg"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "no_cmd_no_entrypoint", @@ -1119,8 +1242,6 @@ CMD ["cmd_arg"]"#, // to fail with a clear error, not crash or hang. dockerfile: r#"FROM busybox RUN echo "no command set""#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "30"], validate: |output| { // The runner should fail (non-zero exit) since there's nothing to run. @@ -1135,6 +1256,7 @@ RUN echo "no command set""#, ) } }, + ..Scenario::default() }, Scenario { name: "bencher_cli_mock", @@ -1146,8 +1268,6 @@ RUN echo "no command set""#, // shared libraries, and ld.so.cache from multi-layer images. dockerfile: r#"FROM ghcr.io/bencherdev/bencher:latest CMD ["mock"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { if output.exit_code == 127 { @@ -1173,6 +1293,7 @@ CMD ["mock"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "distroless_glibc_image", @@ -1189,8 +1310,6 @@ RUN echo '#include \nint main(){printf("distroless_glibc_ok\\n");return FROM gcr.io/distroless/cc-debian12 COPY --from=builder /tmp/hello /usr/bin/hello CMD ["/usr/bin/hello"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "120"], validate: |output| { if output.exit_code == 127 { @@ -1215,6 +1334,7 @@ CMD ["/usr/bin/hello"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // Race condition scenarios @@ -1227,8 +1347,6 @@ CMD ["/usr/bin/hello"]"#, // results are collected even for very short-lived processes. dockerfile: r#"FROM busybox CMD ["echo", "rapid_exit_marker"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1249,6 +1367,7 @@ CMD ["echo", "rapid_exit_marker"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // Exit code scenarios @@ -1260,8 +1379,6 @@ CMD ["echo", "rapid_exit_marker"]"#, // The runner should capture and report this exit code. dockerfile: r#"FROM busybox CMD ["sh", "-c", "exit 137"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // The runner should report exit code 137 somewhere in its output, @@ -1278,6 +1395,7 @@ CMD ["sh", "-c", "exit 137"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // Environment scenarios @@ -1292,8 +1410,6 @@ ENV A1=val1 A2=val2 A3=val3 A4=val4 A5=val5 A6=val6 A7=val7 A8=val8 A9=val9 A10= ENV B1=val11 B2=val12 B3=val13 B4=val14 B5=val15 B6=val16 B7=val17 B8=val18 B9=val19 B10=val20 ENV LARGE_VALUE=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA CMD ["sh", "-c", "echo A1=$A1 B10=$B10 LARGE_LEN=${#LARGE_VALUE}"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1313,6 +1429,7 @@ CMD ["sh", "-c", "echo A1=$A1 B10=$B10 LARGE_LEN=${#LARGE_VALUE}"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // File output edge cases @@ -1324,8 +1441,6 @@ CMD ["sh", "-c", "echo A1=$A1 B10=$B10 LARGE_LEN=${#LARGE_VALUE}"]"#, // The runner should still succeed (exit 0) without crashing. dockerfile: r#"FROM busybox CMD ["echo", "no file written"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--output", "/nonexistent/path.json"], validate: |output| { // Runner should not crash, regardless of exit code. @@ -1336,14 +1451,13 @@ CMD ["echo", "no file written"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "large_file_output", description: "Large output file (~2 MB) transferred via vsock", dockerfile: r#"FROM busybox CMD ["sh", "-c", "dd if=/dev/urandom bs=1024 count=2048 2>/dev/null | base64 > /tmp/output.json && echo done"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--output", "/tmp/output.json"], validate: |output| { if output.exit_code != 0 { @@ -1352,14 +1466,13 @@ CMD ["sh", "-c", "dd if=/dev/urandom bs=1024 count=2048 2>/dev/null | base64 > / } Ok(()) }, + ..Scenario::default() }, Scenario { name: "completed_with_all_fields", description: "Stdout + stderr + output file simultaneously", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo stdout_marker && echo stderr_marker >&2 && echo '{\"data\":true}' > /tmp/out.json"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--output", "/tmp/out.json"], validate: |output| { if output.exit_code != 0 { @@ -1374,14 +1487,13 @@ CMD ["sh", "-c", "echo stdout_marker && echo stderr_marker >&2 && echo '{\"data\ } Ok(()) }, + ..Scenario::default() }, Scenario { name: "multi_file_output", description: "Multiple output files collected via vsock", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo '{\"result\": 1}' > /tmp/a.json && echo '{\"result\": 2}' > /tmp/b.json && echo done"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &[ "--timeout", "60", @@ -1397,6 +1509,7 @@ CMD ["sh", "-c", "echo '{\"result\": 1}' > /tmp/a.json && echo '{\"result\": 2}' } Ok(()) }, + ..Scenario::default() }, // ======================================================================= // OCI image variations @@ -1409,8 +1522,6 @@ RUN echo "a" > /tmp/file_a.txt RUN mkdir -p /opt && echo "b" > /opt/file_b.txt RUN echo "c" > /var/file_c.txt CMD ["sh", "-c", "cat /tmp/file_a.txt /opt/file_b.txt /var/file_c.txt"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1426,6 +1537,7 @@ CMD ["sh", "-c", "cat /tmp/file_a.txt /opt/file_b.txt /var/file_c.txt"]"#, bail!("Expected 'a', 'b', 'c' in output, got: {}", output.stdout) } }, + ..Scenario::default() }, Scenario { name: "image_with_symlinks", @@ -1433,8 +1545,6 @@ CMD ["sh", "-c", "cat /tmp/file_a.txt /opt/file_b.txt /var/file_c.txt"]"#, dockerfile: r#"FROM busybox RUN echo "target" > /tmp/target.txt && ln -s /tmp/target.txt /tmp/link.txt CMD ["cat", "/tmp/link.txt"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1450,6 +1560,7 @@ CMD ["cat", "/tmp/link.txt"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // Error / edge case scenarios @@ -1459,8 +1570,6 @@ CMD ["cat", "/tmp/link.txt"]"#, description: "Writes stdout+stderr then exits non-zero", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo partial_stdout && echo partial_stderr >&2 && exit 1"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { // The runner may succeed (exit 0) even when the guest exits non-zero. @@ -1472,14 +1581,13 @@ CMD ["sh", "-c", "echo partial_stdout && echo partial_stderr >&2 && exit 1"]"#, bail!("Expected partial output to be captured, got: {combined}") } }, + ..Scenario::default() }, Scenario { name: "minimum_timeout", description: "1-second timeout kills long-running process", dockerfile: r#"FROM busybox CMD ["sleep", "3600"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "1"], validate: |output| { if output.exit_code == 0 { @@ -1487,6 +1595,7 @@ CMD ["sleep", "3600"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "max_output_size_truncation", @@ -1494,8 +1603,6 @@ CMD ["sleep", "3600"]"#, // Generate ~50 KB of output, but limit to 1024 bytes. dockerfile: r#"FROM busybox CMD ["sh", "-c", "dd if=/dev/zero bs=1024 count=50 2>/dev/null | tr '\\0' 'X'"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--max-output-size", "1024"], validate: |output| { // Output should be bounded — not the full ~50KB @@ -1508,6 +1615,7 @@ CMD ["sh", "-c", "dd if=/dev/zero bs=1024 count=50 2>/dev/null | tr '\\0' 'X'"]" // Runner didn't OOM or crash — that's a pass Ok(()) }, + ..Scenario::default() }, Scenario { name: "env_var_passthrough", @@ -1517,8 +1625,6 @@ ENV LD_PRELOAD=/test.so ENV LD_LIBRARY_PATH=/testlib ENV SAFE_VAR=safe_value CMD ["sh", "-c", "echo LD_PRELOAD=$LD_PRELOAD LD_LIBRARY_PATH=$LD_LIBRARY_PATH SAFE=$SAFE_VAR"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1545,6 +1651,7 @@ CMD ["sh", "-c", "echo LD_PRELOAD=$LD_PRELOAD LD_LIBRARY_PATH=$LD_LIBRARY_PATH S } Ok(()) }, + ..Scenario::default() }, // ======================================================================= // Resource constraint enforcement @@ -1556,8 +1663,6 @@ CMD ["sh", "-c", "echo LD_PRELOAD=$LD_PRELOAD LD_LIBRARY_PATH=$LD_LIBRARY_PATH S // `free -m` reports total memory; we check it's in the right ballpark. dockerfile: r#"FROM busybox CMD ["free", "-m"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--memory", "64", "--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1573,6 +1678,7 @@ CMD ["free", "-m"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "disk_size_override", @@ -1583,8 +1689,6 @@ CMD ["free", "-m"]"#, // the block device level. This test validates the config path. dockerfile: r#"FROM busybox CMD ["sh", "-c", "df -m / | tail -1 | awk '{print $2}'"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--disk", "64", "--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1601,6 +1705,7 @@ CMD ["sh", "-c", "df -m / | tail -1 | awk '{print $2}'"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "disk_limit_enforced", @@ -1610,8 +1715,6 @@ CMD ["sh", "-c", "df -m / | tail -1 | awk '{print $2}'"]"#, // approximately 64 MiB total (minus overhead), not more. dockerfile: r#"FROM busybox CMD ["sh", "-c", "df -m / | tail -1 | awk '{print \"TOTAL_MB=\" $2}'"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--disk", "64", "--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1636,14 +1739,13 @@ CMD ["sh", "-c", "df -m / | tail -1 | awk '{print \"TOTAL_MB=\" $2}'"]"#, output.stdout ) }, + ..Scenario::default() }, Scenario { name: "cpu_count_visible", description: "Guest sees 1 CPU with default vCPU count", dockerfile: r#"FROM busybox CMD ["nproc"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1656,6 +1758,7 @@ CMD ["nproc"]"#, bail!("Expected '1' CPU from nproc, got: {}", output.stdout) } }, + ..Scenario::default() }, // ======================================================================= // Network enabled @@ -1667,8 +1770,6 @@ CMD ["nproc"]"#, // Use wget to a well-known URL as a connectivity test. dockerfile: r#"FROM busybox CMD ["sh", "-c", "wget -q -O /dev/null http://detectportal.firefox.com/success.txt && echo net_ok || echo net_fail"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "30", "--network"], validate: |output| { let combined = format!("{}{}", output.stdout, output.stderr); @@ -1685,6 +1786,7 @@ CMD ["sh", "-c", "wget -q -O /dev/null http://detectportal.firefox.com/success.t Ok(()) } }, + ..Scenario::default() }, // ======================================================================= // File permissions @@ -1697,8 +1799,6 @@ CMD ["sh", "-c", "wget -q -O /dev/null http://detectportal.firefox.com/success.t dockerfile: r#"FROM busybox RUN mkdir -p /data && echo "content_ok" > /data/file.txt CMD ["cat", "/data/file.txt"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1711,6 +1811,7 @@ CMD ["cat", "/data/file.txt"]"#, bail!("Expected 'content_ok' in output, got: {}", output.stdout) } }, + ..Scenario::default() }, Scenario { name: "file_permissions_preserved", @@ -1720,8 +1821,6 @@ CMD ["cat", "/data/file.txt"]"#, dockerfile: r#"FROM busybox RUN mkdir -p /data && printf '#!/bin/sh\necho hello' > /data/test.sh && chmod +x /data/test.sh CMD ["sh", "-c", "test -x /data/test.sh && echo perm_ok"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1737,6 +1836,7 @@ CMD ["sh", "-c", "test -x /data/test.sh && echo perm_ok"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "directory_permissions_preserved", @@ -1746,8 +1846,6 @@ CMD ["sh", "-c", "test -x /data/test.sh && echo perm_ok"]"#, dockerfile: r#"FROM busybox RUN mkdir -p /data/restricted && chmod 750 /data/restricted CMD ["stat", "-c", "%a", "/data/restricted"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1763,6 +1861,7 @@ CMD ["stat", "-c", "%a", "/data/restricted"]"#, ) } }, + ..Scenario::default() }, // ======================================================================= // Special characters in environment variables @@ -1772,8 +1871,6 @@ CMD ["stat", "-c", "%a", "/data/restricted"]"#, description: "Env vars with spaces, equals, and quotes work", // Use Docker's multi-line ENV syntax with quotes for values with spaces. dockerfile: "FROM busybox\nENV SPACED=\"hello world\" WITH_EQ=\"key=value\"\nCMD [\"sh\", \"-c\", \"echo SPACED=$SPACED EQ=$WITH_EQ\"]", - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60"], validate: |output| { if output.exit_code != 0 { @@ -1791,6 +1888,7 @@ CMD ["stat", "-c", "%a", "/data/restricted"]"#, } Ok(()) }, + ..Scenario::default() }, // ======================================================================= // CLI override scenarios (--entrypoint, --cmd, --env) @@ -1801,8 +1899,6 @@ CMD ["stat", "-c", "%a", "/data/restricted"]"#, dockerfile: r#"FROM busybox ENTRYPOINT ["echo", "image_ep"] CMD ["image_cmd"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--entrypoint", "echo", "cli_ep"], validate: |output| { if output.exit_code != 0 { @@ -1831,6 +1927,7 @@ CMD ["image_cmd"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "cli_cmd_override", @@ -1838,8 +1935,6 @@ CMD ["image_cmd"]"#, dockerfile: r#"FROM busybox ENTRYPOINT ["echo"] CMD ["image_cmd"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--cmd", "cli_cmd"], validate: |output| { if output.exit_code != 0 { @@ -1861,6 +1956,7 @@ CMD ["image_cmd"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "cli_entrypoint_and_cmd_override", @@ -1868,8 +1964,6 @@ CMD ["image_cmd"]"#, dockerfile: r#"FROM busybox ENTRYPOINT ["echo", "image_ep"] CMD ["image_cmd"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &[ "--timeout", "60", @@ -1898,6 +1992,7 @@ CMD ["image_cmd"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "cli_env_override", @@ -1905,8 +2000,6 @@ CMD ["image_cmd"]"#, dockerfile: r#"FROM busybox ENV MY_VAR=image_value CMD ["sh", "-c", "echo MY_VAR=$MY_VAR"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--env", "MY_VAR=cli_value"], validate: |output| { if output.exit_code != 0 { @@ -1923,6 +2016,7 @@ CMD ["sh", "-c", "echo MY_VAR=$MY_VAR"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "cli_env_add", @@ -1930,8 +2024,6 @@ CMD ["sh", "-c", "echo MY_VAR=$MY_VAR"]"#, dockerfile: r#"FROM busybox ENV EXISTING=from_image CMD ["sh", "-c", "echo EXISTING=$EXISTING NEW=$NEW_VAR"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--env", "NEW_VAR=from_cli"], validate: |output| { if output.exit_code != 0 { @@ -1952,14 +2044,13 @@ CMD ["sh", "-c", "echo EXISTING=$EXISTING NEW=$NEW_VAR"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "cli_env_multiple", description: "Multiple --env flags", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo A=$A B=$B"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--env", "A=one", "--env", "B=two"], validate: |output| { if output.exit_code != 0 { @@ -1974,14 +2065,13 @@ CMD ["sh", "-c", "echo A=$A B=$B"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "cli_entrypoint_no_image_entrypoint", description: "Add entrypoint when image only has CMD", dockerfile: r#"FROM busybox CMD ["hello", "world"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--entrypoint", "echo"], validate: |output| { if output.exit_code != 0 { @@ -1999,14 +2089,13 @@ CMD ["hello", "world"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "multiple_iterations", description: "Multiple iterations execute sequentially", dockerfile: r#"FROM busybox CMD ["echo", "iter_output"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--iter", "3"], validate: |output| { if output.exit_code != 0 { @@ -2019,14 +2108,13 @@ CMD ["echo", "iter_output"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "zero_iterations", description: "Zero iterations executes no benchmarks", dockerfile: r#"FROM busybox CMD ["echo", "should_not_appear"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--iter", "0"], validate: |output| { if output.exit_code != 0 { @@ -2037,14 +2125,13 @@ CMD ["echo", "should_not_appear"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "allow_failure_false_aborts", description: "Non-zero exit code aborts iteration without --allow-failure", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--iter", "3"], validate: |output| { if output.exit_code == 0 { @@ -2063,14 +2150,13 @@ CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, } Ok(()) }, + ..Scenario::default() }, Scenario { name: "allow_failure_true_continues", description: "Non-zero exit code continues with --allow-failure", dockerfile: r#"FROM busybox CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, - cancel_after_secs: None, - sandboxed: true, extra_args: &["--timeout", "60", "--iter", "3", "--allow-failure"], validate: |output| { if output.exit_code != 0 { @@ -2089,6 +2175,7 @@ CMD ["sh", "-c", "echo __ITER_DONE__ && exit 1"]"#, } Ok(()) }, + ..Scenario::default() }, ] } @@ -2104,7 +2191,6 @@ fn nosandbox_scenarios() -> Vec { description: "Non-sandboxed: simple echo", dockerfile: r#"FROM busybox:musl CMD ["echo", "hello from host"]"#, - cancel_after_secs: None, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2118,6 +2204,7 @@ CMD ["echo", "hello from host"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "nosandbox_env", @@ -2125,7 +2212,6 @@ CMD ["echo", "hello from host"]"#, dockerfile: r#"FROM busybox:musl ENV MY_VAR=host_test_value CMD ["sh", "-c", "echo $MY_VAR"]"#, - cancel_after_secs: None, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2139,6 +2225,7 @@ CMD ["sh", "-c", "echo $MY_VAR"]"#, ) } }, + ..Scenario::default() }, Scenario { name: "nosandbox_metrics", @@ -2147,7 +2234,6 @@ CMD ["sh", "-c", "echo $MY_VAR"]"#, // transport "local" (it previously emitted no metrics at all). dockerfile: r#"FROM busybox:musl CMD ["echo", "local_metrics_test"]"#, - cancel_after_secs: None, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2173,13 +2259,13 @@ CMD ["echo", "local_metrics_test"]"#, } bail!("Could not find transport in metrics: {json_str}") }, + ..Scenario::default() }, Scenario { name: "nosandbox_exit_code", description: "Non-sandboxed: non-zero exit code propagation", dockerfile: r#"FROM busybox:musl CMD ["sh", "-c", "exit 42"]"#, - cancel_after_secs: None, sandboxed: false, extra_args: &["--timeout", "60"], validate: |output| { @@ -2196,6 +2282,7 @@ CMD ["sh", "-c", "exit 42"]"#, ) } }, + ..Scenario::default() }, ] } @@ -2216,6 +2303,31 @@ fn kvm_available() -> bool { Path::new("/dev/kvm").exists() } +/// The cargo target directory the builds above land in. +/// +/// `CARGO_TARGET_DIR` is honored rather than assumed away: a caller that +/// redirects it would otherwise have the binaries built in one place and +/// looked for in another, and the harness would report a missing binary +/// immediately after reporting a successful build. +fn target_dir() -> Utf8PathBuf { + std::env::var_os("CARGO_TARGET_DIR").map_or_else( + || super::workspace_root().join("target"), + |dir| Utf8PathBuf::from(dir.to_string_lossy().into_owned()), + ) +} + +/// Whether this process is running as root. +fn is_root() -> bool { + #[expect( + unsafe_code, + reason = "geteuid has no std wrapper and cannot fail or touch memory" + )] + // SAFETY: `geteuid` takes no arguments, returns a plain integer, and is + // always successful. + let euid = unsafe { libc::geteuid() }; + euid == 0 +} + /// Check if Docker is available. fn docker_available() -> bool { Command::new("docker") @@ -2333,6 +2445,31 @@ fn run_runner_with_cancel( /// Build bencher-init for the musl target and the runner CLI with `BENCHER_INIT_PATH`, /// then return the path to the runner binary. fn ensure_runner_bin() -> Result { + // The elevated run must not invoke cargo: doing so as root leaves the + // target directory and cargo's cache root-owned, which then breaks the + // unprivileged steps around it. CI builds first and points here. + if let Some(path) = std::env::var_os(RUNNER_BIN_ENV) { + let path = Utf8PathBuf::from(path.to_string_lossy().into_owned()); + if !path.exists() { + bail!("{RUNNER_BIN_ENV} is set to {path}, which does not exist"); + } + println!("Using pre-built runner from {RUNNER_BIN_ENV}: {path}"); + return Ok(path); + } + + // Falling through to cargo as root is the exact outcome the + // build-then-elevate split exists to prevent: it leaves the target + // directory and the cargo cache owned by root, and it does so silently. + // Sudo does pass the variable through on both runner images, so this is + // belt and braces, but a loud failure beats a root-owned cache. + anyhow::ensure!( + !is_root(), + "Running as root without {RUNNER_BIN_ENV} set. Building here would run cargo as root and \ + leave the target directory and cargo cache root-owned. Build unprivileged first:\n\ + \x20 cargo test-runner scenarios --build-only\n\ + \x20 sudo {RUNNER_BIN_ENV}=./target/debug/runner ./target/debug/test_runner scenarios" + ); + let workspace_root = super::workspace_root(); let target_triple = super::musl_target_triple()?; @@ -2347,7 +2484,7 @@ fn ensure_runner_bin() -> Result { bail!("cargo build -p bencher_init --target {target_triple} failed"); } - let init_path = workspace_root.join(format!("target/{target_triple}/debug/bencher-init")); + let init_path = target_dir().join(format!("{target_triple}/debug/bencher-init")); if !init_path.exists() { bail!("bencher-init binary not found at {init_path} after build"); } @@ -2364,7 +2501,7 @@ fn ensure_runner_bin() -> Result { bail!("cargo build -p bencher_runner_cli failed"); } - let runner_bin = workspace_root.join("target/debug/runner"); + let runner_bin = target_dir().join("debug/runner"); if !runner_bin.exists() { bail!("Runner binary not found at {runner_bin} after build"); } @@ -2372,22 +2509,1304 @@ fn ensure_runner_bin() -> Result { Ok(runner_bin) } -/// Run the runner and capture output. -fn run_runner( - image_path: &Utf8Path, - args: &[&str], - runner_bin: &Utf8Path, -) -> Result { - let output = Command::new(runner_bin.as_str()) - .arg("run") - .arg("--image") - .arg(image_path.as_str()) - .args(args) - .output()?; +// --------------------------------------------------------------------------- +// Jail confinement +// --------------------------------------------------------------------------- - Ok(ScenarioOutput { - stdout: String::from_utf8_lossy(&output.stdout).to_string(), - stderr: String::from_utf8_lossy(&output.stderr).to_string(), - exit_code: output.status.code().unwrap_or(-1), - }) +/// Environment variable naming a pre-built runner binary. +const RUNNER_BIN_ENV: &str = "BENCHER_RUNNER_BIN"; + +/// How long to wait for the jailed VMM to appear before giving up. +/// +/// Generous: the runner pulls and unpacks the image and builds the rootfs +/// before the VMM is spawned. +const PROBE_TIMEOUT: Duration = Duration::from_mins(3); + +/// How often to look for the jailed VMM. +const PROBE_INTERVAL: Duration = Duration::from_millis(100); + +/// The state directory scenarios run against. +fn scenario_state_dir() -> Utf8PathBuf { + super::work_dir().join("state") +} + +/// The directory holding one chroot per jailed VMM. +fn jail_parent(state_dir: &Utf8Path) -> Utf8PathBuf { + state_dir.join("jail").join("firecracker") +} + +/// Scenarios covering the confinement of the VMM itself. +fn jail_scenarios() -> Vec { + vec![ + Scenario { + name: "jail_confinement", + description: "A jailed job succeeds with the VMM unprivileged and in its cgroup", + // The guest sleeps so the VMM is alive long enough to be observed + // by a probe that polls every 100ms. + // + // The marker is a token the runner's own output cannot contain. + // "jailed" collided with the runner announcing "Launching jailed + // Firecracker microVM...", so the check passed on the runner + // saying it was about to start a VM that then never booted. + dockerfile: r#"FROM busybox +CMD ["sh", "-c", "echo JAIL_CONFINEMENT_a7f3b2c9 && sleep 5"]"#, + cancel_after_secs: None, + probe: Some(probe_confinement), + orphan_then_rerun: false, + extra_args: &["--timeout", "120"], + validate: |output| { + // The job has to have actually run before anything the probe + // saw means anything. Every confinement property the probe + // checks is equally true of a VMM that started and then never + // booted a guest, so without this the scenario stays green + // while the product is broken. + assert_job_succeeded(output, "JAIL_CONFINEMENT_a7f3b2c9")?; + assert_cpu_isolation_applied(output)?; + assert_no_chroot_remains(&scenario_state_dir()) + }, + ..Scenario::default() + }, + Scenario { + name: "jail_netns_recovers_from_stacked_mounts", + description: "A job succeeds against a network namespace handle carrying stacked mounts", + dockerfile: r#"FROM busybox +CMD ["echo", "JAIL_NETNS_a7f3b2c9"]"#, + setup: Some(stack_netns_mounts), + extra_args: &["--timeout", "120"], + validate: |output| assert_job_succeeded(output, "JAIL_NETNS_a7f3b2c9"), + ..Scenario::default() + }, + Scenario { + name: "jail_sweep_reclaims_orphan", + description: "A chroot orphaned by a runner that never unwound is swept by the next job", + // Likewise a token the runner cannot print: "swept" sits one + // refactor away from colliding with the sweep's own reporting. + dockerfile: r#"FROM busybox +CMD ["sh", "-c", "echo JAIL_SWEEP_a7f3b2c9 && sleep 10"]"#, + cancel_after_secs: None, + probe: None, + orphan_then_rerun: true, + extra_args: &["--timeout", "120"], + validate: |output| { + assert_job_succeeded(output, "JAIL_SWEEP_a7f3b2c9")?; + assert_no_chroot_remains(&scenario_state_dir()) + }, + ..Scenario::default() + }, + ] +} + +/// Assert the runner reported that it confined the VMM to the benchmark cores. +/// +/// The runner prints this only after creating the cgroup, writing the cpuset, +/// and reading the effective set back, so it is the runner's own statement +/// that a cgroup exists for the probe to have checked membership against. +/// Without it the probe could pass on a host where no cgroup was ever made. +fn assert_cpu_isolation_applied(output: &ScenarioOutput) -> Result<()> { + const PINNED: &str = "CPU isolation: Firecracker pinned to cores"; + if output.stdout.contains(PINNED) { + return Ok(()); + } + bail!( + "The runner never reported pinning the VMM to benchmark cores, so no cgroup was created \ + and cgroup placement went unexercised by this run. Expected {PINNED:?}.\nstdout: {}\nstderr: {}", + output.stdout, + output.stderr + ) +} + +/// Stack extra bind mounts on the network namespace handle. +/// +/// Recreating the handle bind mounts over it, and a bind mount over a file +/// reports no error, so mounts stack. Against a stacked handle a single +/// detach leaves one behind, the unlink then fails with EBUSY, and creating +/// the placeholder fails with EPERM even as root: every sandboxed job on the +/// host fails until an operator loops `umount` by hand. The unwind loop exists +/// for exactly this, and nothing else exercises it. +fn stack_netns_mounts() -> Result<()> { + let handle = "/run/netns/bencher-jail"; + fs::create_dir_all("/run/netns").context("Failed to create the netns directory")?; + if !Utf8Path::new(handle).exists() { + fs::File::create(handle).context("Failed to create the netns handle")?; + } + + for _ in 0..2 { + let status = Command::new("unshare") + .args(["--net", "sh", "-c"]) + .arg(format!("mount --bind /proc/self/ns/net {handle}")) + .status() + .context("Failed to run unshare to stack a netns mount")?; + anyhow::ensure!(status.success(), "Failed to stack a netns mount"); + } + + let stacked = fs::read_to_string("/proc/self/mountinfo") + .context("Failed to read mountinfo")? + .lines() + .filter(|line| line.contains(&format!(" {handle} "))) + .count(); + anyhow::ensure!( + stacked >= 2, + "Expected at least two stacked mounts on {handle}, found {stacked}" + ); + println!(" stacked {stacked} mounts on {handle}"); + Ok(()) +} + +/// Assert the runner actually completed the job. +/// +/// A confinement scenario that asserts only confinement passes vacuously when +/// the VM never boots: the VMM process exists, is unprivileged, and is in its +/// cgroup either way. Success of the job itself is the precondition for any of +/// that meaning anything. +/// +/// `marker` has to be a token the runner's own progress output cannot contain. +/// This captures the runner's stdout, not the guest's, and the runner prints +/// plenty about jails and sweeps on its way to launching a VM. +fn assert_job_succeeded(output: &ScenarioOutput, marker: &str) -> Result<()> { + if output.exit_code != 0 { + bail!( + "Expected the job to succeed, got exit code {}.\nstdout: {}\nstderr: {}", + output.exit_code, + output.stdout, + output.stderr + ); + } + if !output.stdout.contains(marker) { + bail!( + "Expected '{marker}' in the guest output, so the VM booted and ran.\nstdout: {}\nstderr: {}", + output.stdout, + output.stderr + ); + } + Ok(()) +} + +/// Assert every chroot has been reclaimed. +/// +/// The jailer cleans up nothing by design, so a leftover here means the +/// runner's teardown did not run: each one holds a copy of the VMM binary and +/// a full guest rootfs image. +fn assert_no_chroot_remains(state_dir: &Utf8Path) -> Result<()> { + let parent = jail_parent(state_dir); + // A read that failed is not an empty directory. An assertion that could not + // look has not passed, it has not run, and a vacuous pass here would hide the + // exact leak it exists to catch. Absence is the one reading that does mean + // nothing was left behind: the runner creates this tree on demand. + let entries = match fs::read_dir(&parent) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => { + return Err(e).with_context(|| { + format!("Failed to read {parent}, so whether a chroot was left behind is unknown") + }); + }, + }; + + let mut leftovers = Vec::new(); + for entry in entries { + let entry = entry.with_context(|| { + format!("Failed to read an entry under {parent}, so whether a chroot was left behind is unknown") + })?; + leftovers.push(entry.file_name().to_string_lossy().into_owned()); + } + + anyhow::ensure!( + leftovers.is_empty(), + "Chroots left behind under {parent}: {leftovers:?}" + ); + Ok(()) +} + +/// Check that the jailed VMM is unprivileged and already in its cgroup. +/// +/// Both invariants disappear with the process, so they cannot be recovered +/// from the runner's output. Cgroup membership in particular must already +/// hold the first time the VMM is seen: it is established before the exec, +/// not after the VM is running. +fn probe_confinement(state_dir: &Utf8Path) -> Result { + let parent = jail_parent(state_dir); + let Some((vm_id, jail_root)) = find_jail(&parent)? else { + return Ok(false); + }; + let Some(pid) = find_jailed_vmm(&jail_root)? else { + return Ok(false); + }; + + if !check_unprivileged(pid, &jail_root)? { + return Ok(false); + } + // Placement happens in `pre_exec`, before the jailer itself starts, so + // membership already holds the first time the process is observable. + // There is no not-ready-yet window for it. + check_cgroup_membership(&vm_id, pid)?; + + Ok(true) +} + +/// Find the single chroot under the jail parent, if one exists yet. +/// +/// `Ok(None)` is "not yet", which the parent not existing also means: the runner +/// creates it on demand. Every other failure is an error, because this drives a +/// poll loop whose only other outcome is a timeout, and a timeout would report +/// that no VMM ever appeared when the truth is that nobody could look. +fn find_jail(parent: &Utf8Path) -> Result> { + let entries = match fs::read_dir(parent) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e).with_context(|| format!("Failed to read {parent}")), + }; + + for entry in entries { + let entry = entry.with_context(|| format!("Failed to read an entry under {parent}"))?; + let file_type = entry + .file_type() + .with_context(|| format!("Failed to read the kind of an entry under {parent}"))?; + if !file_type.is_dir() { + continue; + } + let vm_id = entry.file_name().to_string_lossy().into_owned(); + let jail_root = parent.join(&vm_id).join("root"); + if jail_root.is_dir() { + return Ok(Some((vm_id, jail_root))); + } + } + Ok(None) +} + +/// Find the pid of the VMM confined to `jail_root`, if it is running yet. +/// +/// The jailer pivots into a private mount namespace, so the process's root +/// path reads back as `/` and is useless as an identifier. Its identity is +/// compared instead: the bind mount the jailer pivots onto preserves the +/// device and inode of the chroot directory, so stat'ing through +/// `/proc//root` and stat'ing the jail root agree for exactly the VMM +/// confined to this jail and for no other process on the host. +/// +/// `Ok(None)` is "not running yet", which a jail root that does not exist also +/// means. A jail root that cannot be stat'ed, or a `/proc` that cannot be listed, +/// is neither: it would surface as a probe timeout blaming the runner for +/// something the harness could not see. +fn find_jailed_vmm(jail_root: &Utf8Path) -> Result> { + use std::os::unix::fs::MetadataExt as _; + + let jail = match fs::metadata(jail_root) { + Ok(jail) => jail, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e).with_context(|| format!("Failed to stat {jail_root}")), + }; + for entry in fs::read_dir("/proc").context("Failed to read /proc")? { + let entry = entry.context("Failed to read a /proc entry")?; + let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { + continue; + }; + // Following the magic symlink crosses into the process's own mount + // namespace, which a privileged reader is allowed to do. A read that + // fails is a process that has exited or is not this jail's, which is the + // one failure here that is genuinely an answer. + let Ok(root) = fs::metadata(format!("/proc/{pid}/root")) else { + continue; + }; + if root.dev() == jail.dev() && root.ino() == jail.ino() { + return Ok(Some(pid)); + } + } + Ok(None) +} + +/// Check the VMM dropped root and runs as the user the jail was handed to. +/// +/// Returns `Ok(false)` while the observation is premature rather than wrong. +/// The jailer `pivot_root`s before it drops privilege, so there is a window in +/// which the process root already matches the jail while the process is still +/// root and the jail root is still root-owned. Treating that as a violation +/// would fail the run for catching the jailer mid-flight; the probe's timeout +/// is what catches a VMM that genuinely never drops. +fn check_unprivileged(pid: u32, jail_root: &Utf8Path) -> Result { + let status = fs::read_to_string(format!("/proc/{pid}/status")) + .with_context(|| format!("Failed to read the status of the VMM (pid {pid})"))?; + let uid_line = status + .lines() + .find_map(|line| line.strip_prefix("Uid:")) + .context("No Uid line in the VMM's /proc status")?; + let vmm_uid: u32 = uid_line + .split_whitespace() + .next() + .context("Empty Uid line in the VMM's /proc status")? + .parse() + .context("Unparsable uid in the VMM's /proc status")?; + + if vmm_uid == 0 { + return Ok(false); + } + + // The jailer chowns the chroot root to the jail uid, so the two must + // agree: a VMM running as some other unprivileged user would not be + // confined to the jail it was given. + let Some(jail_uid) = jail_root_uid(jail_root) else { + return Ok(false); + }; + if vmm_uid != jail_uid { + bail!("The VMM (pid {pid}) runs as uid {vmm_uid} but its jail is owned by uid {jail_uid}"); + } + + Ok(true) +} + +/// The uid the jailer handed the chroot root to, once it has handed it over. +/// +/// `None` while the root is still owned by root, which is the same +/// not-ready-yet window `check_unprivileged` documents. +fn jail_root_uid(jail_root: &Utf8Path) -> Option { + use std::os::unix::fs::MetadataExt as _; + + let uid = fs::metadata(jail_root).ok()?.uid(); + (uid != 0).then_some(uid) +} + +/// Check the VMM is in its cgroup. +/// +/// Placement happens before the exec, so the pid is already a member the +/// first time the process is observable. +fn check_cgroup_membership(vm_id: &str, pid: u32) -> Result<()> { + let procs_path = format!("/sys/fs/cgroup/bencher/{vm_id}/cgroup.procs"); + + // A missing cgroup is a failure, not a note. Placement before exec is the + // centrepiece of the jail: it is what fixed the cpuset being applied after + // the VMM was already running. Letting its absence pass quietly is how a + // scenario named for confinement ends up asserting only the uid half of + // it, which is a green that means less than it looks like. + let procs = fs::read_to_string(&procs_path).with_context(|| { + format!( + "No cgroup at {procs_path}, so cgroup placement was not exercised at all. \ + The runner creates one whenever its CPU layout offers isolation, which needs \ + two or more online CPUs and the cpuset controller delegated to this cgroup tree." + ) + })?; + + if procs.lines().any(|line| line.trim() == pid.to_string()) { + Ok(()) + } else { + bail!( + "The VMM (pid {pid}) is not in {procs_path}, which holds: {procs:?}. \ + Placement happens in pre_exec, before the jailer starts, so membership must \ + already hold the first time the process is visible." + ) + } +} + +/// Orphan a jail by killing the runner, then prove the next job sweeps it. +/// +/// SIGKILL rather than SIGTERM, because SIGTERM on the one-shot path takes the +/// default disposition too (signal handlers are installed only by the daemon), +/// and either way nothing unwinds. That is the point: `Drop` cannot reclaim +/// the chroot, so if the next job finds a clean tree it can only be because +/// the sweep reclaimed it. +fn run_runner_after_orphan( + image_path: &Utf8Path, + args: &[&str], + state_dir: &Utf8Path, + runner_bin: &Utf8Path, +) -> Result { + let parent = jail_parent(state_dir); + + let mut child = Command::new(runner_bin.as_str()) + .arg("run") + .arg("--image") + .arg(image_path.as_str()) + .args(args) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn()?; + + // Drain both pipes for the same reason the probe path does. This polls for + // up to three minutes while the runner pulls an image, unpacks it, and + // builds an ext4, which is more than enough output to fill a 64 KiB pipe + // and block the runner. It would surface as "No jailed VMM appeared", + // which points at the sweep rather than at the pipe. + let readers = drain_output(&mut child); + + // Wait for a real orphan: a chroot with a VMM running in it, not just an + // empty directory created microseconds before the kill. + let deadline = std::time::Instant::now() + PROBE_TIMEOUT; + let orphan = loop { + if let Some((vm_id, jail_root)) = find_jail(&parent)? + && let Some(pid) = find_jailed_vmm(&jail_root)? + { + break Some((vm_id, jail_root, pid)); + } + if child.try_wait()?.is_some() || std::time::Instant::now() >= deadline { + break None; + } + std::thread::sleep(PROBE_INTERVAL); + }; + + let Some((vm_id, jail_root, vmm_pid)) = orphan else { + // Killed before the wait. A probe that timed out leaves the runner still + // going, so waiting on it first would sit there until the runner's own + // timeout expired and report the failure minutes late, which is exactly + // when somebody is watching. Only if it has not already been reaped: + // `try_wait` in the loop above reaps it, and signalling a reaped pid can + // reach whatever inherited the number. + if child.try_wait()?.is_none() { + kill_pid(child.id(), libc::SIGKILL); + } + drop(child.wait()); + let (stdout, stderr) = readers.join(); + bail!( + "No jailed VMM appeared within {PROBE_TIMEOUT:?}, so nothing was orphaned and the sweep is untested.\nstdout: {stdout}\nstderr: {stderr}" + ); + }; + + kill_pid(child.id(), libc::SIGKILL); + drop(child.wait()); + drop(readers.join()); + + if !jail_root + .try_exists() + .with_context(|| format!("Failed to check whether {jail_root} was left behind"))? + { + bail!( + "The chroot {jail_root} was reclaimed despite the runner being killed without unwinding, so the sweep is untested" + ); + } + + // Deliberately NOT reaping the VMM here. Killing it by hand would leave + // the next job's sweep with nothing to find, so the reap, the pidfd + // handling, and the cgroup removal would all be skipped and the scenario + // would prove only that a directory can be deleted. The stray Firecracker + // that a hand-reap guards against is exactly what the sweep now exists to + // prevent, so if the sweep fails this scenario has to go red. + let cgroup = stale_cgroup(&vm_id); + let cgroup_existed = cgroup + .try_exists() + .with_context(|| format!("Failed to check whether {cgroup} was created"))?; + println!( + " orphaned jail {vm_id} (VMM pid {vmm_pid}, cgroup present: {cgroup_existed}), running a second job..." + ); + + let output = run_runner(image_path, args, runner_bin)?; + + // `try_exists`, not `exists`: the latter reports false for an error as well + // as for absence, which would pass this assertion for the wrong reason. + if jail_root + .try_exists() + .with_context(|| format!("Failed to check whether {jail_root} survived"))? + { + bail!("The orphaned chroot {jail_root} survived the next job, so it was never swept"); + } + if is_firecracker(vmm_pid)? { + bail!( + "The orphaned VMM (pid {vmm_pid}) is still running after the next job, so the sweep never reaped it. It still holds the benchmark cores." + ); + } + // Only meaningful where a cgroup was created at all: a host with no CPU + // isolation never makes one, and asserting its absence would pass for the + // wrong reason. + if cgroup_existed + && cgroup + .try_exists() + .with_context(|| format!("Failed to check whether {cgroup} survived"))? + { + bail!( + "The orphaned cgroup {cgroup} survived the next job, so the sweep never removed it. Stale cgroups accumulate, and one that will not go away usually means its VMM is still running." + ); + } + + Ok(output) +} + +/// The cgroup a jail leaves behind, which shares the jail's name. +fn stale_cgroup(vm_id: &str) -> Utf8PathBuf { + Utf8PathBuf::from("/sys/fs/cgroup/bencher").join(vm_id) +} + +/// Whether a pid is a running Firecracker. +/// +/// Checking the command as well as the pid keeps a recycled pid from reading +/// as a VMM that was never reaped. +/// +/// A process that is gone has no `comm` to read, and that is the answer the +/// caller wants. Any other read failure is not: the assertion that uses this +/// passes when it returns false, so swallowing an error would pass it for the +/// wrong reason. +fn is_firecracker(pid: u32) -> Result { + match fs::read_to_string(format!("/proc/{pid}/comm")) { + Ok(comm) => Ok(comm.trim() == "firecracker"), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e).with_context(|| { + format!( + "Failed to read the command of pid {pid}, so whether the VMM was reaped is unknown" + ) + }), + } +} + +/// Reader threads draining a child's piped output. +struct DrainedOutput { + stdout: std::thread::JoinHandle, + stderr: std::thread::JoinHandle, +} + +impl DrainedOutput { + /// Wait for both readers and return what they collected. + fn join(self) -> (String, String) { + let stdout = self.stdout.join().unwrap_or_default(); + let stderr = self.stderr.join().unwrap_or_default(); + (stdout, stderr) + } +} + +/// Start reading a child's stdout and stderr so neither pipe can fill. +fn drain_output(child: &mut std::process::Child) -> DrainedOutput { + fn reader( + stream: Option, + ) -> std::thread::JoinHandle { + std::thread::spawn(move || { + let mut buffer = String::new(); + if let Some(mut stream) = stream { + drop(stream.read_to_string(&mut buffer)); + } + buffer + }) + } + + DrainedOutput { + stdout: reader(child.stdout.take()), + stderr: reader(child.stderr.take()), + } +} + +/// Send a signal to a process, ignoring the result. +fn kill_pid(pid: u32, signal: libc::c_int) { + #[expect( + unsafe_code, + clippy::cast_possible_wrap, + reason = "libc::kill requires unsafe; PID fits in i32" + )] + // SAFETY: `kill` takes plain integers and touches no memory. A signal to a + // pid that has already exited fails harmlessly with ESRCH. + unsafe { + libc::kill(pid as i32, signal); + } +} + +/// The host settings the tuning scenario watches, and what the runner sets them +/// to. +/// +/// Only settings whose whole value the runner rewrites. The bracketed sysfs +/// files (`transparent_hugepage/*`) and the cpuset partition are handled +/// separately below, and the knobs this scenario deliberately leaves alone are +/// listed with the arguments that switch them off. +const TUNED_SETTINGS: &[(&str, &str)] = &[ + ("/proc/sys/kernel/randomize_va_space", "0"), + ("/proc/sys/kernel/nmi_watchdog", "0"), + ("/proc/sys/vm/swappiness", "10"), + ("/proc/sys/kernel/perf_event_paranoid", "-1"), + ("/proc/sys/kernel/numa_balancing", "0"), + ("/proc/sys/kernel/timer_migration", "0"), + ("/proc/sys/kernel/soft_watchdog", "0"), + ("/sys/kernel/mm/ksm/run", "0"), +]; + +/// The transparent hugepage settings, whose files list every mode and bracket +/// the selected one. +const TUNED_THP: &[&str] = &[ + "/sys/kernel/mm/transparent_hugepage/enabled", + "/sys/kernel/mm/transparent_hugepage/defrag", +]; + +/// What the runner sets the transparent hugepage mode to. +const THP_TARGET: &str = "never"; + +/// The cpuset partition files, which the tuning writes and the guard restores. +const TUNED_PARTITION: &[&str] = &[ + "/sys/fs/cgroup/bencher/cpuset.cpus", + "/sys/fs/cgroup/bencher/cpuset.mems", + "/sys/fs/cgroup/bencher/cpuset.cpus.partition", +]; + +/// One host setting the scenario expects the runner to change. +#[derive(Debug, Clone)] +struct TunedSetting { + path: Utf8PathBuf, + /// What it held before the runner started, and what it must hold after. + original: String, + /// What the runner should set it to while the Job runs, when this host + /// lets it. `None` for a setting that is present and writable but already + /// holds the target, which the runner leaves alone and reports as such. + expected: Option, + /// Whether the value is the bracketed kind (`always [madvise] never`). + bracketed: bool, +} + +/// Every host setting the tuning scenario touches, as it stood before it ran. +/// +/// The harness restores from this itself rather than trusting the mechanism it +/// is testing. A test of a restore path has to be safe when the restore path is +/// broken, which is the whole reason this scenario can be allowed to run in CI +/// at all. +#[derive(Debug)] +struct TuningSnapshot { + settings: Vec, +} + +impl TuningSnapshot { + /// Read every setting, and work out which of them this host will let the + /// runner change. + /// + /// Writability is established by writing the current value back, which + /// changes nothing and is the only honest way to know: a file that exists + /// may still be read-only, and a scenario that waited for a change the + /// kernel was never going to make would fail for the host's reasons rather + /// than the runner's. + fn take() -> Self { + let mut settings = Vec::new(); + + for (path, target) in TUNED_SETTINGS { + let path = Utf8PathBuf::from(*path); + let Some(original) = readable_setting(&path) else { + println!(" tuning: {path} is not present on this host"); + continue; + }; + let expected = if !writable_setting(&path, &original) { + println!(" tuning: {path} is present but not writable"); + None + } else if original == *target { + println!(" tuning: {path} already holds {target}"); + None + } else { + Some((*target).to_owned()) + }; + settings.push(TunedSetting { + path, + original, + expected, + bracketed: false, + }); + } + + for path in TUNED_THP { + let path = Utf8PathBuf::from(*path); + let Some(original) = readable_setting(&path) else { + println!(" tuning: {path} is not present on this host"); + continue; + }; + // Probed with the mode the file already selects, never with a + // fallback: writing `never` to a file whose selection could not be + // parsed would change the very setting this is only supposed to + // measure. + let Some(selected) = bracketed_value(&original) else { + println!(" tuning: {path} does not read as a mode listing: '{original}'"); + continue; + }; + let expected = if !writable_setting(&path, selected) { + println!(" tuning: {path} is present but not writable"); + None + } else if selected == THP_TARGET { + println!(" tuning: {path} already selects {THP_TARGET}"); + None + } else { + Some(THP_TARGET.to_owned()) + }; + settings.push(TunedSetting { + path, + original, + expected, + bracketed: true, + }); + } + + Self { settings } + } + + /// The settings this host should show changed while the Job runs. + fn expected(&self) -> impl Iterator { + self.settings + .iter() + .filter(|setting| setting.expected.is_some()) + } + + /// Whether every expected setting currently holds its tuned value. + fn all_applied(&self) -> bool { + self.expected().all(|setting| { + let Some(current) = readable_setting(&setting.path) else { + return false; + }; + let Some(target) = setting.expected.as_deref() else { + return true; + }; + if setting.bracketed { + bracketed_value(¤t) == Some(target) + } else { + current == target + } + }) + } + + /// Which expected settings are not showing their tuned value. + fn missing(&self) -> Vec { + self.expected() + .filter(|setting| { + let Some(current) = readable_setting(&setting.path) else { + return true; + }; + let target = setting.expected.as_deref().unwrap_or_default(); + if setting.bracketed { + bracketed_value(¤t) != Some(target) + } else { + current != target + } + }) + .map(|setting| { + let current = readable_setting(&setting.path).unwrap_or_else(|| "?".to_owned()); + format!( + "{} is '{current}', expected '{}'", + setting.path, + setting.expected.as_deref().unwrap_or_default() + ) + }) + .collect() + } + + /// Which settings are not back to what they were. + fn unrestored(&self) -> Vec { + self.settings + .iter() + .filter_map(|setting| { + let current = readable_setting(&setting.path)?; + (current != setting.original).then(|| { + format!( + "{} is '{current}', was '{}'", + setting.path, setting.original + ) + }) + }) + .collect() + } + + /// Put everything back, whatever the runner did or failed to do. + /// + /// Reports what it had to undo: anything here means the guard under test did + /// not do its job, and the scenario has already failed for that reason, but + /// the machine still has to be left as it was found. + fn restore(&self) { + for setting in &self.settings { + let Some(current) = readable_setting(&setting.path) else { + continue; + }; + if current == setting.original { + continue; + } + // The bracketed files take the mode alone, never the whole listing. + let value = if setting.bracketed { + bracketed_value(&setting.original) + .unwrap_or(THP_TARGET) + .to_owned() + } else { + setting.original.clone() + }; + match fs::write(&setting.path, &value) { + Ok(()) => println!(" tuning: harness restored {} to '{value}'", setting.path), + Err(e) => println!( + " tuning: harness could NOT restore {} to '{value}': {e}", + setting.path + ), + } + } + } +} + +/// Restores the host tuning when it goes out of scope. +/// +/// A guard rather than a call at the end, so a panic or an early return in the +/// scenario cannot leave the machine tuned. Nothing survives the harness itself +/// being killed, which is why the scenario runs last. +struct RestoreTuning(TuningSnapshot); + +impl Drop for RestoreTuning { + fn drop(&mut self) { + self.0.restore(); + } +} + +/// Read a host setting, trimmed, if it is there at all. +fn readable_setting(path: &Utf8Path) -> Option { + fs::read_to_string(path).ok().map(|v| v.trim().to_owned()) +} + +/// Whether a setting can be written, established by writing back what it holds. +fn writable_setting(path: &Utf8Path, current: &str) -> bool { + fs::write(path, current).is_ok() +} + +/// The selected mode in a bracketed sysfs listing (`always [madvise] never`). +fn bracketed_value(listing: &str) -> Option<&str> { + let (_, selected) = listing.split_once('[')?; + let (selected, _) = selected.split_once(']')?; + Some(selected) +} + +/// The cpuset partition files that exist, with what they hold. +/// +/// Read separately from the rest because the partition is created by the tuning +/// itself: the files do not exist before the first tuned run on a fresh host, so +/// there is nothing to snapshot and their absence afterwards is the restored +/// state. +fn partition_state() -> Vec<(Utf8PathBuf, String)> { + TUNED_PARTITION + .iter() + .map(Utf8PathBuf::from) + .filter_map(|path| readable_setting(&path).map(|value| (path, value))) + .collect() +} + +/// Reap anything a previous scenario left running, before the wipe strands it. +/// +/// A cancelled scenario SIGTERMs `runner run`, which installs no handler for it, +/// so the process dies without unwinding: its VMM stays alive in its cgroup and +/// its chroot stays on disk. That is the case the runner's sweep exists for, and +/// the sweep finds the VMM by the chroot, comparing device and inode against +/// `/proc//root`. Wiping the state directory destroys that handle, so the +/// next runner sweeps a directory that no longer names anything, reports the +/// host clean, and the orphan runs on through every scenario that follows. +/// +/// The product refuses to remove a chroot whose VMM is alive for exactly this +/// reason. The harness has been doing it once per scenario, so it does the +/// reclaiming the sweep would have done rather than leaving a live VMM with +/// nothing pointing at it. +fn reclaim_stranded_jails(state_dir: &Utf8Path) -> Result<()> { + let parent = jail_parent(state_dir); + let entries = match fs::read_dir(&parent) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(e).with_context(|| format!("Failed to read {parent}")), + }; + + for entry in entries { + let entry = entry.with_context(|| format!("Failed to read an entry under {parent}"))?; + if !entry + .file_type() + .with_context(|| format!("Failed to read the kind of an entry under {parent}"))? + .is_dir() + { + continue; + } + let vm_id = entry.file_name().to_string_lossy().into_owned(); + let jail_root = parent.join(&vm_id).join("root"); + + if let Some(pid) = find_jailed_vmm(&jail_root)? { + println!(" reclaiming VMM (pid {pid}) stranded in {vm_id}"); + kill_pid(pid, libc::SIGKILL); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while is_firecracker(pid)? && std::time::Instant::now() < deadline { + std::thread::sleep(PROBE_INTERVAL); + } + anyhow::ensure!( + !is_firecracker(pid)?, + "A VMM stranded in {vm_id} (pid {pid}) would not die, so it would run on through every scenario that follows" + ); + } + + // The cgroup shares the jail's name, and nothing else will come looking + // for it once the directory below is gone. + let cgroup = stale_cgroup(&vm_id); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while cgroup.exists() { + if fs::remove_dir(&cgroup).is_ok() { + println!(" reclaimed the cgroup {cgroup}"); + break; + } + anyhow::ensure!( + std::time::Instant::now() < deadline, + "The cgroup {cgroup} could not be removed, so it would block the cpuset restore of every run that follows" + ); + std::thread::sleep(PROBE_INTERVAL); + } + } + + Ok(()) +} + +/// What the `bencher` cgroup looks like, for when the partition assertion fails. +/// +/// Clearing a parent's `cpuset.cpus` is refused with `EIO` while any task remains +/// in a descendant, so the useful question after a failed restore is what is +/// still in there. Without this the answer costs a CI round. +fn partition_diagnosis() -> String { + let root = Utf8Path::new("/sys/fs/cgroup/bencher"); + if !root.exists() { + return "the bencher cgroup is gone".to_owned(); + } + let procs = fs::read_to_string(root.join("cgroup.procs")).unwrap_or_default(); + let children: Vec = fs::read_dir(root) + .map(|entries| { + entries + .flatten() + .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_dir())) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect() + }) + .unwrap_or_default(); + let child_procs: Vec = children + .iter() + .map(|child| { + let tasks = + fs::read_to_string(root.join(child).join("cgroup.procs")).unwrap_or_default(); + // Named, not numbered. A bare pid costs a round trip to identify, + // and what the process is decides whose bug it is. + let named: Vec = tasks + .split_whitespace() + .map(|pid| { + let comm = fs::read_to_string(format!("/proc/{pid}/comm")) + .map_or_else(|_| "gone".to_owned(), |comm| comm.trim().to_owned()); + format!("{pid} ({comm})") + }) + .collect(); + format!("{child} holds [{}]", named.join(" ")) + }) + .collect(); + format!( + "the bencher cgroup is still there, holding tasks [{}] and children {child_procs:?}", + procs.split_whitespace().collect::>().join(" ") + ) +} + +/// Run the runner with host tuning on, and assert it both applies and unwinds. +/// +/// The assertion that matters is the pair. Applying is what the runner is for; +/// restoring is what keeps a benchmark host from drifting a knob at a time +/// across every Job it ever runs, and `TuningGuard` restoring on `Drop` had +/// never once executed in CI before this scenario existed. +/// Read the host, and work out what this run should change. +/// +/// Separated so the scenario itself stays readable: everything here happens +/// before the runner starts and decides whether there is anything to test. +fn plan_tuning() -> Result<(TuningSnapshot, Vec)> { + let snapshot = TuningSnapshot::take(); + let expected: Vec = snapshot + .expected() + .map(|setting| { + format!( + "{} -> {}", + setting.path, + setting.expected.as_deref().unwrap_or_default() + ) + }) + .collect(); + + // A scenario that finds nothing to change would pass without testing + // anything, which is the failure this suite has spent the most effort + // removing. If a host really offers none of these, that is a fact worth a + // red build rather than a green one. + anyhow::ensure!( + !expected.is_empty(), + "No tuning knob on this host can be exercised, so the scenario would pass vacuously. Settings considered: {:?}", + snapshot + .settings + .iter() + .map(|s| s.path.as_str()) + .collect::>() + ); + println!( + " tuning: expecting {} setting(s) to change: {}", + expected.len(), + expected.join(", ") + ); + Ok((snapshot, expected)) +} + +fn run_runner_with_tuning( + image_path: &Utf8Path, + args: &[&str], + runner_bin: &Utf8Path, +) -> Result { + let (snapshot, expected) = plan_tuning()?; + let partition_before = partition_state(); + + // Taken before the runner starts, so the machine is put back even if the + // scenario panics, the assertions fail, or the runner dies without + // unwinding. The point of the scenario is that the guard under test might + // not work. + let restore = RestoreTuning(snapshot); + + let mut child = Command::new(runner_bin.as_str()) + .arg("run") + .arg("--image") + .arg(image_path.as_str()) + .args(args) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn()?; + let readers = drain_output(&mut child); + + // Watch for the tuning to land while the Job runs. The runner applies it + // before it pulls the image, so this is looking at a window that lasts the + // whole run. + let deadline = std::time::Instant::now() + PROBE_TIMEOUT; + let mut applied = false; + loop { + if restore.0.all_applied() { + applied = true; + break; + } + if child.try_wait()?.is_some() || std::time::Instant::now() >= deadline { + break; + } + std::thread::sleep(PROBE_INTERVAL); + } + let partition_during = partition_state(); + + if !applied && child.try_wait()?.is_none() { + kill_pid(child.id(), libc::SIGKILL); + } + let status = child.wait()?; + let (stdout, stderr) = readers.join(); + + if !applied { + bail!( + "Host tuning never applied within {PROBE_TIMEOUT:?}: {:?}.\nstdout: {stdout}\nstderr: {stderr}", + restore.0.missing() + ); + } + + // The Job has to have succeeded as well. A scenario that only watched the + // knobs would pass on a runner that tuned the host and then failed to run + // anything, which is the vacuous half of a confinement assertion in another + // dress. + if status.code() != Some(0) { + bail!( + "Tuning applied but the Job failed with exit code {:?}.\nstdout: {stdout}\nstderr: {stderr}", + status.code() + ); + } + + // And it has to be gone now that the runner has exited. + let unrestored = restore.0.unrestored(); + if !unrestored.is_empty() { + bail!( + "Host tuning was not restored when the runner exited: {unrestored:?}.\nstdout: {stdout}\nstderr: {stderr}" + ); + } + + // Only the files that were there to change. The partition creates its own + // cgroup, so a file that did not exist before the run has no previous value + // to be restored to, and asserting on its appearance would fail the scenario + // for the tuning having worked. + let partition_after = partition_state(); + let partition_unrestored: Vec = partition_before + .iter() + .filter_map(|(path, before)| { + let after = partition_after + .iter() + .find_map(|(p, v)| (p == path).then_some(v.as_str()))?; + (after != before).then(|| format!("{path} is '{after}', was '{before}'")) + }) + .collect(); + if !partition_unrestored.is_empty() { + bail!( + "The cpuset partition was not restored: {partition_unrestored:?}. Now {}.\nstdout: {stdout}\nstderr: {stderr}", + partition_diagnosis() + ); + } + if partition_during.is_empty() { + println!(" tuning: no cpuset partition files on this host, so none were asserted"); + } + + println!( + " tuning: {} setting(s) applied and restored, {} partition file(s) checked", + expected.len(), + partition_before.len() + ); + + Ok(ScenarioOutput { + stdout, + stderr, + exit_code: status.code().unwrap_or(-1), + }) +} + +/// Scenarios covering host tuning, which every other scenario switches off. +fn tuning_scenarios() -> Vec { + vec![Scenario { + name: "host_tuning", + description: "Host tuning applies while a Job runs and is restored after", + dockerfile: r#"FROM busybox +CMD ["echo", "tuned run complete"]"#, + extra_args: &["--timeout", "60"], + tuning: true, + // The Job's own output as well as the knobs. A run that tuned the host + // and then never booted a VM would otherwise satisfy this scenario. + validate: |output| assert_job_succeeded(output, "tuned run complete"), + ..Scenario::default() + }] +} + +/// Run the runner while checking a host-side invariant. +fn run_runner_with_probe( + image_path: &Utf8Path, + args: &[&str], + probe: Probe, + state_dir: &Utf8Path, + runner_bin: &Utf8Path, +) -> Result { + let mut child = Command::new(runner_bin.as_str()) + .arg("run") + .arg("--image") + .arg(image_path.as_str()) + .args(args) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn()?; + + // Drain both pipes while the probe runs. Nothing reads them during the + // loop otherwise, so a runner chatty enough to fill the 64 KiB pipe buffer + // blocks on its own output until the probe times out. + let readers = drain_output(&mut child); + + let deadline = std::time::Instant::now() + PROBE_TIMEOUT; + let mut observed = None; + loop { + match probe(state_dir) { + Ok(true) => { + observed = Some(Ok(())); + break; + }, + Ok(false) => {}, + Err(e) => { + observed = Some(Err(e)); + break; + }, + } + // Stop looking once the runner is gone or the wait is hopeless: the + // output is collected either way so the failure can be explained. + if child.try_wait()?.is_some() || std::time::Instant::now() >= deadline { + break; + } + std::thread::sleep(PROBE_INTERVAL); + } + + // A probe that ended without observing the VMM leaves the runner going, and + // waiting on it would sit there until the runner's own timeout expired, + // reporting the failure minutes late. Only a run that observed what it came + // for is allowed to finish, since its output is the result being collected. + // Guarded on the reap, because `try_wait` above reaps and signalling a reaped + // pid can reach whatever inherited the number. + if !matches!(observed, Some(Ok(()))) && child.try_wait()?.is_none() { + kill_pid(child.id(), libc::SIGKILL); + } + let status = child.wait()?; + let (stdout, stderr) = readers.join(); + + match observed { + Some(Ok(())) => Ok(ScenarioOutput { + stdout, + stderr, + exit_code: status.code().unwrap_or(-1), + }), + Some(Err(e)) => Err(e).with_context(|| format!("stdout: {stdout}\nstderr: {stderr}")), + None => bail!( + "The jailed VMM was never observed within {PROBE_TIMEOUT:?}.\nstdout: {stdout}\nstderr: {stderr}" + ), + } +} + +/// Run the runner and capture output. +fn run_runner( + image_path: &Utf8Path, + args: &[&str], + runner_bin: &Utf8Path, +) -> Result { + let output = Command::new(runner_bin.as_str()) + .arg("run") + .arg("--image") + .arg(image_path.as_str()) + .args(args) + .output()?; + + Ok(ScenarioOutput { + stdout: String::from_utf8_lossy(&output.stdout).to_string(), + stderr: String::from_utf8_lossy(&output.stderr).to_string(), + exit_code: output.status.code().unwrap_or(-1), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_selected_mode_is_the_bracketed_one() { + // What the kernel prints for a transparent hugepage setting: every mode + // it offers, with the live one in brackets. Comparing the whole line + // against "never" would never match, and asserting on a substring would + // match a mode that is merely offered. + assert_eq!( + bracketed_value("always [madvise] never"), + Some("madvise"), + "the enabled listing" + ); + assert_eq!( + bracketed_value("always defer defer+madvise [madvise] never"), + Some("madvise"), + "the defrag listing, which offers more modes" + ); + assert_eq!(bracketed_value("[always] madvise never"), Some("always")); + assert_eq!(bracketed_value("always madvise [never]"), Some("never")); + } + + #[test] + fn a_listing_with_no_selection_has_no_value() { + // A plain sysctl is not a listing, and a truncated read is not a mode. + assert_eq!(bracketed_value("never"), None); + assert_eq!(bracketed_value(""), None); + assert_eq!(bracketed_value("always [madvise"), None); + } + + #[test] + fn a_setting_that_is_not_there_reads_as_absent() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8Path::from_path(dir.path()).unwrap(); + + assert_eq!(readable_setting(&root.join("absent")), None); + } + + #[test] + fn a_setting_reads_back_trimmed() { + let dir = tempfile::tempdir().unwrap(); + let root = Utf8Path::from_path(dir.path()).unwrap(); + let path = root.join("swappiness"); + fs::write(&path, "60\n").unwrap(); + + assert_eq!(readable_setting(&path).as_deref(), Some("60")); + } + + #[test] + fn writability_is_established_by_writing_what_is_already_there() { + // The probe that decides whether a knob can be exercised on this host. + // A file that exists may still refuse writes, which is not something a + // stat can answer: `/proc/sys/kernel/nmi_watchdog` is exactly that on a + // kernel without a hardware watchdog, and waiting for it to change would + // fail the scenario for the host's reasons rather than the runner's. + let dir = tempfile::tempdir().unwrap(); + let root = Utf8Path::from_path(dir.path()).unwrap(); + let path = root.join("knob"); + fs::write(&path, "1\n").unwrap(); + + assert!(writable_setting(&path, "1")); + assert_eq!( + readable_setting(&path).as_deref(), + Some("1"), + "the probe writes back what was there, so it changes nothing" + ); + + // Root ignores the permission bits, and the scenarios run as root, so + // this half only means anything unprivileged. + if !is_root() { + let mut perms = fs::metadata(&path).unwrap().permissions(); + perms.set_readonly(true); + fs::set_permissions(&path, perms).unwrap(); + + assert!(!writable_setting(&path, "1")); + } + } }