Confine the Firecracker VMM with the jailer

.github/workflows/runner.yml

@@ -54,8 +54,20 @@ jobs:
         run: |
           echo '{"features": {"containerd-snapshotter": true}}' | sudo tee /etc/docker/daemon.json
           sudo systemctl restart docker
+      - name: Build scenario binaries
+        run: cargo test-runner scenarios --build-only
       - name: Run scenarios
-        run: cargo test-runner scenarios
+        # The scenarios run as root. The sandbox is built by dropping
+        # privilege rather than by starting without it: the jailer creates the
+        # chroot's device nodes with mknod, chowns the tree to the jail user,
+        # pivot_roots, and joins a network namespace. The udev rule above makes
+        # /dev/kvm usable by an unprivileged process, which is necessary but
+        # not sufficient.
+        #
+        # Only this step is elevated. The build above stays as the CI user, so
+        # cargo's cache and target directory are not left root-owned, and
+        # BENCHER_RUNNER_BIN keeps cargo out of the elevated step entirely.
+        run: sudo BENCHER_RUNNER_BIN=./target/debug/runner ./target/debug/test_runner scenarios
 
   build_runner:
     name: Build Runner (${{ matrix.build }})

CLAUDE.md

@@ -168,7 +168,10 @@ Defined in `.cargo/config.toml`:
   - `cargo test-api seed` needs the API server already running (`cargo run` in `services/api`) with a fresh database (`services/api/data` holds only a tracked `.gitignore`; delete `services/api/data/bencher.db`, not the whole directory)
   - `cargo test-api seed` also needs the `bencher` CLI binary already built (`cargo build --bin bencher`); it shells out via `assert_cmd`, which panics with `` `CARGO_BIN_EXE_bencher` is unset `` if the binary is missing
   - Pass `--no-git` when running the seed test in this repo: there is no colocated `.git`, so `bencher run` cannot derive a git context and the on-the-fly project naming assertions (`bencher` vs `Project`) will fail without it
-- `cargo test-runner` - Runner integration tests (requires Linux + KVM)
+- `cargo test-runner` - Runner integration tests (requires Linux + KVM + root)
+  - `cargo test-runner scenarios` always fails unelevated: the sandbox is built by dropping privilege, so the scenarios refuse to start without it. Build unprivileged, then run elevated, which also keeps `cargo` from leaving root-owned artifacts in `target`:
+    - `cargo test-runner scenarios --build-only`
+    - `sudo BENCHER_RUNNER_BIN=./target/debug/runner ./target/debug/test_runner scenarios`
 
 ## Git Flow
 

plus/bencher_runner/Cargo.toml

@@ -60,7 +60,13 @@ libc = { workspace = true }
 
 # Linux-only dependencies
 [target.'cfg(target_os = "linux")'.dependencies]
-nix = { workspace = true, features = ["feature", "poll", "sched", "signal"] }
+nix = { workspace = true, features = [
+    "feature",
+    "mount",
+    "poll",
+    "sched",
+    "signal",
+] }
 
 [build-dependencies]
 flate2 = { workspace = true }

plus/bencher_runner/build.rs

@@ -1,7 +1,7 @@
 //! Build script for `bencher_runner`.
 //!
-//! Bundles the `bencher-init`, `firecracker`, and `vmlinux` binaries
-//! for distribution as a single binary.
+//! Bundles the `bencher-init`, `firecracker`, `jailer`, and `vmlinux`
+//! binaries for distribution as a single binary.
 //!
 //! In release builds, binaries are embedded via `include_bytes!`.
 //! In debug builds, they are downloaded/cached locally and loaded from disk at runtime.
@@ -20,9 +20,10 @@
 //!
 //! # Environment Variable Overrides
 //!
-//! - `BENCHER_INIT_PATH` — path to a pre-built bencher-init binary
-//! - `BENCHER_FIRECRACKER_PATH` — path to a pre-built firecracker binary
-//! - `BENCHER_KERNEL_PATH` — path to a pre-built vmlinux kernel
+//! - `BENCHER_INIT_PATH`: path to a pre-built bencher-init binary
+//! - `BENCHER_FIRECRACKER_PATH`: path to a pre-built firecracker binary
+//! - `BENCHER_JAILER_PATH`: path to a pre-built jailer binary
+//! - `BENCHER_KERNEL_PATH`: path to a pre-built vmlinux kernel
 
 #![expect(
     clippy::expect_used,
@@ -103,16 +104,26 @@ fn main() {
         generate_stub_module("init", &out_dir);
     }
 
-    // --- firecracker ---
-    let firecracker_path = find_or_download_firecracker(&out_dir);
-    if is_release {
-        let firecracker_path = firecracker_path.unwrap_or_else(|| panic!("firecracker binary not found. Set BENCHER_FIRECRACKER_PATH or ensure download succeeds."));
-        generate_binary_module("firecracker", &firecracker_path, is_release, &out_dir);
-    } else if let Some(firecracker_path) = firecracker_path {
-        generate_binary_module("firecracker", &firecracker_path, is_release, &out_dir);
-    } else {
-        eprintln!("WARNING: firecracker not found, generating stub module for debug build");
-        generate_stub_module("firecracker", &out_dir);
+    // --- firecracker and jailer ---
+    // Both ship in the same release archive, so a single download under a
+    // single hash check yields both. Bundling them together also keeps the
+    // VMM and its jailer at the same version across a runner self-update.
+    let (firecracker_path, jailer_path) = find_or_download_firecracker_release(&out_dir);
+    for (name, path) in [("firecracker", firecracker_path), ("jailer", jailer_path)] {
+        if is_release {
+            let path = path.unwrap_or_else(|| {
+                panic!(
+                    "{name} binary not found. Set BENCHER_{}_PATH or ensure download succeeds.",
+                    name.to_uppercase()
+                )
+            });
+            generate_binary_module(name, &path, is_release, &out_dir);
+        } else if let Some(path) = path {
+            generate_binary_module(name, &path, is_release, &out_dir);
+        } else {
+            eprintln!("WARNING: {name} not found, generating stub module for debug build");
+            generate_stub_module(name, &out_dir);
+        }
     }
 
     // --- kernel (vmlinux) ---
@@ -132,6 +143,7 @@ fn main() {
     println!("cargo:rerun-if-changed=build.rs");
     println!("cargo:rerun-if-env-changed=BENCHER_INIT_PATH");
     println!("cargo:rerun-if-env-changed=BENCHER_FIRECRACKER_PATH");
+    println!("cargo:rerun-if-env-changed=BENCHER_JAILER_PATH");
     println!("cargo:rerun-if-env-changed=BENCHER_KERNEL_PATH");
     println!("cargo:rerun-if-env-changed=PROFILE");
 }
@@ -200,70 +212,88 @@ fn find_init_binary() -> Option<PathBuf> {
     None
 }
 
-/// Find or download the Firecracker binary.
+/// Find or download the `firecracker` and `jailer` binaries.
 ///
-/// Checks `BENCHER_FIRECRACKER_PATH` env var first, then tries to download
-/// the `.tgz` release archive from GitHub and extract the binary to `OUT_DIR`.
-fn find_or_download_firecracker(out_dir: &Path) -> Option<PathBuf> {
-    // 1. Check explicit env var
-    if let Ok(path) = env::var("BENCHER_FIRECRACKER_PATH") {
-        let path = PathBuf::from(path);
-        if path.exists() {
-            eprintln!(
-                "Using firecracker from BENCHER_FIRECRACKER_PATH: {}",
-                path.display()
-            );
-            return Some(path);
-        }
-        eprintln!(
-            "WARNING: BENCHER_FIRECRACKER_PATH set but file not found: {}",
-            path.display()
-        );
-    }
+/// Checks the `BENCHER_FIRECRACKER_PATH` and `BENCHER_JAILER_PATH` env vars
+/// first, then downloads the `.tgz` release archive from GitHub once and
+/// extracts whichever binaries are still missing into `OUT_DIR`.
+///
+/// Returns `(firecracker, jailer)`.
+fn find_or_download_firecracker_release(out_dir: &Path) -> (Option<PathBuf>, Option<PathBuf>) {
+    let firecracker_override = binary_path_override("firecracker", "BENCHER_FIRECRACKER_PATH");
+    let jailer_override = binary_path_override("jailer", "BENCHER_JAILER_PATH");
 
-    // 2. Download from GitHub releases
     let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default();
     let arch = match target_arch.as_str() {
         "x86_64" => "x86_64",
         "aarch64" => "aarch64",
         _ => {
-            eprintln!("Unsupported architecture for firecracker: {target_arch}");
-            return None;
+            eprintln!("Unsupported architecture for the Firecracker release: {target_arch}");
+            return (firecracker_override, jailer_override);
         },
     };
 
-    let dest = out_dir.join("firecracker");
-    if dest.exists() {
-        eprintln!("Using cached firecracker at: {}", dest.display());
-        return Some(dest);
-    }
-
-    let url = format!(
-        "https://github.com/firecracker-microvm/firecracker/releases/download/{DEFAULT_FIRECRACKER_VERSION}/firecracker-{DEFAULT_FIRECRACKER_VERSION}-{arch}.tgz",
-    );
+    // The binaries inside the tgz are at:
+    // release-{version}-{arch}/{name}-{version}-{arch}
+    let wanted: Vec<(String, PathBuf)> = ["firecracker", "jailer"]
+        .into_iter()
+        .zip([&firecracker_override, &jailer_override])
+        .filter(|&(_, overridden)| overridden.is_none())
+        .map(|(name, _)| {
+            (
+                format!(
+                    "release-{DEFAULT_FIRECRACKER_VERSION}-{arch}/{name}-{DEFAULT_FIRECRACKER_VERSION}-{arch}",
+                ),
+                out_dir.join(name),
+            )
+        })
+        .filter(|(_, dest)| {
+            let cached = dest.exists();
+            if cached {
+                eprintln!("Using cached binary at: {}", dest.display());
+            }
+            !cached
+        })
+        .collect();
 
-    // The binary inside the tgz is at:
-    // release-{version}-{arch}/firecracker-{version}-{arch}
-    let entry_name = format!(
-        "release-{DEFAULT_FIRECRACKER_VERSION}-{arch}/firecracker-{DEFAULT_FIRECRACKER_VERSION}-{arch}",
-    );
+    if !wanted.is_empty() {
+        let url = format!(
+            "https://github.com/firecracker-microvm/firecracker/releases/download/{DEFAULT_FIRECRACKER_VERSION}/firecracker-{DEFAULT_FIRECRACKER_VERSION}-{arch}.tgz",
+        );
+        let expected_hash = match arch {
+            "x86_64" => FIRECRACKER_TGZ_SHA256_X86_64,
+            "aarch64" => FIRECRACKER_TGZ_SHA256_AARCH64,
+            _ => unreachable!(),
+        };
+
+        eprintln!("Downloading the Firecracker release from: {url}");
+        if let Err(e) = download_and_extract_tgz(&url, &wanted, Some(expected_hash)) {
+            eprintln!("WARNING: Failed to download/extract the Firecracker release: {e}");
+        }
+    }
 
-    let expected_hash = match arch {
-        "x86_64" => FIRECRACKER_TGZ_SHA256_X86_64,
-        "aarch64" => FIRECRACKER_TGZ_SHA256_AARCH64,
-        _ => unreachable!(),
+    let resolved = |overridden: Option<PathBuf>, name: &str| {
+        overridden.or_else(|| {
+            let dest = out_dir.join(name);
+            dest.exists().then_some(dest)
+        })
     };
 
-    eprintln!("Downloading firecracker from: {url}");
-    match download_and_extract_tgz(&url, &entry_name, &dest, Some(expected_hash)) {
-        Ok(()) => {
-            eprintln!("Extracted firecracker to: {}", dest.display());
-            Some(dest)
-        },
-        Err(e) => {
-            eprintln!("WARNING: Failed to download/extract firecracker: {e}");
-            None
-        },
+    (
+        resolved(firecracker_override, "firecracker"),
+        resolved(jailer_override, "jailer"),
+    )
+}
+
+/// Resolve a build-time binary path override from an env var.
+fn binary_path_override(name: &str, var: &str) -> Option<PathBuf> {
+    let path = PathBuf::from(env::var(var).ok()?);
+    if path.exists() {
+        eprintln!("Using {name} from {var}: {}", path.display());
+        Some(path)
+    } else {
+        eprintln!("WARNING: {var} set but file not found: {}", path.display());
+        None
     }
 }
 
@@ -351,18 +381,17 @@ fn download_file(url: &str, dest: &Path, expected_sha256: Option<&str>) -> Resul
     Ok(())
 }
 
-/// Download a `.tgz` archive and extract a single file from it.
+/// Download a `.tgz` archive and extract the requested files from it.
 ///
 /// # Arguments
 ///
 /// * `url` - URL of the `.tgz` archive
-/// * `entry_name` - Path of the entry to extract (e.g., `release-v1.15.1-x86_64/firecracker-v1.15.1-x86_64`)
-/// * `dest` - Destination path for the extracted file
+/// * `wanted` - `(entry_name, dest)` pairs, where `entry_name` is the path of
+///   the entry inside the archive (e.g., `release-v1.15.1-x86_64/firecracker-v1.15.1-x86_64`)
 /// * `expected_sha256` - If `Some`, verify the archive's SHA256 before extracting
 fn download_and_extract_tgz(
     url: &str,
-    entry_name: &str,
-    dest: &Path,
+    wanted: &[(String, PathBuf)],
     expected_sha256: Option<&str>,
 ) -> Result<(), String> {
     let response = ureq::get(url)
@@ -392,6 +421,13 @@ fn download_and_extract_tgz(
     let gz = flate2::read::GzDecoder::new(archive_bytes.as_slice());
     let mut archive = tar::Archive::new(gz);
 
+    // The names still to be found, not a count of them. A tar archive may
+    // legally carry the same path more than once, and counting extractions down
+    // gets that wrong in both directions: a third match against two wanted
+    // entries underflows and panics with a message that says nothing about the
+    // archive, and two matches on the same entry reach zero and report success
+    // while the other is still missing.
+    let mut outstanding: Vec<&str> = wanted.iter().map(|(name, _)| name.as_str()).collect();
     for entry in archive
         .entries()
         .map_err(|e| format!("Failed to read tar entries: {e}"))?
@@ -400,19 +436,32 @@ fn download_and_extract_tgz(
         let path = entry
             .path()
             .map_err(|e| format!("Failed to read entry path: {e}"))?;
-
-        if path.to_string_lossy() == entry_name {
-            let mut bytes = Vec::new();
-            entry
-                .read_to_end(&mut bytes)
-                .map_err(|e| format!("Failed to read entry data: {e}"))?;
-            fs::write(dest, &bytes)
-                .map_err(|e| format!("Failed to write to {}: {e}", dest.display()))?;
+        let path = path.to_string_lossy().into_owned();
+
+        let Some((_, dest)) = wanted.iter().find(|(name, _)| *name == path) else {
+            continue;
+        };
+
+        let mut bytes = Vec::new();
+        entry
+            .read_to_end(&mut bytes)
+            .map_err(|e| format!("Failed to read entry data: {e}"))?;
+        fs::write(dest, &bytes)
+            .map_err(|e| format!("Failed to write to {}: {e}", dest.display()))?;
+        eprintln!("Extracted '{path}' to: {}", dest.display());
+        outstanding.retain(|name| *name != path);
+        if outstanding.is_empty() {
             return Ok(());
         }
     }
 
-    Err(format!("Entry '{entry_name}' not found in archive"))
+    let missing = wanted
+        .iter()
+        .filter(|(_, dest)| !dest.exists())
+        .map(|(name, _)| name.as_str())
+        .collect::<Vec<_>>()
+        .join(", ");
+    Err(format!("Entries not found in archive: {missing}"))
 }
 
 // ---------------------------------------------------------------------------
@@ -489,7 +538,7 @@ pub const {name_upper}_BUNDLED: bool = true;
 fn generate_stub_modules() {
     let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set"));
 
-    for name in &["init", "firecracker", "kernel"] {
+    for name in &["init", "firecracker", "jailer", "kernel"] {
         generate_stub_module(name, &out_dir);
     }
 }