From 33c0290f7bb26e7e7b4a1ce4b6e8a624c3948759 Mon Sep 17 00:00:00 2001 From: Paulo Alvarado Date: Tue, 18 Aug 2026 12:21:24 +0200 Subject: [PATCH] Add the docker-sbx provider The provider runs each host as a Docker Sandboxes microVM. The template image starts sshd and does not need environment variables. The provider injects the SSH key and the caller environment with sbx exec. Then it publishes port 22 on a host port that the daemon selects. The provider sets supports_tailnet to False. The host machine owns the sbx installation, the login, and the daemon. A drukbox container connects through a socket mount and the DOCKER_SANDBOXES_API variable. The images/sbx/ directory contains the template. --- AGENTS.md | 3 +- docs/deploy.md | 112 +++++++++- docs/networking.md | 5 +- docs/security.md | 10 +- images/sbx/Dockerfile | 20 ++ images/sbx/entrypoint.sh | 16 ++ src/providers/__init__.py | 1 + src/providers/docker_sbx/__init__.py | 4 + src/providers/docker_sbx/api.py | 133 +++++++++++ src/providers/docker_sbx/exceptions.py | 10 + src/providers/docker_sbx/provider.py | 182 ++++++++++++++++ src/providers/docker_sbx/settings.py | 57 +++++ src/providers/docker_sbx/tests/__init__.py | 0 src/providers/docker_sbx/tests/test_api.py | 193 ++++++++++++++++ .../docker_sbx/tests/test_diagnose.py | 30 +++ .../docker_sbx/tests/test_provider.py | 206 ++++++++++++++++++ 16 files changed, 966 insertions(+), 16 deletions(-) create mode 100644 images/sbx/Dockerfile create mode 100755 images/sbx/entrypoint.sh create mode 100644 src/providers/docker_sbx/__init__.py create mode 100644 src/providers/docker_sbx/api.py create mode 100644 src/providers/docker_sbx/exceptions.py create mode 100644 src/providers/docker_sbx/provider.py create mode 100644 src/providers/docker_sbx/settings.py create mode 100644 src/providers/docker_sbx/tests/__init__.py create mode 100644 src/providers/docker_sbx/tests/test_api.py create mode 100644 src/providers/docker_sbx/tests/test_diagnose.py create mode 100644 src/providers/docker_sbx/tests/test_provider.py diff --git a/AGENTS.md b/AGENTS.md index e3a604f..c844a9e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,8 @@ It owns: - Host records and lifecycle state in Postgres - Inline provisioning in `POST /hosts` -- Provider VM creation and deletion (exe.dev, AWS, Hetzner, local Docker) +- Provider VM creation and deletion (exe.dev, AWS, Hetzner, Exoscale, + local Docker, Docker Sandboxes) - Tailscale auth key creation, device discovery, and cleanup - SSH host key scanning and `known_hosts` material - Account-bound exe.dev HTTP proxy resources diff --git a/docs/deploy.md b/docs/deploy.md index f6b0bba..881dbac 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -41,14 +41,18 @@ to keep the credential-holding control plane off other interfaces. ## Choose a provider -Four remote providers are supported and verified end to end: `exe` -(exe.dev), `aws` (EC2), `hetzner` (Hetzner Cloud), and `exoscale` -(Exoscale). A fifth, `docker`, -runs sandboxes as local containers and needs no external account — see -[Local sandboxes with Docker](#local-sandboxes-with-docker). `DEFAULT_HOST_PROVIDER` -selects which one serves `POST /hosts` (default `exe`). Set the matching -provider variables below. The image ships with all provider extras -installed. +| Provider | Sandboxes | Where | +| --- | --- | --- | +| `exe` | exe.dev VMs | Remote | +| `aws` | EC2 instances | Remote | +| `hetzner` | Hetzner Cloud VMs | Remote | +| `exoscale` | Exoscale VMs | Remote | +| `docker` | Containers ([Local sandboxes with Docker](#local-sandboxes-with-docker)) | Local, no external account | +| `docker-sbx` | microVMs ([Local microVMs with Docker Sandboxes](#local-microvms-with-docker-sandboxes)) | Local | + +`DEFAULT_HOST_PROVIDER` selects the provider for `POST /hosts` (default +`exe`). Set the matching provider variables below. The image contains +all provider extras. ## Local sandboxes with Docker @@ -100,6 +104,81 @@ same socket mount and socket-GID supplemental group. `DOCKER_HOST` remains available when the daemon is remote or rootless instead of exposed through `/var/run/docker.sock`. +## Local microVMs with Docker Sandboxes + +The `docker-sbx` provider runs each sandbox as a +[Docker Sandboxes](https://docs.docker.com/ai/sandboxes/) microVM. Each +microVM has its own kernel, its own filesystem, and its own Docker +daemon. The sandboxd network policy controls the egress. This provider +is local to the drukbox machine, the same as the `docker` provider. It +does not support Tailscale. + +Prepare the host fully before drukbox starts. drukbox only connects to +the host: + +1. Install Docker Engine and `docker-sbx`. Ubuntu 24.04+ with KVM is + necessary: `/dev/kvm` must exist, and the service user must be in the + `kvm` group. +2. Sign in one time with `sbx login`. Headless hosts use a device-code + flow. +3. Start the daemon: `sbx daemon start -d --policy balanced`. + +Docker documents `sbx` as a tool for the daemon owner's own user on the +host. Thus the simplest deployment runs drukbox directly on the host, as +the same user: + +```bash +uv run uvicorn api.app:app --host 127.0.0.1 --port 8780 +``` + +In this mode, no mounts and no extra variables are necessary. The CLI +finds the daemon socket automatically, and the `127.0.0.1` default for +`DOCKER_SBX_ADVERTISE_HOST` is correct. + +drukbox can also run as a container adjacent to the daemon. Docker does +not document this mode; drukbox uses the CLI's own daemon-endpoint +variable, `DOCKER_SANDBOXES_API`. Mount the daemon socket, the `sbx` +binary of the host (then the CLI version and the daemon version always +agree), the CLI auth store, and the workspace root: + +```bash +docker run --rm --network host \ + --mount type=bind,src=$HOME/.local/state/sandboxes/sandboxes/sandboxd/sandboxd.sock,dst=/run/sandboxd.sock \ + --mount type=bind,src=$(command -v sbx),dst=/usr/local/bin/sbx,readonly \ + --mount type=bind,src=$HOME/.config/com.docker.sandboxes,dst=/root/.config/com.docker.sandboxes,readonly \ + --mount type=bind,src=$HOME/.drukbox/sbx-workspaces,dst=$HOME/.drukbox/sbx-workspaces \ + --env DOCKER_SANDBOXES_API=unix:///run/sandboxd.sock \ + --env DOCKER_SBX_WORKSPACE_ROOT=$HOME/.drukbox/sbx-workspaces \ + --env DOCKER_SBX_ADVERTISE_HOST=172.17.0.1 \ + --env-file drukbox.env \ + ghcr.io/czpython/drukbox:latest +``` + +The daemon reads workspace paths on its own filesystem. Thus the +workspace mount must have the same path on the host and in the +container. The janitor and pool containers need the same mounts and +variables. `DOCKER_SBX_ADVERTISE_HOST` is the Docker bridge address +here, because the sandbox SSH ports must be open to the drukbox +container, not only to the host loopback interface. + +Only this machine can connect to the sandboxes. The key for each host +is the auth boundary. Sandboxes have no `SERVICE_LABEL` tag, because +`sbx create` has no label option. + +The template image (`DOCKER_SBX_DEFAULT_IMAGE`, default +`ghcr.io/czpython/drukbox/sbx-sandbox:latest`) must start sshd without +environment variables. `sbx create` sends none. drukbox injects the key +for each host through the exec channel after the start. Build +[images/sbx/](../images/sbx/) to change the template. The +`images/local/` entrypoint needs boot-time environment variables and +cannot start as a sandbox template. + +A sandbox creation takes approximately 20 seconds with a warm template +cache, and more than 30 seconds at the first pull. Thus a warm pool +(`POOL_SIZES`) is useful. Each sandbox gets the explicit +`DOCKER_SBX_CPUS` and `DOCKER_SBX_MEMORY` sizes. Without them, +the daemon gives one sandbox all host CPUs and half of the host memory. + ## Choose a networking mode `TAILSCALE_ENABLED=false` (default): callers reach sandboxes over the @@ -258,3 +337,20 @@ rootless daemon. Drukbox mints a per-VM ed25519 key and publishes sshd on a random `127.0.0.1` port. See [Local sandboxes with Docker](#local-sandboxes-with-docker) for the container command and the trust caveat. + +Docker Sandboxes provider: + +| Variable | Default | Purpose | +| --- | --- | --- | +| `DOCKER_SBX_DEFAULT_IMAGE` | `ghcr.io/czpython/drukbox/sbx-sandbox:latest` | Template image that contains sshd and starts without environment variables. Build `images/sbx/Dockerfile` to change it. | +| `DOCKER_SBX_SSH_USERNAME` | `root` | User in the sandbox for caller SSH access. | +| `DOCKER_SBX_BOOTSTRAP_SSH_TIMEOUT_SECONDS` | `30.0` | Time limit for the ssh-keyscan tries on a new sandbox. | +| `DOCKER_SBX_ADVERTISE_HOST` | `127.0.0.1` | Host address for the published SSH ports. Use the Docker bridge address when drukbox runs in a container. | +| `DOCKER_SBX_CPUS` | `2` | Number of CPUs for each sandbox. | +| `DOCKER_SBX_MEMORY` | `2g` | Memory for each sandbox, in binary units. | +| `DOCKER_SBX_WORKSPACE_ROOT` | `~/.drukbox/sbx-workspaces` | Directory with one temporary workspace for each sandbox. The path must be the same for drukbox and for the daemon. | + +The published image does not contain the `sbx` CLI. Mount the binary and +the auth store of the host, as +[Local microVMs with Docker Sandboxes](#local-microvms-with-docker-sandboxes) +shows. Set `DOCKER_SANDBOXES_API` to the mounted daemon socket. diff --git a/docs/networking.md b/docs/networking.md index c05c01a..5104a22 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -7,8 +7,9 @@ shaped the way it is. For turning these modes on, read ## Two modes `TAILSCALE_ENABLED` selects between two networking models. A provider -whose hosts cannot join a tailnet (docker — local containers) always -takes the external path, whatever the mode. The API response carries +whose hosts cannot join a tailnet (docker — local containers, +docker-sbx — local microVMs) always takes the external path, +whatever the mode. The API response carries both addresses; which is populated depends on the mode and the provider: diff --git a/docs/security.md b/docs/security.md index abaea9e..c97a045 100644 --- a/docs/security.md +++ b/docs/security.md @@ -54,11 +54,11 @@ no rate limiting (see [Resource exhaustion](#resource-exhaustion)). How a caller reaches a sandbox, and the tradeoffs of each path, are covered in [Networking](networking.md). The security-relevant summary: -- **Per-VM keys.** On AWS (Tailscale off) and Hetzner, drukbox mints a - fresh ed25519 keypair per VM, returns the private half **once** in - the create response, and never persists it — a later - `GET /hosts/{id}` returns `private_key: null`. The key is the auth - boundary; password auth is never enabled. +- **Per-VM keys.** On AWS (Tailscale off), Hetzner, docker, and + docker-sbx, drukbox mints a fresh ed25519 keypair per VM, returns + the private half **once** in the create response, and never persists + it — a later `GET /hosts/{id}` returns `private_key: null`. The key + is the auth boundary; password auth is never enabled. - **AWS ingress fail-open.** The managed `drukbox-managed` security group opens SSH to the detected egress `/32`, or to whatever `AWS_SSH_CIDRS` specifies. If egress detection fails and no CIDRs are diff --git a/images/sbx/Dockerfile b/images/sbx/Dockerfile new file mode 100644 index 0000000..2432af3 --- /dev/null +++ b/images/sbx/Dockerfile @@ -0,0 +1,20 @@ +# Sandbox template for the `docker-sandbox` provider: Ubuntu + sshd. +# +# docker build -t drukbox/sbx-sandbox:latest images/sbx/ +# +# This template does not need environment variables at start, because +# `sbx create` cannot send them. drukbox injects the SSH key through the exec +# channel when the sandbox runs. The images/local/ entrypoint needs boot-time +# environment variables. Thus it cannot operate as a sandbox template. +FROM ubuntu:24.04 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends openssh-server ca-certificates sudo \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /run/sshd + +COPY entrypoint.sh /usr/local/bin/drukbox-entrypoint +RUN chmod +x /usr/local/bin/drukbox-entrypoint + +EXPOSE 22 +ENTRYPOINT ["/usr/local/bin/drukbox-entrypoint"] diff --git a/images/sbx/entrypoint.sh b/images/sbx/entrypoint.sh new file mode 100755 index 0000000..331eea1 --- /dev/null +++ b/images/sbx/entrypoint.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Boot entrypoint for the drukbox Docker Sandboxes template. The script starts +# sshd. drukbox injects the SSH key through the exec channel after the start. +# sshd reads authorized_keys for each authentication. A restart is not +# necessary after the key injection. +# +# Do not make an existing authorized_keys file empty. The entrypoint runs +# again at each sandbox restart, after the key injection. +set -euo pipefail + +install -d -m 700 /root/.ssh +[ -f /root/.ssh/authorized_keys ] || install -m 600 /dev/null /root/.ssh/authorized_keys + +ssh-keygen -A + +exec /usr/sbin/sshd -D -e diff --git a/src/providers/__init__.py b/src/providers/__init__.py index 72d8fd2..4091128 100644 --- a/src/providers/__init__.py +++ b/src/providers/__init__.py @@ -1,5 +1,6 @@ import providers.aws import providers.docker +import providers.docker_sbx import providers.exe import providers.exoscale import providers.hetzner # noqa: F401 diff --git a/src/providers/docker_sbx/__init__.py b/src/providers/docker_sbx/__init__.py new file mode 100644 index 0000000..b7f9ace --- /dev/null +++ b/src/providers/docker_sbx/__init__.py @@ -0,0 +1,4 @@ +from providers.docker_sbx.provider import DockerSbxProvider +from providers.registry import register_vm_provider + +register_vm_provider(DockerSbxProvider) diff --git a/src/providers/docker_sbx/api.py b/src/providers/docker_sbx/api.py new file mode 100644 index 0000000..c4a7202 --- /dev/null +++ b/src/providers/docker_sbx/api.py @@ -0,0 +1,133 @@ +import asyncio +import json +import re + +from .exceptions import DockerSbxNotFoundError, DockerSbxTransportError + +# The first `sbx create` can pull a large template. The time limit is large +# because it must stop only a blocked daemon, not a slow pull. +_SBX_TIMEOUT_SECONDS = 600.0 + +# Only the CLI message for a missing sandbox is a not-found error. Messages +# such as "credentials not found" must stay transport errors. If not, +# delete_vm can identify a live sandbox as removed. +_SANDBOX_NOT_FOUND_RE = re.compile(r"sandbox '[^']*' not found") + + +class SbxCLI: + """Thin async wrapper for the local ``sbx`` command-line interface. + + Each method starts ``sbx`` with ``create_subprocess_exec``. Thus the + subprocess boundary stays in one place. The CLI selects the daemon: the + user socket by default, or the socket that ``DOCKER_SANDBOXES_API`` gives + when drukbox runs in a container. + """ + + async def create_sandbox( + self, + *, + name: str, + template: str, + workspace: str, + cpus: int, + memory: str, + ) -> None: + # The `shell` agent makes the sandbox start the template entrypoint, + # not an AI agent. The sizes are always explicit. Without them, the + # daemon gives one sandbox all host CPUs and half of the host memory. + await self._run( + "create", + "--name", + name, + "--template", + template, + "--cpus", + str(cpus), + "--memory", + memory, + "--quiet", + "shell", + workspace, + ) + + async def run_bootstrap(self, name: str, script: str) -> None: + # The script contains caller environment values. All processes can + # read argv through /proc. Thus the script goes through stdin, not + # argv. `bash -s` reads the program from stdin. + await self._run( + "exec", + "--interactive", + "--user", + "root", + name, + "bash", + "-s", + stdin=script, + ) + + async def publish_ssh_port(self, name: str, *, host_ip: str) -> int: + # An empty host port tells the daemon to select a free port. Thus + # sandboxes do not compete for port numbers. An IPv6 address must have + # brackets in the port specification. + spec_host = f"[{host_ip}]" if ":" in host_ip else host_ip + output = await self._run("ports", name, "--publish", f"{spec_host}::22") + # Each binding shows as ": -> 22/tcp". The output can + # have one line for each address family. The lines share the host port. + for line in output.splitlines(): + binding, arrow, target = line.partition("->") + if arrow and target.strip().startswith("22/"): + try: + return int(binding.strip().rsplit(":", 1)[1]) + except (IndexError, ValueError) as error: + raise DockerSbxTransportError( + f"sandbox {name!r} published an unparsable SSH port: {line.strip()!r}" + ) from error + raise DockerSbxTransportError(f"sandbox {name!r} published no SSH port") + + async def remove_sandbox(self, name: str) -> None: + # The --force flag stops the confirmation prompt. It also removes a + # sandbox that has an open SSH session. + await self._run("rm", "--force", name) + + async def sandbox_count(self) -> int: + output = await self._run("ls", "--json") + try: + payload = json.loads(output) + # Go writes an empty list as null. An unused daemon shows null. + return len(payload["sandboxes"] or []) + except (json.JSONDecodeError, KeyError, TypeError) as error: + raise DockerSbxTransportError( + f"sbx ls returned an unreadable sandbox list: {output.strip()!r}" + ) from error + + async def _run(self, *args: str, stdin: str | None = None) -> str: + try: + process = await asyncio.create_subprocess_exec( + "sbx", + *args, + stdin=asyncio.subprocess.PIPE if stdin else asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except OSError as error: + # The binary can be missing (FileNotFoundError) or not executable + # (PermissionError). Translate each OSError type. A raw OSError + # must not go out of the provider boundary. + raise DockerSbxTransportError(f"sbx CLI could not be started: {error}") from error + try: + stdout, stderr = await asyncio.wait_for( + process.communicate(stdin.encode() if stdin else None), + timeout=_SBX_TIMEOUT_SECONDS, + ) + except TimeoutError as error: + process.kill() + await process.wait() + raise DockerSbxTransportError( + f"sbx {args[0]} did not finish within {_SBX_TIMEOUT_SECONDS:.0f}s" + ) from error + if process.returncode != 0: + detail = stderr.decode().strip() or f"sbx {args[0]} exited {process.returncode}" + if _SANDBOX_NOT_FOUND_RE.search(detail): + raise DockerSbxNotFoundError(detail) + raise DockerSbxTransportError(detail) + return stdout.decode() diff --git a/src/providers/docker_sbx/exceptions.py b/src/providers/docker_sbx/exceptions.py new file mode 100644 index 0000000..d2ba127 --- /dev/null +++ b/src/providers/docker_sbx/exceptions.py @@ -0,0 +1,10 @@ +class DockerSbxProviderError(RuntimeError): + """Base error for the Docker Sandboxes provider.""" + + +class DockerSbxNotFoundError(DockerSbxProviderError): + """The sandbox was not found.""" + + +class DockerSbxTransportError(DockerSbxProviderError): + """The sbx command failed because of a transport problem.""" diff --git a/src/providers/docker_sbx/provider.py b/src/providers/docker_sbx/provider.py new file mode 100644 index 0000000..0ce07f9 --- /dev/null +++ b/src/providers/docker_sbx/provider.py @@ -0,0 +1,182 @@ +import contextlib +import shlex +import shutil +from pathlib import Path +from typing import ClassVar, Self + +from providers.base import VMCreateResult, VMProvider +from providers.exceptions import ( + ProviderCommandError, + ProviderNotFoundError, + ProviderTransportError, +) +from providers.ssh_keys import generate_ed25519_keypair + +from .api import SbxCLI +from .exceptions import DockerSbxNotFoundError, DockerSbxProviderError +from .settings import DockerSbxSettings + +# The /etc/environment format has one entry on each line. A NUL or a newline +# in a value can add unwanted entries. Thus these characters are not permitted +# in a value. The schema (hosts.schemas) validates the keys before this point. +_UNSAFE_ENV_VALUE_CHARS = frozenset("\x00\r\n") + + +def _bootstrap_script(*, public_key: str, env: dict[str, str], ssh_username: str) -> str: + """Make the root script that prepares SSH access to a new sandbox.""" + home = "/root" if ssh_username == "root" else f"/home/{ssh_username}" + owner = shlex.quote(ssh_username) + lines = [ + "set -euo pipefail", + f"install -d -m 700 -o {owner} -g {owner} {home}/.ssh", + f"printf '%s\\n' {shlex.quote(public_key)} > {home}/.ssh/authorized_keys", + f"chmod 600 {home}/.ssh/authorized_keys", + f"chown {owner}:{owner} {home}/.ssh/authorized_keys", + ] + # pam_env reads /etc/environment and gives the caller environment to SSH + # sessions. The sandbox runtime cannot receive environment variables at + # create time. This file is the only path. + for key, value in env.items(): + lines.append(f"printf '%s\\n' {shlex.quote(f'{key}={value}')} >> /etc/environment") + return "\n".join(lines) + "\n" + + +class DockerSbxProvider(VMProvider): + name: ClassVar[str] = "docker-sbx" + diagnose_hint: ClassVar[str] = "check_sandboxd_is_running_and_logged_in" + # A local microVM cannot connect to the tailnet: the host proxies all + # sandbox egress, and the template has no init system. These hosts keep + # the published sshd port, also on a tailnet-mode service. + supports_tailnet: ClassVar[bool] = False + + def __init__(self, api: SbxCLI, settings: DockerSbxSettings) -> None: + self.api = api + self.settings = settings + + @classmethod + def from_settings(cls) -> Self: + return cls( + SbxCLI(), + DockerSbxSettings(), # pyright: ignore[reportCallIssue] + ) + + @property + def default_image(self) -> str: + return self.settings.default_image + + @property + def bootstrap_ssh_timeout_seconds(self) -> float: + return self.settings.bootstrap_ssh_timeout_seconds + + async def create_vm( + self, + *, + name: str, + image: str, + env: dict[str, str] | None = None, + setup_script: str | None = None, + instance_type: str | None = None, + disk_gb: int | None = None, + ) -> VMCreateResult: + # The service does not send a setup script, because supports_tailnet + # is False. A script here shows a defect in the caller. Stop with an + # error. Do not start a sandbox that cannot obey the script. + if setup_script: + raise ProviderCommandError( + "docker-sbx provider runs sandboxes locally and does not " + "support Tailscale networking" + ) + + caller_env = env or {} + if unsafe_keys := sorted( + key for key, value in caller_env.items() if _UNSAFE_ENV_VALUE_CHARS.intersection(value) + ): + raise ProviderCommandError( + f"env values must not contain NUL or newline characters: {', '.join(unsafe_keys)}" + ) + + private_key, public_key = generate_ed25519_keypair() + workspace = self._workspace(name) + try: + workspace.mkdir(parents=True, exist_ok=True) + except OSError as exc: + # The workspace root can be not writable (no bind mount, or a + # read-only filesystem). The service cannot classify a raw + # OSError, thus the error becomes a provider error here. + raise ProviderTransportError(f"cannot create sandbox workspace: {exc}") from exc + + try: + await self.api.create_sandbox( + name=name, + template=image, + workspace=str(workspace), + cpus=self.settings.cpus, + memory=self.settings.memory, + ) + except DockerSbxProviderError as exc: + # The CLI can stop after the daemon makes the sandbox. Thus a + # failed create also tries to remove the sandbox. An error in this + # cleanup must not hide the first error. The janitor removes the + # sandbox by name if the cleanup fails. + with contextlib.suppress(DockerSbxProviderError): + await self.api.remove_sandbox(name) + self._remove_workspace(name) + raise ProviderTransportError(str(exc)) from exc + + try: + # The template starts sshd with an empty authorized_keys file. The + # sandbox accepts SSH only after this key is in the file. sshd + # reads the file for each authentication; a restart is not + # necessary. + script = _bootstrap_script( + public_key=public_key, + env=caller_env, + ssh_username=self.settings.ssh_username, + ) + await self.api.run_bootstrap(name, script) + ssh_port = await self.api.publish_ssh_port(name, host_ip=self.settings.advertise_host) + except DockerSbxProviderError as exc: + with contextlib.suppress(DockerSbxProviderError): + await self.api.remove_sandbox(name) + self._remove_workspace(name) + raise ProviderTransportError(str(exc)) from exc + + return VMCreateResult( + provider_id=name, + name=name, + ssh_port=ssh_port, + ssh_host=self.settings.advertise_host, + ssh_username=self.settings.ssh_username, + private_key=private_key, + ) + + async def delete_vm(self, name: str) -> None: + try: + await self.api.remove_sandbox(name) + except DockerSbxNotFoundError as exc: + # The sandbox is not there, but its workspace can be. Remove the + # workspace also. + self._remove_workspace(name) + raise ProviderNotFoundError(f"sandbox '{name}' was not found") from exc + except DockerSbxProviderError as exc: + # Keep the workspace. The sandbox can continue to operate on it. + # HostService keeps the record and can try the deletion again. + raise ProviderTransportError(str(exc)) from exc + + self._remove_workspace(name) + + async def diagnose(self) -> str: + # The sandbox list is one fast check of the CLI, the daemon + # connection, and the Docker login. + return f"sandboxd reachable, {await self.api.sandbox_count()} sandbox(es)" + + async def aclose(self) -> None: + return + + def _workspace(self, name: str) -> Path: + return self.settings.workspace_root / name + + def _remove_workspace(self, name: str) -> None: + # The workspace is temporary data for one sandbox. An error here must + # not block the host deletion. + shutil.rmtree(self._workspace(name), ignore_errors=True) diff --git a/src/providers/docker_sbx/settings.py b/src/providers/docker_sbx/settings.py new file mode 100644 index 0000000..754935a --- /dev/null +++ b/src/providers/docker_sbx/settings.py @@ -0,0 +1,57 @@ +from pathlib import Path + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class DockerSbxSettings(BaseSettings): + """Docker Sandboxes (sbx) provider configuration.""" + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + env_prefix="DOCKER_SBX_", + extra="ignore", + ) + + default_image: str = Field( + default="ghcr.io/czpython/drukbox/sbx-sandbox:latest", + description="Template image that contains sshd. Build images/sbx/ to change it.", + ) + ssh_username: str = Field( + default="root", + description="User in the sandbox for caller SSH access.", + ) + bootstrap_ssh_timeout_seconds: float = Field( + default=30.0, + description="Time limit for the ssh-keyscan tries on a new sandbox.", + ) + advertise_host: str = Field( + default="127.0.0.1", + description=( + "Host address for the published SSH ports. The drukbox process must " + "have access to this address. Use the Docker bridge address when " + "drukbox runs in a container." + ), + ) + cpus: int = Field( + default=2, + ge=1, + description="Number of CPUs for each sandbox. The daemon default is all host CPUs.", + ) + memory: str = Field( + default="2g", + description=( + "Memory for each sandbox, in binary units. The daemon default is " + "half of the host memory." + ), + ) + workspace_root: Path = Field( + default_factory=lambda: Path.home() / ".drukbox" / "sbx-workspaces", + description=( + "Directory that holds one temporary workspace for each sandbox. The " + "daemon reads workspace paths on its own filesystem. When drukbox " + "runs in a container, the path must be the same in the container " + "and on the host." + ), + ) diff --git a/src/providers/docker_sbx/tests/__init__.py b/src/providers/docker_sbx/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/providers/docker_sbx/tests/test_api.py b/src/providers/docker_sbx/tests/test_api.py new file mode 100644 index 0000000..f272dcd --- /dev/null +++ b/src/providers/docker_sbx/tests/test_api.py @@ -0,0 +1,193 @@ +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from providers.docker_sbx.api import SbxCLI +from providers.docker_sbx.exceptions import ( + DockerSbxNotFoundError, + DockerSbxTransportError, +) + + +def _process(*, returncode: int = 0, stdout: bytes = b"", stderr: bytes = b"") -> SimpleNamespace: + return SimpleNamespace( + returncode=returncode, + communicate=AsyncMock(return_value=(stdout, stderr)), + ) + + +@pytest.mark.asyncio +async def test_create_sandbox_requests_the_shell_agent_with_explicit_sizing(monkeypatch): + captured: dict = {} + + async def fake_exec(*args, **kwargs): + captured["args"] = args + return _process() + + monkeypatch.setattr("providers.docker_sbx.api.asyncio.create_subprocess_exec", fake_exec) + + await SbxCLI().create_sandbox( + name="sb-test", + template="drukbox/sbx-sandbox:latest", + workspace="/var/lib/drukbox/sbx-workspaces/sb-test", + cpus=4, + memory="8g", + ) + + args = captured["args"] + # The `shell` agent makes the sandbox start the template entrypoint. + # The workspace is the last positional argument. + assert args[-2:] == ("shell", "/var/lib/drukbox/sbx-workspaces/sb-test") + assert args[args.index("--template") + 1] == "drukbox/sbx-sandbox:latest" + # Without the size flags, the daemon gives the sandbox all host CPUs + # and half of the host memory. + assert args[args.index("--cpus") + 1] == "4" + assert args[args.index("--memory") + 1] == "8g" + + +@pytest.mark.asyncio +async def test_run_bootstrap_feeds_the_script_over_stdin_never_argv(monkeypatch): + process = _process() + captured: dict = {} + + async def fake_exec(*args, **kwargs): + captured["args"] = args + captured["stdin"] = kwargs["stdin"] + return process + + monkeypatch.setattr("providers.docker_sbx.api.asyncio.create_subprocess_exec", fake_exec) + + script = "printf '%s\\n' 'API_TOKEN=s3cr3t' >> /etc/environment\n" + await SbxCLI().run_bootstrap("sb-test", script) + + # All processes can read argv through /proc. A caller secret must not + # show in argv. The check examines each token for the substring. + assert all("s3cr3t" not in arg for arg in captured["args"]) + assert captured["stdin"] == asyncio.subprocess.PIPE + process.communicate.assert_awaited_once_with(script.encode()) + + +@pytest.mark.asyncio +async def test_publish_ssh_port_asks_for_an_ephemeral_port_and_parses_the_binding(monkeypatch): + captured: dict = {} + + async def fake_exec(*args, **kwargs): + captured["args"] = args + return _process(stdout=b"Published 172.17.0.1:49160 -> 22/tcp\n") + + monkeypatch.setattr("providers.docker_sbx.api.asyncio.create_subprocess_exec", fake_exec) + + port = await SbxCLI().publish_ssh_port("sb-test", host_ip="172.17.0.1") + + assert port == 49160 + # An empty host port tells the daemon to select a free port. + assert "172.17.0.1::22" in captured["args"] + + +@pytest.mark.asyncio +async def test_publish_ssh_port_picks_the_ssh_binding_out_of_a_multi_line_listing(monkeypatch): + captured: dict = {} + + async def fake_exec(*args, **kwargs): + captured["args"] = args + return _process( + stdout=(b"Published 172.17.0.1:8080 -> 80/tcp\nPublished [::1]:49161 -> 22/tcp\n") + ) + + monkeypatch.setattr("providers.docker_sbx.api.asyncio.create_subprocess_exec", fake_exec) + + assert await SbxCLI().publish_ssh_port("sb-test", host_ip="::1") == 49161 + # An IPv6 address must have brackets in the port specification. + assert "[::1]::22" in captured["args"] + + +@pytest.mark.asyncio +async def test_publish_ssh_port_raises_when_nothing_was_published(monkeypatch): + create = AsyncMock(return_value=_process(stdout=b"\n")) + monkeypatch.setattr("providers.docker_sbx.api.asyncio.create_subprocess_exec", create) + + with pytest.raises(DockerSbxTransportError, match="no SSH port"): + await SbxCLI().publish_ssh_port("sb-test", host_ip="172.17.0.1") + + +@pytest.mark.asyncio +async def test_publish_ssh_port_raises_on_unparsable_output(monkeypatch): + create = AsyncMock(return_value=_process(stdout=b"Published 172.17.0.1:notaport -> 22/tcp\n")) + monkeypatch.setattr("providers.docker_sbx.api.asyncio.create_subprocess_exec", create) + + with pytest.raises(DockerSbxTransportError, match="unparsable"): + await SbxCLI().publish_ssh_port("sb-test", host_ip="172.17.0.1") + + +@pytest.mark.asyncio +async def test_sandbox_count_reads_the_listing(monkeypatch): + listing = b'{"sandboxes": [{"name": "sb-a"}, {"name": "sb-b"}]}' + create = AsyncMock(return_value=_process(stdout=listing)) + monkeypatch.setattr("providers.docker_sbx.api.asyncio.create_subprocess_exec", create) + + assert await SbxCLI().sandbox_count() == 2 + + +@pytest.mark.asyncio +async def test_sandbox_count_treats_a_null_list_as_empty(monkeypatch): + # Go writes an empty list as null. An unused daemon shows null. + create = AsyncMock(return_value=_process(stdout=b'{"sandboxes": null}')) + monkeypatch.setattr("providers.docker_sbx.api.asyncio.create_subprocess_exec", create) + + assert await SbxCLI().sandbox_count() == 0 + + +@pytest.mark.asyncio +async def test_remove_sandbox_forces_removal_of_an_attached_sandbox(monkeypatch): + captured: dict = {} + + async def fake_exec(*args, **kwargs): + captured["args"] = args + return _process() + + monkeypatch.setattr("providers.docker_sbx.api.asyncio.create_subprocess_exec", fake_exec) + + await SbxCLI().remove_sandbox("sb-test") + + # Without --force, the CLI asks for confirmation. It also refuses a + # sandbox that has an open SSH session. + assert captured["args"] == ("sbx", "rm", "--force", "sb-test") + + +@pytest.mark.asyncio +async def test_missing_sandbox_maps_to_not_found(monkeypatch): + create = AsyncMock( + return_value=_process( + returncode=1, + stderr=b"Error: sandbox 'sb-test' not found (run 'sbx ls' to see your sandboxes)", + ) + ) + monkeypatch.setattr("providers.docker_sbx.api.asyncio.create_subprocess_exec", create) + + with pytest.raises(DockerSbxNotFoundError): + await SbxCLI().remove_sandbox("sb-test") + + +@pytest.mark.asyncio +async def test_stderr_merely_containing_not_found_stays_a_transport_error(monkeypatch): + # Only the CLI message for a missing sandbox can map to not-found. If an + # auth error maps to not-found, delete_vm removes the record and the + # workspace of a live sandbox. + create = AsyncMock( + return_value=_process(returncode=1, stderr=b"Error: credentials not found; run 'sbx login'") + ) + monkeypatch.setattr("providers.docker_sbx.api.asyncio.create_subprocess_exec", create) + + with pytest.raises(DockerSbxTransportError): + await SbxCLI().remove_sandbox("sb-test") + + +@pytest.mark.asyncio +async def test_missing_sbx_binary_maps_to_transport_error(monkeypatch): + create = AsyncMock(side_effect=FileNotFoundError("sbx")) + monkeypatch.setattr("providers.docker_sbx.api.asyncio.create_subprocess_exec", create) + + with pytest.raises(DockerSbxTransportError, match="could not be started"): + await SbxCLI().sandbox_count() diff --git a/src/providers/docker_sbx/tests/test_diagnose.py b/src/providers/docker_sbx/tests/test_diagnose.py new file mode 100644 index 0000000..7ad772c --- /dev/null +++ b/src/providers/docker_sbx/tests/test_diagnose.py @@ -0,0 +1,30 @@ +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from providers.docker_sbx.exceptions import DockerSbxTransportError +from providers.docker_sbx.provider import DockerSbxProvider +from providers.docker_sbx.settings import DockerSbxSettings + + +def _provider(api: MagicMock) -> DockerSbxProvider: + return DockerSbxProvider(api, DockerSbxSettings()) + + +@pytest.mark.asyncio +async def test_diagnose_reports_daemon_reachability(): + api = MagicMock() + api.sandbox_count = AsyncMock(return_value=2) + + assert await _provider(api).diagnose() == "sandboxd reachable, 2 sandbox(es)" + + +@pytest.mark.asyncio +async def test_diagnose_raises_so_doctor_can_classify_the_failure(): + api = MagicMock() + api.sandbox_count = AsyncMock( + side_effect=DockerSbxTransportError("Not authenticated to Docker") + ) + + with pytest.raises(DockerSbxTransportError): + await _provider(api).diagnose() diff --git a/src/providers/docker_sbx/tests/test_provider.py b/src/providers/docker_sbx/tests/test_provider.py new file mode 100644 index 0000000..7fdecde --- /dev/null +++ b/src/providers/docker_sbx/tests/test_provider.py @@ -0,0 +1,206 @@ +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from providers.docker_sbx.exceptions import ( + DockerSbxNotFoundError, + DockerSbxTransportError, +) +from providers.docker_sbx.provider import DockerSbxProvider +from providers.docker_sbx.settings import DockerSbxSettings +from providers.exceptions import ( + ProviderCommandError, + ProviderNotFoundError, + ProviderTransportError, +) + + +def _settings(workspace_root: Path, **overrides: Any) -> DockerSbxSettings: + return DockerSbxSettings(workspace_root=workspace_root, **overrides) + + +def _api_mock() -> MagicMock: + api = MagicMock() + api.create_sandbox = AsyncMock() + api.run_bootstrap = AsyncMock() + api.publish_ssh_port = AsyncMock(return_value=49160) + api.remove_sandbox = AsyncMock() + api.sandbox_count = AsyncMock(return_value=2) + return api + + +@pytest.mark.asyncio +async def test_create_vm_creates_a_sized_sandbox_and_returns_published_coords(tmp_path): + api = _api_mock() + provider = DockerSbxProvider( + api, _settings(tmp_path, advertise_host="172.17.0.1", cpus=4, memory="8g") + ) + + result = await provider.create_vm(name="sb-test", image="drukbox/sbx-sandbox:latest", env={}) + + create_kwargs = api.create_sandbox.await_args.kwargs + assert create_kwargs["name"] == "sb-test" + assert create_kwargs["template"] == "drukbox/sbx-sandbox:latest" + assert create_kwargs["cpus"] == 4 + assert create_kwargs["memory"] == "8g" + # The daemon reads the workspace path on its own filesystem. The + # directory must exist before the sandbox creation. + assert create_kwargs["workspace"] == str(tmp_path / "sb-test") + assert (tmp_path / "sb-test").is_dir() + + api.publish_ssh_port.assert_awaited_once_with("sb-test", host_ip="172.17.0.1") + assert result.ssh_host == "172.17.0.1" + assert result.ssh_port == 49160 + assert result.ssh_username == "root" + assert result.private_key is not None + assert "-----BEGIN OPENSSH PRIVATE KEY-----" in result.private_key + + +@pytest.mark.asyncio +async def test_create_vm_bootstrap_installs_the_public_key_and_caller_env(tmp_path): + api = _api_mock() + provider = DockerSbxProvider(api, _settings(tmp_path)) + + await provider.create_vm(name="sb-test", image="img", env={"API_TOKEN": "s3cr3t"}) + + name, script = api.run_bootstrap.await_args.args + assert name == "sb-test" + assert "ssh-ed25519 " in script + assert "/root/.ssh/authorized_keys" in script + # The caller environment goes to SSH sessions through pam_env. + assert "API_TOKEN=s3cr3t" in script + assert "/etc/environment" in script + + +@pytest.mark.asyncio +async def test_create_vm_installs_the_key_for_the_configured_ssh_user(tmp_path): + api = _api_mock() + provider = DockerSbxProvider(api, _settings(tmp_path, ssh_username="dev")) + + result = await provider.create_vm(name="sb-test", image="img", env={}) + + _, script = api.run_bootstrap.await_args.args + # The key must go where sshd examines it for the given user. If not, + # the host becomes ACTIVE but refuses each login. + assert "/home/dev/.ssh/authorized_keys" in script + assert result.ssh_username == "dev" + + +@pytest.mark.asyncio +async def test_create_vm_rejects_setup_script_because_tailscale_is_unsupported(tmp_path): + api = _api_mock() + provider = DockerSbxProvider(api, _settings(tmp_path)) + + # The service does not send a script, because supports_tailnet is + # False. This guard finds a defect in the caller. + with pytest.raises(ProviderCommandError, match="Tailscale"): + await provider.create_vm(name="sb-test", image="img", env={}, setup_script="#!/bin/sh\n") + api.create_sandbox.assert_not_called() + + +@pytest.mark.asyncio +async def test_create_vm_rejects_env_values_that_would_forge_environment_entries(tmp_path): + api = _api_mock() + provider = DockerSbxProvider(api, _settings(tmp_path)) + + with pytest.raises(ProviderCommandError, match="NUL or newline"): + await provider.create_vm(name="sb-test", image="img", env={"EVIL": "value\nINJECTED=x"}) + api.create_sandbox.assert_not_called() + assert not (tmp_path / "sb-test").exists() + + +@pytest.mark.asyncio +async def test_create_vm_cleans_up_when_the_sandbox_cannot_be_created(tmp_path): + api = _api_mock() + api.create_sandbox.side_effect = DockerSbxTransportError("daemon unavailable") + provider = DockerSbxProvider(api, _settings(tmp_path)) + + with pytest.raises(ProviderTransportError): + await provider.create_vm(name="sb-test", image="img", env={}) + # The CLI can stop after the daemon makes the sandbox. A failed create + # also tries the removal. + api.remove_sandbox.assert_awaited_once_with("sb-test") + assert not (tmp_path / "sb-test").exists() + + +@pytest.mark.asyncio +async def test_create_vm_translates_an_unwritable_workspace_root(tmp_path): + api = _api_mock() + # A file at the workspace root location makes mkdir fail. The error is + # the same OSError type as for a missing bind mount. + blocked_root = tmp_path / "blocked" + blocked_root.touch() + provider = DockerSbxProvider(api, _settings(blocked_root)) + + with pytest.raises(ProviderTransportError, match="workspace"): + await provider.create_vm(name="sb-test", image="img", env={}) + api.create_sandbox.assert_not_called() + + +@pytest.mark.asyncio +async def test_create_vm_tears_down_sandbox_and_workspace_when_publishing_fails(tmp_path): + api = _api_mock() + api.publish_ssh_port.side_effect = DockerSbxTransportError("no port") + provider = DockerSbxProvider(api, _settings(tmp_path)) + + with pytest.raises(ProviderTransportError): + await provider.create_vm(name="sb-test", image="img", env={}) + api.remove_sandbox.assert_awaited_once_with("sb-test") + assert not (tmp_path / "sb-test").exists() + + +@pytest.mark.asyncio +async def test_create_vm_publish_error_survives_failed_cleanup(tmp_path): + # The cleanup of the sandbox can also fail. The caller must get the + # first error, not the cleanup error. + api = _api_mock() + api.publish_ssh_port.side_effect = DockerSbxTransportError("no port") + api.remove_sandbox.side_effect = DockerSbxTransportError("cleanup failed") + provider = DockerSbxProvider(api, _settings(tmp_path)) + + with pytest.raises(ProviderTransportError, match="no port"): + await provider.create_vm(name="sb-test", image="img", env={}) + api.remove_sandbox.assert_awaited_once_with("sb-test") + + +@pytest.mark.asyncio +async def test_delete_vm_removes_the_sandbox_and_its_workspace(tmp_path): + api = _api_mock() + workspace = tmp_path / "sb-test" + workspace.mkdir(parents=True) + provider = DockerSbxProvider(api, _settings(tmp_path)) + + await provider.delete_vm("sb-test") + + api.remove_sandbox.assert_awaited_once_with("sb-test") + assert not workspace.exists() + + +@pytest.mark.asyncio +async def test_delete_vm_drops_the_workspace_of_a_sandbox_that_never_existed(tmp_path): + api = _api_mock() + api.remove_sandbox.side_effect = DockerSbxNotFoundError("sandbox 'sb-test' not found") + workspace = tmp_path / "sb-test" + workspace.mkdir(parents=True) + provider = DockerSbxProvider(api, _settings(tmp_path)) + + with pytest.raises(ProviderNotFoundError): + await provider.delete_vm("sb-test") + assert not workspace.exists() + + +@pytest.mark.asyncio +async def test_delete_vm_keeps_the_workspace_when_teardown_fails(tmp_path): + # The sandbox may still be running on the workspace, and HostService keeps + # the row so deletion can be retried. + api = _api_mock() + api.remove_sandbox.side_effect = DockerSbxTransportError("daemon unavailable") + workspace = tmp_path / "sb-test" + workspace.mkdir(parents=True) + provider = DockerSbxProvider(api, _settings(tmp_path)) + + with pytest.raises(ProviderTransportError): + await provider.delete_vm("sb-test") + assert workspace.is_dir()