diff --git a/.changeset/docker-worker-control-plane-trpc.md b/.changeset/docker-worker-control-plane-trpc.md new file mode 100644 index 000000000..ace1d7691 --- /dev/null +++ b/.changeset/docker-worker-control-plane-trpc.md @@ -0,0 +1,5 @@ +--- +'@roomote/web': patch +--- + +Fix self-hosted Docker tasks stuck at Booting environment when workers cannot reach a public TRPC_URL from the task network. diff --git a/apps/controller/src/compute-providers/__tests__/spawn-docker-worker.test.ts b/apps/controller/src/compute-providers/__tests__/spawn-docker-worker.test.ts index a345a889c..3c2ff4aeb 100644 --- a/apps/controller/src/compute-providers/__tests__/spawn-docker-worker.test.ts +++ b/apps/controller/src/compute-providers/__tests__/spawn-docker-worker.test.ts @@ -6,12 +6,17 @@ import { type DockerCommand, } from '../docker-sandbox-security'; import { + assertDockerWorkerLaunchEnv, buildDockerSandboxServerUrl, + buildDockerWorkerExecEnvArgs, + DOCKER_CONTROL_PLANE_TRPC_URL, DOCKER_SPAWN_TIMEOUT_MS, getDockerWorkerCommand, preflightDockerSpawn, resolveDockerSpawnCleanupMode, resolveDockerWorkerOwnershipTargetFromLookup, + resolveDockerWorkerTrpcUrl, + sanitizeDockerWorkerTrpcUrlForLog, resumeDockerTaskDaemon, shouldPreserveFailedDockerWorkerContainer, shouldRetryDockerWorkerWithoutDiskLimit, @@ -229,6 +234,89 @@ describe('toContainerReachableUrl', () => { }); }); +describe('resolveDockerWorkerTrpcUrl', () => { + it('uses the in-network API alias when a control network is configured', () => { + expect( + resolveDockerWorkerTrpcUrl({ + trpcUrl: 'https://roomote.example.com/_roomote-api', + controlNetwork: 'roomote_worker', + }), + ).toBe(DOCKER_CONTROL_PLANE_TRPC_URL); + }); + + it('strips public-proxy path prefixes from already-in-network api hosts', () => { + expect( + resolveDockerWorkerTrpcUrl({ + trpcUrl: 'http://api:3001/_roomote-api', + controlNetwork: 'roomote_worker', + }), + ).toBe(DOCKER_CONTROL_PLANE_TRPC_URL); + }); + + it('rewrites localhost TRPC URLs for non-control-network Docker workers', () => { + expect( + resolveDockerWorkerTrpcUrl({ + trpcUrl: 'http://localhost:13001', + }), + ).toBe('http://host.docker.internal:13001'); + }); +}); + +describe('sanitizeDockerWorkerTrpcUrlForLog', () => { + it('strips URL userinfo credentials and query/hash before logging', () => { + expect( + sanitizeDockerWorkerTrpcUrlForLog( + 'https://worker:s3cret@api.example.com:8443/_roomote-api?token=abc#frag', + ), + ).toBe('https://api.example.com:8443/_roomote-api'); + }); + + it('keeps plain in-network control-plane URLs unchanged', () => { + expect( + sanitizeDockerWorkerTrpcUrlForLog(DOCKER_CONTROL_PLANE_TRPC_URL), + ).toBe(DOCKER_CONTROL_PLANE_TRPC_URL); + }); + + it('returns a stable placeholder for unparseable values', () => { + expect(sanitizeDockerWorkerTrpcUrlForLog('not a url')).toBe( + '[invalid-trpc-url]', + ); + }); +}); + +describe('assertDockerWorkerLaunchEnv', () => { + it('rejects missing required launch env values', () => { + expect(() => + assertDockerWorkerLaunchEnv({ + AUTH_TOKEN: '', + TRPC_URL: 'http://api:3001', + R_APP_URL: 'https://roomote.example.com', + }), + ).toThrow('AUTH_TOKEN'); + }); + + it('accepts a complete launch env', () => { + expect(() => + assertDockerWorkerLaunchEnv({ + AUTH_TOKEN: 'token', + TRPC_URL: 'http://api:3001', + R_APP_URL: 'https://roomote.example.com', + }), + ).not.toThrow(); + }); +}); + +describe('buildDockerWorkerExecEnvArgs', () => { + it('flattens env into docker exec -e KEY=value pairs', () => { + expect( + buildDockerWorkerExecEnvArgs({ + AUTH_TOKEN: 'token', + TRPC_URL: 'http://api:3001', + }), + ).toEqual(['-e', 'AUTH_TOKEN=token', '-e', 'TRPC_URL=http://api:3001']); + }); +}); + describe('shouldAutoRemoveDockerWorkerContainer', () => { it('preserves containers in every environment for bounded standby retention', () => { expect(shouldAutoRemoveDockerWorkerContainer('production')).toBe(false); diff --git a/apps/controller/src/compute-providers/spawn-docker-worker.ts b/apps/controller/src/compute-providers/spawn-docker-worker.ts index 4edfc8fba..d1e8808f9 100644 --- a/apps/controller/src/compute-providers/spawn-docker-worker.ts +++ b/apps/controller/src/compute-providers/spawn-docker-worker.ts @@ -468,6 +468,10 @@ export async function spawnDockerWorker( throwIfSpawnAborted(); + const workerTrpcUrl = resolveDockerWorkerTrpcUrl({ + trpcUrl: process.env.TRPC_URL ?? Env.TRPC_URL, + controlNetwork, + }); const workerEnv = buildDockerWorkerEnv({ authToken, sandboxExpiresAtMs: Date.now() + config.dockerTimeoutMs, @@ -476,7 +480,7 @@ export async function spawnDockerWorker( image: config.image, extraEnv: { SANDBOX_TIMEOUT_MS: String(config.dockerTimeoutMs), - TRPC_URL: toContainerReachableUrl(process.env.TRPC_URL ?? Env.TRPC_URL), + TRPC_URL: workerTrpcUrl, // Mock-Slack parity: worker-side SlackNotifier calls (question blocks, // reactions) must reach the same mock harness the API uses. ...(process.env.SLACK_API_BASE_URL && { @@ -511,14 +515,17 @@ export async function spawnDockerWorker( }, }); + assertDockerWorkerLaunchEnv(workerEnv); + const workerCommand = getDockerWorkerCommand(taskRun.payloadKind); + // Inject via `docker exec -e` so the worker process gets AUTH_TOKEN / + // TRPC_URL without baking secrets into `docker inspect` Config.Env. + // Operators checking a later plain `docker exec` shell will not see these + // keys — that is expected and does not mean spawn skipped injection. await runDocker([ 'exec', '-d', - ...Object.entries(workerEnv).flatMap(([key, value]) => [ - '-e', - `${key}=${value}`, - ]), + ...buildDockerWorkerExecEnvArgs(workerEnv), containerName, 'bash', '-lc', @@ -529,7 +536,12 @@ export async function spawnDockerWorker( console.log( `[spawnDockerWorker] Docker worker launched for task run #${taskRun.id} ${JSON.stringify( - { containerName, containerId }, + { + containerName, + containerId, + trpcUrl: sanitizeDockerWorkerTrpcUrlForLog(workerTrpcUrl), + envKeys: Object.keys(workerEnv).sort(), + }, )}`, ); @@ -961,6 +973,77 @@ export function getDockerWorkerCommand( return payloadKind === TaskPayloadKind.SnapshotResume ? 'resume' : 'run'; } +/** + * Docker Compose self-host/prod attaches the `api` service to each task + * network. When a control network is configured, egress policy blackholes the + * docker bridge gateway so sandboxes cannot hairpin through the public edge. + * Workers must call the in-network API alias directly (no `/_roomote-api` + * prefix — that path only exists on the public reverse proxy). + */ +export const DOCKER_CONTROL_PLANE_TRPC_URL = 'http://api:3001'; + +export function resolveDockerWorkerTrpcUrl(params: { + trpcUrl: string | undefined; + controlNetwork?: string; +}): string { + // Control-plane isolation only trusts the `api` service on the task + // network at the app origin/root. Public reverse-proxy path prefixes such + // as `/_roomote-api` must not be preserved even when hostname is already + // `api`. + if (params.controlNetwork?.trim()) { + return DOCKER_CONTROL_PLANE_TRPC_URL; + } + + return toContainerReachableUrl(params.trpcUrl); +} + +/** + * Log-safe view of the worker TRPC URL: drops userinfo credentials and + * query/hash so operator logs never capture embedded secrets while still + * showing host + path for spawn diagnosis. + */ +export function sanitizeDockerWorkerTrpcUrlForLog(trpcUrl: string): string { + try { + const url = new URL(trpcUrl); + url.username = ''; + url.password = ''; + url.search = ''; + url.hash = ''; + return trimTrailingSlash(url.toString()); + } catch { + return '[invalid-trpc-url]'; + } +} + +const REQUIRED_DOCKER_WORKER_LAUNCH_ENV_KEYS = [ + 'AUTH_TOKEN', + 'TRPC_URL', + 'R_APP_URL', +] as const; + +export function assertDockerWorkerLaunchEnv( + workerEnv: Record, +): void { + const missing = REQUIRED_DOCKER_WORKER_LAUNCH_ENV_KEYS.filter( + (key) => !workerEnv[key]?.trim(), + ); + + if (missing.length > 0) { + throw new Error( + `Docker worker launch env missing required value(s): ${missing.join(', ')}`, + ); + } +} + +export function buildDockerWorkerExecEnvArgs( + workerEnv: Record, +): string[] { + return Object.entries(workerEnv).flatMap(([key, value]) => [ + '-e', + `${key}=${value}`, + ]); +} + export function toContainerReachableUrl(value: string | undefined): string { if (!value) { return '';