diff --git a/apps/controller/src/BaseController.ts b/apps/controller/src/BaseController.ts index fc642cd55..2e4bc2bac 100644 --- a/apps/controller/src/BaseController.ts +++ b/apps/controller/src/BaseController.ts @@ -494,7 +494,7 @@ export abstract class BaseController { taskRun: TaskRun, error: unknown, ): Promise { - const errorMessage = error instanceof Error ? error.message : String(error); + const errorMessage = formatSpawnErrorMessage(error); captureControllerException(error, { runId: taskRun.id, @@ -544,3 +544,35 @@ export abstract class BaseController { throw error; } } + +/** + * Build the user-visible spawn failure string. Prefer stderr/stdout from + * child-process errors so daemon causes (missing image, pull denied) are not + * buried under "Command failed: docker run …" argv dumps. + */ +export function formatSpawnErrorMessage(error: unknown): string { + if (!(error instanceof Error)) { + return String(error); + } + + const errorRecord = error as Error & { + stderr?: unknown; + stdout?: unknown; + }; + const stderr = + typeof errorRecord.stderr === 'string' ? errorRecord.stderr.trim() : ''; + const stdout = + typeof errorRecord.stdout === 'string' ? errorRecord.stdout.trim() : ''; + const message = error.message.trim(); + + const hasStructuredOutput = Boolean(stderr || stdout); + const messageAlreadyIncludesOutput = + (stderr && message.includes(stderr)) || + (stdout && message.includes(stdout)); + + if (!hasStructuredOutput || messageAlreadyIncludesOutput) { + return message || stderr || stdout || 'Unknown spawn error'; + } + + return [message, stderr, stdout].filter(Boolean).join('\n'); +} diff --git a/apps/controller/src/__tests__/BaseController.test.ts b/apps/controller/src/__tests__/BaseController.test.ts index eeb8099f2..36ca67e0d 100644 --- a/apps/controller/src/__tests__/BaseController.test.ts +++ b/apps/controller/src/__tests__/BaseController.test.ts @@ -183,6 +183,19 @@ function resetControllerMocks() { // ── Tests ──────────────────────────────────────────────────────────────────── +describe('formatSpawnErrorMessage', () => { + it('prefers child-process stderr when the message does not already include it', async () => { + const { formatSpawnErrorMessage } = await import('../BaseController'); + const error = Object.assign(new Error('Command failed: docker run -d'), { + stderr: "Unable to find image 'roomote-worker:local' locally\n", + }); + + expect(formatSpawnErrorMessage(error)).toBe( + "Command failed: docker run -d\nUnable to find image 'roomote-worker:local' locally", + ); + }); +}); + describe('BaseController.handleSpawnTaskRunError', () => { let controller: TestController; const originalCwd = process.cwd(); @@ -210,6 +223,26 @@ describe('BaseController.handleSpawnTaskRunError', () => { delete process.env.USE_WORKER_RELEASE; }); + it('includes child-process stderr in finishRun when spawn fails with exec output', async () => { + const job = makeTaskRun({ id: 43 }); + const error = Object.assign(new Error('Command failed: docker run -d'), { + stderr: + "Unable to find image 'roomote-worker:local' locally\ndocker: Error response from daemon: pull access denied for roomote-worker\n", + }); + + await expect( + controller.testHandleSpawnTaskRunError(job, error), + ).rejects.toThrow(/Command failed: docker run -d/); + + expect(mockFinishRun).toHaveBeenCalledWith({ + id: 43, + status: RunStatus.Failed, + error: expect.stringContaining( + "Unable to find image 'roomote-worker:local'", + ), + }); + }); + it('calls finishRun with Failed status and error message', async () => { const job = makeTaskRun({ id: 42 }); const error = new Error('Machine unavailable'); diff --git a/apps/controller/src/compute-providers/__tests__/docker-sandbox-security.test.ts b/apps/controller/src/compute-providers/__tests__/docker-sandbox-security.test.ts index 6d3dd13af..95eb77d87 100644 --- a/apps/controller/src/compute-providers/__tests__/docker-sandbox-security.test.ts +++ b/apps/controller/src/compute-providers/__tests__/docker-sandbox-security.test.ts @@ -5,6 +5,7 @@ import { buildDockerTaskDaemonResourceArgs, buildDockerWorkerResourceArgs, cleanupStaleDockerSandboxes, + formatDockerCommandError, getDockerTaskNetworkName, isUnsupportedDockerDiskLimitError, prepareDockerTaskNetwork, @@ -120,6 +121,37 @@ describe('isUnsupportedDockerDiskLimitError', () => { }); }); +describe('formatDockerCommandError', () => { + it('prefers docker stderr over the full argv command-failed dump', () => { + const original = Object.assign(new Error('Command failed: docker run -d'), { + stderr: + "Unable to find image 'roomote-worker:local' locally\ndocker: Error response from daemon: pull access denied for roomote-worker, repository does not exist or may require 'docker login'\n", + stdout: '', + code: 125, + }); + + const formatted = formatDockerCommandError( + ['run', '-d', '--name', 'roomote-worker-42', 'roomote-worker:local'], + original, + ); + + expect(formatted.message).toContain( + 'Docker command failed (docker run roomote-worker:local):', + ); + expect(formatted.message).toContain( + "Unable to find image 'roomote-worker:local' locally", + ); + expect(formatted.message).toContain( + 'pull access denied for roomote-worker', + ); + expect(formatted.message).not.toContain('--name'); + expect((formatted as { stderr?: string }).stderr).toContain( + 'pull access denied', + ); + expect(isUnsupportedDockerDiskLimitError(formatted)).toBe(false); + }); +}); + describe('prepareDockerTaskNetwork', () => { it('creates a dedicated internal network for a no-egress task', async () => { const runDocker = vi.fn().mockResolvedValue(''); diff --git a/apps/controller/src/compute-providers/docker-sandbox-security.ts b/apps/controller/src/compute-providers/docker-sandbox-security.ts index 99a020dec..21188263d 100644 --- a/apps/controller/src/compute-providers/docker-sandbox-security.ts +++ b/apps/controller/src/compute-providers/docker-sandbox-security.ts @@ -91,8 +91,92 @@ export async function docker( return ''; } - throw error; + throw formatDockerCommandError(args, error); + } +} + +/** + * Prefer docker daemon stderr/stdout over Node's default "Command failed: + * docker run ..." dump so spawn failures keep the actionable cause (image + * missing, pull denied, etc.) without burying it under the full argv list. + */ +export function formatDockerCommandError( + args: string[], + error: unknown, +): Error { + const stderr = getStringErrorField(error, 'stderr'); + const stdout = getStringErrorField(error, 'stdout'); + const originalMessage = + error instanceof Error ? error.message.trim() : String(error).trim(); + const details = + [stderr, stdout].filter(Boolean).join('\n').trim() || + stripDockerCommandFailedPrefix(originalMessage) || + originalMessage || + 'unknown docker error'; + const commandSummary = summarizeDockerCommand(args); + const message = `Docker command failed (${commandSummary}):\n${details}`; + + if (error instanceof Error) { + const wrapped = new Error(message, { cause: error }); + wrapped.name = error.name; + // Preserve Node exec stderr/stdout so callers that inspect those fields + // (disk-limit detection, not-found checks) keep working. + Object.assign(wrapped, { + stderr: getRawErrorField(error, 'stderr'), + stdout: getRawErrorField(error, 'stdout'), + code: getRawErrorField(error, 'code'), + signal: getRawErrorField(error, 'signal'), + killed: getRawErrorField(error, 'killed'), + cmd: getRawErrorField(error, 'cmd'), + }); + return wrapped; + } + + return new Error(message); +} + +function getStringErrorField(error: unknown, field: string): string { + const value = getRawErrorField(error, field); + return typeof value === 'string' ? value.trim() : ''; +} + +function getRawErrorField(error: unknown, field: string): unknown { + if (!error || typeof error !== 'object') { + return undefined; + } + + return (error as Record)[field]; +} + +function stripDockerCommandFailedPrefix(message: string): string { + return message + .replace(/^Command failed:\s*docker(?:\s+[^\n]+)?\n?/i, '') + .trim(); +} + +function summarizeDockerCommand(args: string[]): string { + const verb = args[0]?.trim(); + const nonOptionArgs = args.filter( + (arg) => arg.length > 0 && !arg.startsWith('-'), + ); + // Prefer an image-like arg (contains ':' or '/') over container names. + const imageLike = [...nonOptionArgs] + .reverse() + .find((arg) => arg.includes(':') || arg.includes('/')); + + if (verb && imageLike) { + return `docker ${verb} ${imageLike}`; + } + + if (verb && nonOptionArgs[1]) { + return `docker ${verb} ${nonOptionArgs[1]}`; } + + if (verb) { + return `docker ${verb}`; + } + + return 'docker'; } export function getDockerTaskNetworkName(taskRunId: number): string { diff --git a/apps/web/src/app/(sandbox)/task/[taskId]/startup/Startup.stories.tsx b/apps/web/src/app/(sandbox)/task/[taskId]/startup/Startup.stories.tsx index 15cfbdbf8..401782063 100644 --- a/apps/web/src/app/(sandbox)/task/[taskId]/startup/Startup.stories.tsx +++ b/apps/web/src/app/(sandbox)/task/[taskId]/startup/Startup.stories.tsx @@ -123,6 +123,38 @@ export const StepDequeued: Story = { ), }; +/** Active boot step that has been waiting long enough to show elapsed time. */ +export const StepDequeuedStillBooting: Story = { + name: 'Step – Dequeued still booting', + render: () => ( + + ), +}; + +/** Failed boot with a missing/unpullable Docker worker image error. */ +export const FailedMissingWorkerImage: Story = { + name: 'Failed – Missing worker image', + render: () => ( + undefined, + }} + /> + ), +}; + /** A completed step (no shimmer animation). */ export const StepCompletedProcessing: Story = { name: 'Step – Processing (completed)', diff --git a/apps/web/src/app/(sandbox)/task/[taskId]/startup/Startup.tsx b/apps/web/src/app/(sandbox)/task/[taskId]/startup/Startup.tsx index fdf0e278b..87bd5a04c 100644 --- a/apps/web/src/app/(sandbox)/task/[taskId]/startup/Startup.tsx +++ b/apps/web/src/app/(sandbox)/task/[taskId]/startup/Startup.tsx @@ -152,8 +152,15 @@ const StartupInner = ({ const restoreSnapshot = useRestoreTaskRunSnapshot(); const retryFailedStart = useRetryFailedTaskStart(); - const { steps, error, showLogs, sandboxLogs, logsConnected, logsError } = - useStartupProgress({ runId, initialTaskRun, onStatusChange }); + const { + steps, + error, + showLogs, + sandboxLogs, + logsConnected, + logsError, + activeStepSinceMs, + } = useStartupProgress({ runId, initialTaskRun, onStatusChange }); return ( { ).toHaveAttribute('href', 'https://example.com/shot.png'); }); + it('shows actionable missing worker image copy on failed environment starts', () => { + render( + , + ); + + expect( + screen.getByText('There was an error starting this environment:'), + ).toBeInTheDocument(); + expect( + screen.getByText( + "Roomote couldn't start because the worker image `roomote-worker:local` is missing or can't be pulled. Build or pull that image on the host (for local Docker, run the worker image build), or set DOCKER_WORKER_IMAGE to an image the host can access.", + ), + ).toBeInTheDocument(); + expect(screen.queryByText(/docker run/i)).not.toBeInTheDocument(); + }); + + it('shows still-booting elapsed time after the active step has been waiting', () => { + vi.useFakeTimers(); + const startedAt = Date.now() - 20_000; + + render( + , + ); + + expect(screen.getByText('Booting environment')).toBeInTheDocument(); + expect( + screen.getByText(/Still booting… \(elapsed: 20s\)/), + ).toBeInTheDocument(); + + vi.useRealTimers(); + }); + it('uses a flex-bounded scroll container when startup content exceeds the available height', () => { const { container } = render( { if ( step.completed && @@ -97,21 +103,83 @@ const getStepMessage = (step: StartupStep) => { })(); }; -const StartupMessage = ({ step, isActive }: StartupMessageProps) => { +function formatBootElapsed(elapsedMs: number): string { + const totalSeconds = Math.max(1, Math.floor(elapsedMs / 1000)); + + if (totalSeconds < 60) { + return `${totalSeconds}s`; + } + + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + + if (minutes < 60) { + return seconds > 0 ? `${minutes}m ${seconds}s` : `${minutes}m`; + } + + const hours = Math.floor(minutes / 60); + const remainingMinutes = minutes % 60; + + return remainingMinutes > 0 ? `${hours}h ${remainingMinutes}m` : `${hours}h`; +} + +function useBootElapsedSeconds( + isActive: boolean, + activeSinceMs?: number, +): number | null { + const [nowMs, setNowMs] = useState(() => Date.now()); + + useEffect(() => { + if (!isActive || activeSinceMs === undefined) { + return; + } + + setNowMs(Date.now()); + const intervalId = window.setInterval(() => { + setNowMs(Date.now()); + }, 1_000); + + return () => { + window.clearInterval(intervalId); + }; + }, [isActive, activeSinceMs]); + + if (!isActive || activeSinceMs === undefined) { + return null; + } + + return Math.max(0, nowMs - activeSinceMs); +} + +const StartupMessage = ({ + step, + isActive, + activeSinceMs, +}: StartupMessageProps) => { const Icon = getStepIcon(step); const message = getStepMessage(step); + const elapsedMs = useBootElapsedSeconds(isActive, activeSinceMs); + const showElapsedHint = + isActive && elapsedMs !== null && elapsedMs >= BOOT_ELAPSED_HINT_AFTER_MS; return ( -
- - {isActive ? ( - - {message} - - ) : ( - {message} +
+
+ + {isActive ? ( + + {message} + + ) : ( + {message} + )} +
+ {showElapsedHint && elapsedMs !== null && ( +
+ Still booting… (elapsed: {formatBootElapsed(elapsedMs)}) +
)}
@@ -278,6 +346,8 @@ interface StartupSequenceProps { logsError?: string | null; prompt?: StartupPromptPreview | null; retryAction?: StartupRetryAction; + /** Epoch ms when the latest active step started. */ + activeStepSinceMs?: number; } export const StartupSequence = ({ @@ -288,6 +358,7 @@ export const StartupSequence = ({ logsError = null, prompt, retryAction, + activeStepSinceMs, }: StartupSequenceProps) => { const lastStep = steps[steps.length - 1]; const status = lastStep?.status ?? RunStatus.Pending; @@ -325,6 +396,11 @@ export const StartupSequence = ({ {index === logInsertIndex && ( void; } +function resolveActiveStepSinceMs(taskRun?: TaskRun | null): number { + const timestampCandidates = [ + taskRun?.startedAt, + taskRun?.dequeuedAt, + taskRun?.createdAt, + ]; + + for (const value of timestampCandidates) { + if (!value) { + continue; + } + + const ms = new Date(value).getTime(); + + if (Number.isFinite(ms)) { + return ms; + } + } + + return Date.now(); +} + export function useStartupProgress({ runId, initialTaskRun, @@ -35,6 +57,9 @@ export function useStartupProgress({ completed: isExitedRunStatus(initialStatus), }, ]); + const [activeStepSinceMs, setActiveStepSinceMs] = useState(() => + resolveActiveStepSinceMs(initialTaskRun), + ); const streamedTaskRun = useSSE('message', undefined); @@ -58,6 +83,7 @@ export function useStartupProgress({ useEffect(() => { if (status !== statusRef.current) { statusRef.current = status; + setActiveStepSinceMs(Date.now()); onStatusChange?.(status); } @@ -103,5 +129,6 @@ export function useStartupProgress({ sandboxLogs, logsConnected, logsError, + activeStepSinceMs, }; } diff --git a/apps/web/src/lib/task-run-errors.test.ts b/apps/web/src/lib/task-run-errors.test.ts index 3b6a656fd..5f70c3056 100644 --- a/apps/web/src/lib/task-run-errors.test.ts +++ b/apps/web/src/lib/task-run-errors.test.ts @@ -24,4 +24,31 @@ stderr -> fatal: 'origin/main' is not a commit and a branch 'main' cannot be cre 'Environment not found', ); }); + + it('rewrites missing or unpullable worker image failures', () => { + const error = `Docker command failed (docker run sleep): +Unable to find image 'roomote-worker:local' locally +docker: Error response from daemon: pull access denied for roomote-worker, repository does not exist or may require 'docker login'`; + + expect(getTaskRunErrorDisplayMessage(error)).toBe( + "Roomote couldn't start because the worker image `roomote-worker:local` is missing or can't be pulled. Build or pull that image on the host (for local Docker, run the worker image build), or set DOCKER_WORKER_IMAGE to an image the host can access.", + ); + }); + + it('surfaces docker stderr instead of a raw command-failed dump', () => { + const error = `Command failed: docker run -d --name roomote-worker-12 roomote-worker:local sleep infinity +permission denied while trying to connect to the Docker daemon socket`; + + expect(getTaskRunErrorDisplayMessage(error)).toBe( + 'Docker failed while starting the environment:\npermission denied while trying to connect to the Docker daemon socket', + ); + }); + + it('rewrites worker fetch-failed boot errors', () => { + expect( + getTaskRunErrorDisplayMessage('❌ Job sandbox-1 failed: fetch failed'), + ).toBe( + 'Roomote reached the worker, but the sandbox failed while contacting the Roomote API (`fetch failed`). Check that the API is reachable from the worker network and that API/controller URLs are configured correctly.', + ); + }); }); diff --git a/apps/web/src/lib/task-run-errors.ts b/apps/web/src/lib/task-run-errors.ts index 3695b0e89..83983e8f5 100644 --- a/apps/web/src/lib/task-run-errors.ts +++ b/apps/web/src/lib/task-run-errors.ts @@ -16,6 +16,9 @@ const OPENAI_ADMIN_ERROR_PREFIX = /^OpenAI admin request failed \((\d{3})\):\s*([\s\S]+)$/; const MISSING_REMOTE_BRANCH_DURING_WORKSPACE_PREP = /^Failed to prepare 1 workspace repository:\n- (?[^:]+):[\s\S]*?git checkout -B (?\S+) origin\/\S+[\s\S]*?stderr -> fatal: 'origin\/[^']+' is not a commit and a branch '[^']+' cannot be created from it$/; +const DOCKER_WORKER_IMAGE_REF = + /(?:Unable to find image ['"](?[^'"]+)['"] locally|pull access denied for (?[^\s,]+)|Error response from daemon: pull access denied for (?[^\s,]+))/i; +const DOCKER_COMMAND_FAILED_PREFIX = /^Docker command failed \(([^)]+)\):\n?/; function parseOpenAiAdminErrorBody( body: string, @@ -46,6 +49,70 @@ function getWorkspacePreparationDisplayMessage( return `Roomote couldn't start because the configured branch \`${branch}\` for \`${repository}\` no longer exists on GitHub. Update the repository branch setting, or leave it blank to use the repository's default branch.`; } +function getDockerBootDisplayMessage(error: string): string | undefined { + const imageMatch = error.match(DOCKER_WORKER_IMAGE_REF); + const image = + imageMatch?.groups?.image?.trim() || + imageMatch?.groups?.deniedImage?.trim() || + imageMatch?.groups?.deniedImageAlt?.trim(); + + if (image) { + return `Roomote couldn't start because the worker image \`${image}\` is missing or can't be pulled. Build or pull that image on the host (for local Docker, run the worker image build), or set DOCKER_WORKER_IMAGE to an image the host can access.`; + } + + if ( + /Docker worker container exited before task run #\d+ started/i.test(error) + ) { + const logsMatch = error.match(/Recent Docker logs:\n([\s\S]+)$/i); + const logs = logsMatch?.[1]?.trim(); + + if (logs) { + return `Roomote started a worker container, but it exited before the environment came up.\n\n${logs}`; + } + + return 'Roomote started a worker container, but it exited before the environment came up. Check controller and Docker worker logs on the host for details.'; + } + + if ( + /Docker worker command for task run #\d+ was not observed during startup/i.test( + error, + ) + ) { + return 'Roomote started a worker container, but the sandbox process never became ready. The host may be under load, or the worker failed before reporting status. Check Docker worker logs on the host.'; + } + + if ( + /Job\s+\S+\s+failed:\s*fetch failed/i.test(error) || + /^❌?\s*Job\s+\S+\s+failed:\s*fetch failed\s*$/im.test(error) || + /^fetch failed$/i.test(error.trim()) + ) { + return 'Roomote reached the worker, but the sandbox failed while contacting the Roomote API (`fetch failed`). Check that the API is reachable from the worker network and that API/controller URLs are configured correctly.'; + } + + // Prefer the docker daemon detail block over a raw "Command failed: docker run …" dump. + const dockerFailedMatch = error.match(DOCKER_COMMAND_FAILED_PREFIX); + + if (dockerFailedMatch) { + const detail = error.slice(dockerFailedMatch[0].length).trim(); + + if (detail) { + return `Docker failed while starting the environment:\n${detail}`; + } + } + + if (/^Command failed:\s*docker\b/i.test(error)) { + const detail = error + .replace(/^Command failed:\s*docker(?:\s+[^\n]+)?\n?/i, '') + .trim(); + + if (detail) { + return `Docker failed while starting the environment:\n${detail}`; + } + } + + return undefined; +} + export function getTaskRunError( taskRun?: TaskRunErrorSource | null, ): string | undefined { @@ -84,6 +151,12 @@ export function getTaskRunErrorDisplayMessage( return workspacePreparationMessage; } + const dockerBootMessage = getDockerBootDisplayMessage(stripped); + + if (dockerBootMessage) { + return dockerBootMessage; + } + const openAiMatch = stripped.match(OPENAI_ADMIN_ERROR_PREFIX); if (!openAiMatch) {