From 4d5e413db17f972b00b096415d8c06e8bc7c3c68 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:36:23 +0100 Subject: [PATCH 1/7] feat(supervisor): cancel a resumed run's in-flight checkpoint --- apps/supervisor/src/workloadServer/index.ts | 38 +++++++++++++++++++ .../src/v3/serverOnly/checkpointClient.ts | 17 +++++++++ 2 files changed, 55 insertions(+) diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index e14060e609..17f027b079 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -60,6 +60,13 @@ const checkpointDeleteRequests = new Counter({ registers: [register], }); +const checkpointCancelRequests = new Counter({ + name: "checkpoint_cancel_requests_total", + help: "Checkpoint cancel requests attempted when a run continues, by outcome", + labelNames: ["result"], // "sent" | "no_client" | "not_applicable" | "http_error" + registers: [register], +}); + const WorkloadActionParams = z.object({ runFriendlyId: z.string(), snapshotFriendlyId: z.string(), @@ -283,6 +290,35 @@ export class WorkloadServer extends EventEmitter { checkpointDeleteRequests.inc({ result: "sent" }); } + /** + * cancelCheckpoints tells the checkpoint service a resumed run's in-flight checkpoint is moot. + * Must be called after the reply is sent: it never delays the runner. Without it the checkpoint + * is abandoned by a periodic snapshot poll instead, up to that interval later. + */ + private async cancelCheckpoints(runFriendlyId: string): Promise { + if (!this.checkpointClient) { + checkpointCancelRequests.inc({ result: "no_client" }); + return; + } + + if (this.snapshotService) { + checkpointCancelRequests.inc({ result: "not_applicable" }); + return; + } + + const [error, accepted] = await tryCatch( + this.checkpointClient.cancelCheckpoints({ runFriendlyId }) + ); + + if (error || !accepted) { + checkpointCancelRequests.inc({ result: "http_error" }); + this.logger.error("Failed to request checkpoint cancel", { runFriendlyId, error }); + return; + } + + checkpointCancelRequests.inc({ result: "sent" }); + } + /** * Sets common route meta on the wide-event state from URL params. */ @@ -643,6 +679,8 @@ export class WorkloadServer extends EventEmitter { } reply.json(continuationResult.data as WorkloadContinueRunExecutionResponseBody); + + await this.cancelCheckpoints(params.runFriendlyId); } ), } diff --git a/packages/core/src/v3/serverOnly/checkpointClient.ts b/packages/core/src/v3/serverOnly/checkpointClient.ts index 936557c0ac..b3b4633e4b 100644 --- a/packages/core/src/v3/serverOnly/checkpointClient.ts +++ b/packages/core/src/v3/serverOnly/checkpointClient.ts @@ -157,4 +157,21 @@ export class CheckpointClient { return true; } + + async cancelCheckpoints({ runFriendlyId }: { runFriendlyId: string }): Promise { + const res = await fetch( + new URL(`/api/v1/runs/${runFriendlyId}/checkpoints/cancel`, this.opts.apiUrl), + { method: "POST" } + ); + + if (!res.ok) { + this.logger.error("[CheckpointClient] Cancel checkpoints request failed", { + runFriendlyId, + status: res.status, + }); + return false; + } + + return true; + } } From d85d7704609255c2d8542a966a412a0574d2da42 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:50:54 +0100 Subject: [PATCH 2/7] fix(supervisor): treat a missing cancel route as unsupported, not an error --- apps/supervisor/src/workloadServer/index.ts | 12 +++++++++--- .../core/src/v3/serverOnly/checkpointClient.ts | 18 +++++++++++++++--- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index 17f027b079..4471f77437 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -63,7 +63,7 @@ const checkpointDeleteRequests = new Counter({ const checkpointCancelRequests = new Counter({ name: "checkpoint_cancel_requests_total", help: "Checkpoint cancel requests attempted when a run continues, by outcome", - labelNames: ["result"], // "sent" | "no_client" | "not_applicable" | "http_error" + labelNames: ["result"], registers: [register], }); @@ -306,16 +306,22 @@ export class WorkloadServer extends EventEmitter { return; } - const [error, accepted] = await tryCatch( + const [error, outcome] = await tryCatch( this.checkpointClient.cancelCheckpoints({ runFriendlyId }) ); - if (error || !accepted) { + if (error || outcome === "failed") { checkpointCancelRequests.inc({ result: "http_error" }); this.logger.error("Failed to request checkpoint cancel", { runFriendlyId, error }); return; } + if (outcome === "unsupported") { + checkpointCancelRequests.inc({ result: "unsupported" }); + this.logger.debug("Checkpoint cancel not supported", { runFriendlyId }); + return; + } + checkpointCancelRequests.inc({ result: "sent" }); } diff --git a/packages/core/src/v3/serverOnly/checkpointClient.ts b/packages/core/src/v3/serverOnly/checkpointClient.ts index b3b4633e4b..e38d93cd14 100644 --- a/packages/core/src/v3/serverOnly/checkpointClient.ts +++ b/packages/core/src/v3/serverOnly/checkpointClient.ts @@ -158,20 +158,32 @@ export class CheckpointClient { return true; } - async cancelCheckpoints({ runFriendlyId }: { runFriendlyId: string }): Promise { + /** + * cancelCheckpoints returns "unsupported" when the route is absent, which is expected while a + * newer caller runs against an older checkpoint service. + */ + async cancelCheckpoints({ + runFriendlyId, + }: { + runFriendlyId: string; + }): Promise<"ok" | "unsupported" | "failed"> { const res = await fetch( new URL(`/api/v1/runs/${runFriendlyId}/checkpoints/cancel`, this.opts.apiUrl), { method: "POST" } ); + if (res.status === 404) { + return "unsupported"; + } + if (!res.ok) { this.logger.error("[CheckpointClient] Cancel checkpoints request failed", { runFriendlyId, status: res.status, }); - return false; + return "failed"; } - return true; + return "ok"; } } From 7aae3bf6a2700046898341ed5f30d5a9697eeaf1 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:58:03 +0100 Subject: [PATCH 3/7] docs(core): state the cancel route's 404 contract --- packages/core/src/v3/serverOnly/checkpointClient.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/src/v3/serverOnly/checkpointClient.ts b/packages/core/src/v3/serverOnly/checkpointClient.ts index e38d93cd14..574c8ceb42 100644 --- a/packages/core/src/v3/serverOnly/checkpointClient.ts +++ b/packages/core/src/v3/serverOnly/checkpointClient.ts @@ -160,7 +160,8 @@ export class CheckpointClient { /** * cancelCheckpoints returns "unsupported" when the route is absent, which is expected while a - * newer caller runs against an older checkpoint service. + * newer caller runs against an older checkpoint service. The route answers 202 even when the run + * has nothing in flight, so a 404 only ever means the route itself is missing. */ async cancelCheckpoints({ runFriendlyId, From d072615faabfb4242d266a9abea566f464354382 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:12:38 +0100 Subject: [PATCH 4/7] fix(core): bound the checkpoint cancel request with a timeout --- packages/core/src/v3/serverOnly/checkpointClient.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/core/src/v3/serverOnly/checkpointClient.ts b/packages/core/src/v3/serverOnly/checkpointClient.ts index 574c8ceb42..87df10c72e 100644 --- a/packages/core/src/v3/serverOnly/checkpointClient.ts +++ b/packages/core/src/v3/serverOnly/checkpointClient.ts @@ -13,6 +13,8 @@ export type CheckpointClientOptions = { orchestrator: CheckpointType; }; +const CANCEL_TIMEOUT_MS = 5_000; + export class CheckpointClient { private readonly logger = new SimpleStructuredLogger("checkpoint-client"); @@ -170,7 +172,7 @@ export class CheckpointClient { }): Promise<"ok" | "unsupported" | "failed"> { const res = await fetch( new URL(`/api/v1/runs/${runFriendlyId}/checkpoints/cancel`, this.opts.apiUrl), - { method: "POST" } + { method: "POST", signal: AbortSignal.timeout(CANCEL_TIMEOUT_MS) } ); if (res.status === 404) { From 95179cfad724533c2d7673a8eac91acb2eb3476b Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:10:34 +0100 Subject: [PATCH 5/7] feat(supervisor): make checkpoint request timeouts configurable --- apps/supervisor/src/env.ts | 2 ++ apps/supervisor/src/index.ts | 2 ++ apps/supervisor/src/workloadServer/index.ts | 9 ++------- .../src/v3/serverOnly/checkpointClient.ts | 19 ++++++++++++------- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 6830d5b864..35c3e0512f 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -115,6 +115,8 @@ export const Env = z TRIGGER_WARM_START_URL: z.string().optional(), TRIGGER_WARM_START_DISPATCH_URL: z.string().optional(), TRIGGER_CHECKPOINT_URL: z.string().optional(), + TRIGGER_CHECKPOINT_TIMEOUT_MS: z.coerce.number().int().positive().default(5_000), + TRIGGER_CHECKPOINT_RESTORE_TIMEOUT_MS: z.coerce.number().int().positive().default(30_000), TRIGGER_METADATA_URL: z.string().optional(), // Warm-start delivery verification: after a warm-start hit, probe the diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index 542841bd6e..79ac470865 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -382,6 +382,8 @@ class ManagedSupervisor { apiUrl: new URL(env.TRIGGER_CHECKPOINT_URL), workerClient: this.workerSession.httpClient, orchestrator: this.isKubernetes ? "KUBERNETES" : "DOCKER", + timeoutMs: env.TRIGGER_CHECKPOINT_TIMEOUT_MS, + restoreTimeoutMs: env.TRIGGER_CHECKPOINT_RESTORE_TIMEOUT_MS, }); } diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index 4471f77437..21f76625a7 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -290,12 +290,7 @@ export class WorkloadServer extends EventEmitter { checkpointDeleteRequests.inc({ result: "sent" }); } - /** - * cancelCheckpoints tells the checkpoint service a resumed run's in-flight checkpoint is moot. - * Must be called after the reply is sent: it never delays the runner. Without it the checkpoint - * is abandoned by a periodic snapshot poll instead, up to that interval later. - */ - private async cancelCheckpoints(runFriendlyId: string): Promise { + private async cancelCheckpointsAfterReply(runFriendlyId: string): Promise { if (!this.checkpointClient) { checkpointCancelRequests.inc({ result: "no_client" }); return; @@ -686,7 +681,7 @@ export class WorkloadServer extends EventEmitter { reply.json(continuationResult.data as WorkloadContinueRunExecutionResponseBody); - await this.cancelCheckpoints(params.runFriendlyId); + await this.cancelCheckpointsAfterReply(params.runFriendlyId); } ), } diff --git a/packages/core/src/v3/serverOnly/checkpointClient.ts b/packages/core/src/v3/serverOnly/checkpointClient.ts index 87df10c72e..fc36d3eba9 100644 --- a/packages/core/src/v3/serverOnly/checkpointClient.ts +++ b/packages/core/src/v3/serverOnly/checkpointClient.ts @@ -11,15 +11,22 @@ export type CheckpointClientOptions = { apiUrl: URL; workerClient: SupervisorHttpClient; orchestrator: CheckpointType; + timeoutMs?: number; + restoreTimeoutMs?: number; }; -const CANCEL_TIMEOUT_MS = 5_000; +const DEFAULT_TIMEOUT_MS = 5_000; +const DEFAULT_RESTORE_TIMEOUT_MS = 30_000; export class CheckpointClient { private readonly logger = new SimpleStructuredLogger("checkpoint-client"); constructor(private readonly opts: CheckpointClientOptions) {} + private timeout(ms = this.opts.timeoutMs ?? DEFAULT_TIMEOUT_MS): AbortSignal { + return AbortSignal.timeout(ms); + } + async suspendRun({ runFriendlyId, snapshotFriendlyId, @@ -43,6 +50,7 @@ export class CheckpointClient { type: this.opts.orchestrator, ...body, } satisfies CheckpointServiceSuspendRequestBodyInput), + signal: this.timeout(), } ); @@ -107,6 +115,7 @@ export class CheckpointClient { "Content-Type": "application/json", }, body: JSON.stringify(body), + signal: this.timeout(this.opts.restoreTimeoutMs ?? DEFAULT_RESTORE_TIMEOUT_MS), } ); @@ -146,6 +155,7 @@ export class CheckpointClient { "Content-Type": "application/json", }, body: JSON.stringify(body), + signal: this.timeout(), } ); @@ -160,11 +170,6 @@ export class CheckpointClient { return true; } - /** - * cancelCheckpoints returns "unsupported" when the route is absent, which is expected while a - * newer caller runs against an older checkpoint service. The route answers 202 even when the run - * has nothing in flight, so a 404 only ever means the route itself is missing. - */ async cancelCheckpoints({ runFriendlyId, }: { @@ -172,7 +177,7 @@ export class CheckpointClient { }): Promise<"ok" | "unsupported" | "failed"> { const res = await fetch( new URL(`/api/v1/runs/${runFriendlyId}/checkpoints/cancel`, this.opts.apiUrl), - { method: "POST", signal: AbortSignal.timeout(CANCEL_TIMEOUT_MS) } + { method: "POST", signal: this.timeout() } ); if (res.status === 404) { From 164d81c5d3523d4626c4ec585a1b1d62a5d66755 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:35:56 +0100 Subject: [PATCH 6/7] revert(supervisor): drop the checkpoint client timeout work from this pr --- apps/supervisor/src/env.ts | 2 -- apps/supervisor/src/index.ts | 2 -- apps/supervisor/src/workloadServer/index.ts | 10 ++----- .../src/v3/serverOnly/checkpointClient.ts | 28 +++---------------- 4 files changed, 6 insertions(+), 36 deletions(-) diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 35c3e0512f..6830d5b864 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -115,8 +115,6 @@ export const Env = z TRIGGER_WARM_START_URL: z.string().optional(), TRIGGER_WARM_START_DISPATCH_URL: z.string().optional(), TRIGGER_CHECKPOINT_URL: z.string().optional(), - TRIGGER_CHECKPOINT_TIMEOUT_MS: z.coerce.number().int().positive().default(5_000), - TRIGGER_CHECKPOINT_RESTORE_TIMEOUT_MS: z.coerce.number().int().positive().default(30_000), TRIGGER_METADATA_URL: z.string().optional(), // Warm-start delivery verification: after a warm-start hit, probe the diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index 79ac470865..542841bd6e 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -382,8 +382,6 @@ class ManagedSupervisor { apiUrl: new URL(env.TRIGGER_CHECKPOINT_URL), workerClient: this.workerSession.httpClient, orchestrator: this.isKubernetes ? "KUBERNETES" : "DOCKER", - timeoutMs: env.TRIGGER_CHECKPOINT_TIMEOUT_MS, - restoreTimeoutMs: env.TRIGGER_CHECKPOINT_RESTORE_TIMEOUT_MS, }); } diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index 21f76625a7..72d3b0e12b 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -301,22 +301,16 @@ export class WorkloadServer extends EventEmitter { return; } - const [error, outcome] = await tryCatch( + const [error, accepted] = await tryCatch( this.checkpointClient.cancelCheckpoints({ runFriendlyId }) ); - if (error || outcome === "failed") { + if (error || !accepted) { checkpointCancelRequests.inc({ result: "http_error" }); this.logger.error("Failed to request checkpoint cancel", { runFriendlyId, error }); return; } - if (outcome === "unsupported") { - checkpointCancelRequests.inc({ result: "unsupported" }); - this.logger.debug("Checkpoint cancel not supported", { runFriendlyId }); - return; - } - checkpointCancelRequests.inc({ result: "sent" }); } diff --git a/packages/core/src/v3/serverOnly/checkpointClient.ts b/packages/core/src/v3/serverOnly/checkpointClient.ts index fc36d3eba9..b3b4633e4b 100644 --- a/packages/core/src/v3/serverOnly/checkpointClient.ts +++ b/packages/core/src/v3/serverOnly/checkpointClient.ts @@ -11,22 +11,13 @@ export type CheckpointClientOptions = { apiUrl: URL; workerClient: SupervisorHttpClient; orchestrator: CheckpointType; - timeoutMs?: number; - restoreTimeoutMs?: number; }; -const DEFAULT_TIMEOUT_MS = 5_000; -const DEFAULT_RESTORE_TIMEOUT_MS = 30_000; - export class CheckpointClient { private readonly logger = new SimpleStructuredLogger("checkpoint-client"); constructor(private readonly opts: CheckpointClientOptions) {} - private timeout(ms = this.opts.timeoutMs ?? DEFAULT_TIMEOUT_MS): AbortSignal { - return AbortSignal.timeout(ms); - } - async suspendRun({ runFriendlyId, snapshotFriendlyId, @@ -50,7 +41,6 @@ export class CheckpointClient { type: this.opts.orchestrator, ...body, } satisfies CheckpointServiceSuspendRequestBodyInput), - signal: this.timeout(), } ); @@ -115,7 +105,6 @@ export class CheckpointClient { "Content-Type": "application/json", }, body: JSON.stringify(body), - signal: this.timeout(this.opts.restoreTimeoutMs ?? DEFAULT_RESTORE_TIMEOUT_MS), } ); @@ -155,7 +144,6 @@ export class CheckpointClient { "Content-Type": "application/json", }, body: JSON.stringify(body), - signal: this.timeout(), } ); @@ -170,28 +158,20 @@ export class CheckpointClient { return true; } - async cancelCheckpoints({ - runFriendlyId, - }: { - runFriendlyId: string; - }): Promise<"ok" | "unsupported" | "failed"> { + async cancelCheckpoints({ runFriendlyId }: { runFriendlyId: string }): Promise { const res = await fetch( new URL(`/api/v1/runs/${runFriendlyId}/checkpoints/cancel`, this.opts.apiUrl), - { method: "POST", signal: this.timeout() } + { method: "POST" } ); - if (res.status === 404) { - return "unsupported"; - } - if (!res.ok) { this.logger.error("[CheckpointClient] Cancel checkpoints request failed", { runFriendlyId, status: res.status, }); - return "failed"; + return false; } - return "ok"; + return true; } } From 3cf277b585454827f2b671d3d5c98f617ce8ec73 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:42:33 +0100 Subject: [PATCH 7/7] fix(core): bound the new cancel request with a timeout --- packages/core/src/v3/serverOnly/checkpointClient.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/core/src/v3/serverOnly/checkpointClient.ts b/packages/core/src/v3/serverOnly/checkpointClient.ts index b3b4633e4b..f89c1d1f43 100644 --- a/packages/core/src/v3/serverOnly/checkpointClient.ts +++ b/packages/core/src/v3/serverOnly/checkpointClient.ts @@ -13,6 +13,8 @@ export type CheckpointClientOptions = { orchestrator: CheckpointType; }; +const CANCEL_TIMEOUT_MS = 5_000; + export class CheckpointClient { private readonly logger = new SimpleStructuredLogger("checkpoint-client"); @@ -161,7 +163,7 @@ export class CheckpointClient { async cancelCheckpoints({ runFriendlyId }: { runFriendlyId: string }): Promise { const res = await fetch( new URL(`/api/v1/runs/${runFriendlyId}/checkpoints/cancel`, this.opts.apiUrl), - { method: "POST" } + { method: "POST", signal: AbortSignal.timeout(CANCEL_TIMEOUT_MS) } ); if (!res.ok) {