diff --git a/packages/junior-evals/README.md b/packages/junior-evals/README.md index 449a5d751..92ff44054 100644 --- a/packages/junior-evals/README.md +++ b/packages/junior-evals/README.md @@ -69,7 +69,7 @@ For each `it()` case inside a `describeEval()` suite: - Use the Slack eval harness for Slack/runtime behavior: mentions, thread/channel delivery, OAuth privacy, lifecycle/resume behavior, reactions, and Slack-visible side effects. - Use an agent-level harness for prompt, skill routing, tool choice, provider/tool calls, and reply quality when Slack transport is not the behavior under test. -- The Slack eval harness preserves inbound messages and direct thread replies in observed order. Slack API-captured side effects may be collected afterward. The rubric judge receives only non-empty user-visible text from normalized user and assistant messages; tool calls, artifacts, logs, metadata, and other runtime observations stay outside its prompt. +- The Slack eval harness preserves inbound messages and direct thread replies. Replies delivered through the Slack API may be collected afterward; other captured Slack side effects stay outside the rubric. The rubric judge receives only non-empty user-visible text from normalized user and assistant messages; tool calls, artifacts, logs, metadata, and other runtime observations stay outside its prompt. - When the eval boundary is Junior's Pi agent or needs an ordered full-turn transcript, prefer `@vitest-evals/harness-pi-ai` primitives instead of rebuilding transcript capture locally. The Pi harness already owns normalized `session.messages`, `toolCalls(result.session)`, artifacts, traces, replay, and judge context. - Do not assert against logs, spans, or status telemetry for product behavior. Use `vitest-evals` session/tool/artifact primitives for behavior contracts; reserve traces/spans for instrumentation tests or diagnostics. diff --git a/packages/junior-evals/evals/core/oauth-workflows.eval.ts b/packages/junior-evals/evals/core/oauth-workflows.eval.ts index eba2cea03..5e1852f15 100644 --- a/packages/junior-evals/evals/core/oauth-workflows.eval.ts +++ b/packages/junior-evals/evals/core/oauth-workflows.eval.ts @@ -10,6 +10,8 @@ import { } from "../../src/helpers"; type EvalRun = HarnessRun; +const EVAL_OAUTH_IDENTITY_ENDPOINT = + "https://example.com/junior-eval-oauth/whoami"; function textContent(value: unknown): string { return typeof value === "string" ? value : ""; @@ -25,7 +27,8 @@ function expectNoPublicOAuthUrl(result: EvalRun): void { } function expectEvalOauthIdentityCheck(result: EvalRun): void { - expect(toolCalls(result.session)).toEqual( + const calls = toolCalls(result.session); + expect(calls).toEqual( expect.arrayContaining([ expect.objectContaining({ name: "loadSkill", @@ -33,14 +36,24 @@ function expectEvalOauthIdentityCheck(result: EvalRun): void { skill_name: "eval-oauth", }), }), - expect.objectContaining({ - name: "bash", - arguments: expect.objectContaining({ - command: "curl -fsSL https://example.com/junior-eval-oauth/whoami", - }), - }), ]), ); + expect( + evalOauthIdentityChecks(result).some((call) => + JSON.stringify(call.result ?? "").includes("eval-oauth-user"), + ), + ).toBe(true); +} + +function evalOauthIdentityChecks(result: EvalRun) { + return toolCalls(result.session).filter( + (call) => + call.name === "bash" && + typeof call.arguments?.command === "string" && + call.arguments.command + .split(/\s+/u) + .some((token) => token === EVAL_OAUTH_IDENTITY_ENDPOINT), + ); } function matchingToolCalls( @@ -185,14 +198,10 @@ describeEval("OAuth Workflows", slackEvals, (it) => { }, ]); expectEvalOauthIdentityCheck(result); - expect( - matchingToolCalls(result, "bash", { - command: "curl -fsSL https://example.com/junior-eval-oauth/whoami", - }).length, - ).toBeGreaterThanOrEqual(3); + expect(evalOauthIdentityChecks(result).length).toBeGreaterThanOrEqual(3); expectFinalThreadReply(result, oauthResumeThread, /\bFriday\b/i); expectFinalThreadReply(result, oauthResumeThread, /eval-oauth-user/i); - }); + }, 120_000); const oauthRefreshThread = { id: "thread-oauth-refresh", @@ -279,5 +288,5 @@ describeEval("OAuth Workflows", slackEvals, (it) => { oauthReconnectThread, /connected|reconnected/i, ); - }); + }, 120_000); }); diff --git a/packages/junior-evals/src/behavior-harness.ts b/packages/junior-evals/src/behavior-harness.ts index ceff4a9ac..31b214103 100644 --- a/packages/junior-evals/src/behavior-harness.ts +++ b/packages/junior-evals/src/behavior-harness.ts @@ -26,6 +26,7 @@ import { createSlackRuntime } from "@/chat/app/factory"; import { getConversationEventStore, getDb } from "@/chat/db"; import type { AssistantLifecycleEvent } from "@/chat/runtime/slack-runtime"; import type { JuniorRuntimeServiceOverrides } from "@/chat/app/services"; +import { createProductionRecoverableSlackDelivery } from "@/chat/app/services"; import { createUserTokenStore } from "@/chat/capabilities/factory"; import { parseOAuthStatePayload } from "@/chat/oauth-flow"; import type { EmittedLogRecord } from "@/chat/logging"; @@ -52,6 +53,7 @@ import { appendAndEnqueueInboundMessage } from "@/chat/task-execution/store"; import { executeAgentRun } from "@/chat/agent"; import { completedAgentRun } from "@/chat/runtime/agent-run-outcome"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; +import type { RecoverableSlackDelivery } from "@/chat/slack/recoverable-delivery"; import { addAgentTurnUsage, type AgentTurnUsage } from "@/chat/usage"; import { resumeAwaitingSlackContinuation } from "@/chat/runtime/agent-continue-runner"; import { scheduleAgentContinue } from "@/chat/services/agent-continue"; @@ -63,14 +65,14 @@ import { } from "@sentry/junior-scheduler"; import { createMemoryPlugin } from "@sentry/junior-memory"; import { runPluginHeartbeats } from "@/chat/agent-dispatch/heartbeat"; -import { runAgentDispatchSlice } from "@/chat/agent-dispatch/runner"; +import { processAgentDispatchCallback } from "@/chat/agent-dispatch/runner"; +import { verifyDispatchCallbackRequest } from "@/chat/agent-dispatch/signing"; +import { getDispatchRecord } from "@/chat/agent-dispatch/store"; +import type { DispatchCallback } from "@/chat/agent-dispatch/types"; import { ConversationTurnLifecycleService, type ConversationTurnLifecycle, } from "@/chat/conversations/turn-lifecycle"; -import { verifyDispatchCallbackRequest } from "@/chat/agent-dispatch/signing"; -import { getDispatchRecord } from "@/chat/agent-dispatch/store"; -import type { DispatchCallback } from "@/chat/agent-dispatch/types"; import { ingestResourceEvent } from "@/chat/resource-events/ingest"; import { createResourceEventSubscription } from "@/chat/resource-events/store"; import { getStateAdapter } from "@/chat/state/adapter"; @@ -1542,6 +1544,7 @@ async function autoCompleteMcpOauth(args: { completions: AuthorizationCompletion[]; provider: string; consumedStates: Set; + recoverableSlackDelivery: RecoverableSlackDelivery; }): Promise { const provider = args.provider.trim() || EVAL_MCP_AUTH_PROVIDER; if (provider !== EVAL_MCP_AUTH_PROVIDER) { @@ -1578,6 +1581,7 @@ async function autoCompleteMcpOauth(args: { state: delivered.state, code: getDefaultAuthCode("mcp-oauth", provider), agentRunner: args.agentRunner, + recoverableSlackDelivery: args.recoverableSlackDelivery, }); if (response.status !== 200) { throw new Error( @@ -1609,6 +1613,7 @@ async function autoCompleteOauth(args: { completions: AuthorizationCompletion[]; provider: string; consumedStates: Set; + recoverableSlackDelivery: RecoverableSlackDelivery; }): Promise { const provider = args.provider.trim() || EVAL_OAUTH_PROVIDER; const providerConfig = pluginCatalogRuntime.getOAuthConfig(provider); @@ -1646,6 +1651,7 @@ async function autoCompleteOauth(args: { state: delivered.state, code: getDefaultAuthCode("oauth", provider), agentRunner: args.agentRunner, + recoverableSlackDelivery: args.recoverableSlackDelivery, }); if (response.status !== 200) { throw new Error( @@ -1824,6 +1830,7 @@ function buildRuntimeServices( conversationWorkQueue: ConversationWorkQueueTestAdapter, steeringDelivery: SteeringDelivery, turnLifecycle: ConversationTurnLifecycle, + recoverableSlackDelivery: RecoverableSlackDelivery, signal?: AbortSignal, ): JuniorRuntimeServiceOverrides { const replyResults = scenario.overrides?.reply_results ?? []; @@ -1872,6 +1879,7 @@ function buildRuntimeServices( : {}), replyExecutor: { turnLifecycle, + recoverableSlackDelivery, agentRunner: { run: async (request) => { const pendingSteeringDelivery = steeringDelivery.deliver; @@ -2098,6 +2106,7 @@ async function processEvents(args: { env: HarnessEnvironment; agentRunner: AgentRunner; turnLifecycle: ConversationTurnLifecycle; + recoverableSlackDelivery: RecoverableSlackDelivery; getSlackAdapter: () => FakeSlackAdapter; conversationWorkQueue: ConversationWorkQueueTestAdapter; slackRuntime: ReturnType; @@ -2112,6 +2121,7 @@ async function processEvents(args: { env, agentRunner, turnLifecycle, + recoverableSlackDelivery, getSlackAdapter, conversationWorkQueue, slackRuntime, @@ -2125,12 +2135,14 @@ async function processEvents(args: { const consumedMcpOauthStates = new Set(); const maybeAutoCompleteAuth = async (): Promise => { + const callbackCallOffset = readCapturedSlackApiCalls().length; for (const provider of env.autoCompleteMcpOauthProviders) { await autoCompleteMcpOauth({ agentRunner, completions: args.observations.authorizationCompletions, provider, consumedStates: consumedMcpOauthStates, + recoverableSlackDelivery, }); } for (const provider of env.autoCompleteOauthProviders) { @@ -2139,6 +2151,27 @@ async function processEvents(args: { completions: args.observations.authorizationCompletions, provider, consumedStates: consumedOauthStates, + recoverableSlackDelivery, + }); + } + const callbackPosts = collectSlackArtifactsFromCapturedCalls( + readCapturedSlackApiCalls().slice(callbackCallOffset), + ).channelPosts; + for (const post of callbackPosts) { + if ( + !post.thread_ts || + !findEvalThread(`slack:${post.channel}:${post.thread_ts}`) + ) { + continue; + } + args.observations.sessionMessages.push({ + role: "assistant", + content: post.text, + metadata: { + event_type: "thread_post", + channel: post.channel, + thread_ts: post.thread_ts, + }, }); } }; @@ -2202,6 +2235,7 @@ async function processEvents(args: { resumeAwaitingContinuation: async (conversationId) => await resumeAwaitingSlackContinuation(conversationId, { agentRunner, + recoverableSlackDelivery, scheduleAgentContinue: async (request) => { await scheduleAgentContinue(request, { queue: conversationWorkQueue, @@ -2215,6 +2249,7 @@ async function processEvents(args: { }, }); }, + turnLifecycle, }), runtime: workerRuntime, state: env.stateAdapter, @@ -2387,6 +2422,7 @@ async function processEvents(args: { `Scheduled eval task did not create a dispatch: ${JSON.stringify({ runs, savedTask })}`, ); } + const dispatchCallOffset = readCapturedSlackApiCalls().length; for (const run of dispatchedRuns) { const dispatch = await getDispatchRecord(run.dispatchId!); if (!dispatch) { @@ -2398,7 +2434,25 @@ async function processEvents(args: { if (!callback) { throw new Error("Scheduled eval dispatch callback was not captured."); } - await runAgentDispatchSlice(callback, { agentRunner, turnLifecycle }); + await processAgentDispatchCallback(callback, { + agentRunner, + recoverableSlackDelivery, + turnLifecycle, + }); + } + const deliveredPosts = collectSlackArtifactsFromCapturedCalls( + readCapturedSlackApiCalls().slice(dispatchCallOffset), + ).channelPosts; + for (const post of deliveredPosts) { + args.observations.sessionMessages.push({ + role: "assistant", + content: post.text, + metadata: { + event_type: post.thread_ts ? "thread_post" : "channel_post", + channel: post.channel, + ...(post.thread_ts ? { thread_ts: post.thread_ts } : {}), + }, + }); } }; @@ -2674,6 +2728,9 @@ export async function runEvalScenario( const turnLifecycle = new ConversationTurnLifecycleService( getConversationEventStore(), ); + const recoverableSlackDelivery = createProductionRecoverableSlackDelivery({ + getSlackAdapter: () => slackAdapter as unknown as SlackAdapter, + }); const services = buildRuntimeServices( scenario, env, @@ -2682,6 +2739,7 @@ export async function runEvalScenario( conversationWorkQueue, steeringDelivery, turnLifecycle, + recoverableSlackDelivery, options.signal, ); const evalAgentRunner = services.replyExecutor?.agentRunner; @@ -2699,6 +2757,7 @@ export async function runEvalScenario( env, agentRunner: evalAgentRunner, turnLifecycle, + recoverableSlackDelivery, getSlackAdapter: () => slackAdapter, conversationWorkQueue, slackRuntime, diff --git a/packages/junior/migrations/0006_pending_delivery_outbox.sql b/packages/junior/migrations/0006_pending_delivery_outbox.sql new file mode 100644 index 000000000..47164e2cc --- /dev/null +++ b/packages/junior/migrations/0006_pending_delivery_outbox.sql @@ -0,0 +1,16 @@ +CREATE TABLE "junior_pending_deliveries" ( + "delivery_id" text PRIMARY KEY NOT NULL, + "conversation_id" text NOT NULL, + "turn_id" text NOT NULL, + "command_json" jsonb NOT NULL, + "progress_json" jsonb NOT NULL, + "next_attempt_at" timestamp with time zone NOT NULL, + "lease_owner" text, + "lease_version" integer DEFAULT 0 NOT NULL, + "lease_expires_at" timestamp with time zone, + CONSTRAINT "junior_pending_deliveries_lease_version_check" CHECK ("junior_pending_deliveries"."lease_version" >= 0) +); +--> statement-breakpoint +ALTER TABLE "junior_pending_deliveries" ADD CONSTRAINT "junior_pending_deliveries_conversation_id_fk" FOREIGN KEY ("conversation_id") REFERENCES "public"."junior_conversations"("conversation_id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "junior_pending_deliveries_conversation_idx" ON "junior_pending_deliveries" USING btree ("conversation_id");--> statement-breakpoint +CREATE INDEX "junior_pending_deliveries_due_idx" ON "junior_pending_deliveries" USING btree ("next_attempt_at"); \ No newline at end of file diff --git a/packages/junior/migrations/meta/0006_snapshot.json b/packages/junior/migrations/meta/0006_snapshot.json new file mode 100644 index 000000000..d7110530b --- /dev/null +++ b/packages/junior/migrations/meta/0006_snapshot.json @@ -0,0 +1,1168 @@ +{ + "id": "33056a28-5ffc-4b7b-8c6c-8045411e5cae", + "prevId": "5bcf5afe-2cc5-497a-a873-08c54ab1dfb0", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.junior_conversation_events": { + "name": "junior_conversation_events", + "schema": "", + "columns": { + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "context_epoch": { + "name": "context_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "schema_version": { + "name": "schema_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "junior_conversation_events_epoch_idx": { + "name": "junior_conversation_events_epoch_idx", + "columns": [ + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "context_epoch", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversation_events_type_idx": { + "name": "junior_conversation_events_type_idx", + "columns": [ + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversation_events_idempotency_idx": { + "name": "junior_conversation_events_idempotency_idx", + "columns": [ + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk": { + "name": "junior_conversation_events_conversation_id_junior_conversations_conversation_id_fk", + "tableFrom": "junior_conversation_events", + "tableTo": "junior_conversations", + "columnsFrom": ["conversation_id"], + "columnsTo": ["conversation_id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "junior_conversation_events_conversation_id_seq_pk": { + "name": "junior_conversation_events_conversation_id_seq_pk", + "columns": ["conversation_id", "seq"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_conversation_messages": { + "name": "junior_conversation_messages", + "schema": "", + "columns": { + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author_identity_id": { + "name": "author_identity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "meta": { + "name": "meta", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "replied_at": { + "name": "replied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "junior_conversation_messages_activity_idx": { + "name": "junior_conversation_messages_activity_idx", + "columns": [ + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversation_messages_search_idx": { + "name": "junior_conversation_messages_search_idx", + "columns": [ + { + "expression": "to_tsvector('english', \"text\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "junior_conversation_messages_conversation_id_junior_conversations_conversation_id_fk": { + "name": "junior_conversation_messages_conversation_id_junior_conversations_conversation_id_fk", + "tableFrom": "junior_conversation_messages", + "tableTo": "junior_conversations", + "columnsFrom": ["conversation_id"], + "columnsTo": ["conversation_id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversation_messages_author_identity_id_junior_identities_id_fk": { + "name": "junior_conversation_messages_author_identity_id_junior_identities_id_fk", + "tableFrom": "junior_conversation_messages", + "tableTo": "junior_identities", + "columnsFrom": ["author_identity_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "junior_conversation_messages_conversation_id_message_id_pk": { + "name": "junior_conversation_messages_conversation_id_message_id_pk", + "columns": ["conversation_id", "message_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_conversations": { + "name": "junior_conversations", + "schema": "", + "columns": { + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "schema_version": { + "name": "schema_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_type": { + "name": "origin_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_id": { + "name": "origin_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_run_id": { + "name": "origin_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "destination_id": { + "name": "destination_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "destination_json": { + "name": "destination_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "actor_identity_id": { + "name": "actor_identity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "creator_identity_id": { + "name": "creator_identity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_subject_identity_id": { + "name": "credential_subject_identity_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_json": { + "name": "actor_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "channel_name": { + "name": "channel_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_activity_at": { + "name": "last_activity_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "execution_updated_at": { + "name": "execution_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "execution_status": { + "name": "execution_status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_checkpoint_at": { + "name": "last_checkpoint_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_enqueued_at": { + "name": "last_enqueued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "parent_conversation_id": { + "name": "parent_conversation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "transcript_purged_at": { + "name": "transcript_purged_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "usage_json": { + "name": "usage_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_duration_ms": { + "name": "execution_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "execution_usage_json": { + "name": "execution_usage_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metric_run_id": { + "name": "metric_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "junior_conversations_last_activity_idx": { + "name": "junior_conversations_last_activity_idx", + "columns": [ + { + "expression": "last_activity_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_active_idx": { + "name": "junior_conversations_active_idx", + "columns": [ + { + "expression": "coalesce(\"execution_updated_at\", \"updated_at\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"junior_conversations\".\"execution_status\" <> 'idle'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_destination_activity_idx": { + "name": "junior_conversations_destination_activity_idx", + "columns": [ + { + "expression": "destination_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_activity_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_actor_activity_idx": { + "name": "junior_conversations_actor_activity_idx", + "columns": [ + { + "expression": "actor_identity_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_activity_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_origin_idx": { + "name": "junior_conversations_origin_idx", + "columns": [ + { + "expression": "origin_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "origin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_activity_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_conversations_parent_idx": { + "name": "junior_conversations_parent_idx", + "columns": [ + { + "expression": "parent_conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "junior_conversations_destination_id_junior_destinations_id_fk": { + "name": "junior_conversations_destination_id_junior_destinations_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_destinations", + "columnsFrom": ["destination_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversations_actor_identity_id_junior_identities_id_fk": { + "name": "junior_conversations_actor_identity_id_junior_identities_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_identities", + "columnsFrom": ["actor_identity_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversations_creator_identity_id_junior_identities_id_fk": { + "name": "junior_conversations_creator_identity_id_junior_identities_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_identities", + "columnsFrom": ["creator_identity_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversations_credential_subject_identity_id_junior_identities_id_fk": { + "name": "junior_conversations_credential_subject_identity_id_junior_identities_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_identities", + "columnsFrom": ["credential_subject_identity_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "junior_conversations_parent_conversation_id_junior_conversations_conversation_id_fk": { + "name": "junior_conversations_parent_conversation_id_junior_conversations_conversation_id_fk", + "tableFrom": "junior_conversations", + "tableTo": "junior_conversations", + "columnsFrom": ["parent_conversation_id"], + "columnsTo": ["conversation_id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_destinations": { + "name": "junior_destinations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_tenant_id": { + "name": "provider_tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "provider_destination_id": { + "name": "provider_destination_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parent_destination_id": { + "name": "parent_destination_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "metadata_json": { + "name": "metadata_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "junior_destinations_provider_destination_uidx": { + "name": "junior_destinations_provider_destination_uidx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_destination_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_destinations_provider_kind_idx": { + "name": "junior_destinations_provider_kind_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_identities": { + "name": "junior_identities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_tenant_id": { + "name": "provider_tenant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "provider_subject_id": { + "name": "provider_subject_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "handle": { + "name": "handle", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_normalized": { + "name": "email_normalized", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "junior_identities_provider_subject_uidx": { + "name": "junior_identities_provider_subject_uidx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_tenant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_subject_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_identities_user_idx": { + "name": "junior_identities_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_identities_verified_email_idx": { + "name": "junior_identities_verified_email_idx", + "columns": [ + { + "expression": "email_normalized", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"junior_identities\".\"email_verified\" = true AND \"junior_identities\".\"email_normalized\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_identities_kind_provider_idx": { + "name": "junior_identities_kind_provider_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "junior_identities_user_id_junior_users_id_fk": { + "name": "junior_identities_user_id_junior_users_id_fk", + "tableFrom": "junior_identities", + "tableTo": "junior_users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.junior_pending_deliveries": { + "name": "junior_pending_deliveries", + "schema": "", + "columns": { + "delivery_id": { + "name": "delivery_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "command_json": { + "name": "command_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "progress_json": { + "name": "progress_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_version": { + "name": "lease_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "junior_pending_deliveries_conversation_idx": { + "name": "junior_pending_deliveries_conversation_idx", + "columns": [ + { + "expression": "conversation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "junior_pending_deliveries_due_idx": { + "name": "junior_pending_deliveries_due_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "junior_pending_deliveries_conversation_id_fk": { + "name": "junior_pending_deliveries_conversation_id_fk", + "tableFrom": "junior_pending_deliveries", + "tableTo": "junior_conversations", + "columnsFrom": ["conversation_id"], + "columnsTo": ["conversation_id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "junior_pending_deliveries_lease_version_check": { + "name": "junior_pending_deliveries_lease_version_check", + "value": "\"junior_pending_deliveries\".\"lease_version\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.junior_users": { + "name": "junior_users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "primary_email": { + "name": "primary_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "primary_email_normalized": { + "name": "primary_email_normalized", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "junior_users_primary_email_normalized_uidx": { + "name": "junior_users_primary_email_normalized_uidx", + "columns": [ + { + "expression": "primary_email_normalized", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/junior/migrations/meta/_journal.json b/packages/junior/migrations/meta/_journal.json index 7f4c61ea1..0825c99d5 100644 --- a/packages/junior/migrations/meta/_journal.json +++ b/packages/junior/migrations/meta/_journal.json @@ -43,6 +43,13 @@ "when": 1784137322421, "tag": "0005_conversation_events", "breakpoints": true + }, + { + "idx": 6, + "version": "7", + "when": 1784340837059, + "tag": "0006_pending_delivery_outbox", + "breakpoints": true } ] } diff --git a/packages/junior/src/app.ts b/packages/junior/src/app.ts index 46307fb71..4e5a3a4dd 100644 --- a/packages/junior/src/app.ts +++ b/packages/junior/src/app.ts @@ -8,8 +8,12 @@ import { getConfigDefaults, setConfigDefaults, } from "@/chat/configuration/defaults"; -import { getSlackReactionConfig, setSlackReactionConfig } from "@/chat/config"; -import { getDb } from "@/chat/db"; +import { + getChatConfig, + getSlackReactionConfig, + setSlackReactionConfig, +} from "@/chat/config"; +import { getConversationEventStore, getDb } from "@/chat/db"; import { logException } from "@/chat/logging"; import { executeAgentRun } from "@/chat/agent"; import { normalizeSandboxEgressTracePropagationDomains } from "@/chat/sandbox/egress/tracing"; @@ -47,7 +51,10 @@ import { GET as mcpOauthCallbackGET } from "@/handlers/mcp-oauth-callback"; import { GET as oauthCallbackGET } from "@/handlers/oauth-callback"; import { handleSandboxEgressRoute } from "@/handlers/sandbox-egress-route"; import { POST as slackWebhookPOST } from "@/handlers/slack-webhook"; -import { JUNIOR_PLUGIN_TASK_CALLBACK_ROUTE } from "@/deployment"; +import { + JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE, + JUNIOR_PLUGIN_TASK_CALLBACK_ROUTE, +} from "@/deployment"; import { createVercelConversationWorkCallback, registerVercelConversationWorkDevConsumer, @@ -62,9 +69,12 @@ import { import { createProductionConversationWorkOptions, createProductionSlackWebhookServices, + getProductionSlackAdapter, } from "@/chat/app/production"; import { createAgentRunner } from "@/chat/runtime/agent-runner"; import type { WaitUntilFn } from "@/handlers/types"; +import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; +import { createProductionRecoverableSlackDelivery } from "@/chat/app/services"; export { defineJuniorPlugins } from "./plugins"; export type { @@ -141,6 +151,7 @@ type CreateDashboardApp = ( interface JuniorVirtualConfig { createDashboardApp?: CreateDashboardApp; dashboard?: JuniorVirtualDashboardOptions; + functionMaxDurationSeconds?: number; pluginSet?: JuniorPluginSet; plugins?: PluginCatalogConfig; pluginRuntimeRegistrations: string[]; @@ -179,13 +190,21 @@ async function resolveVirtualConfig(): Promise< const mod: { createDashboardApp?: CreateDashboardApp; dashboard?: JuniorVirtualDashboardOptions; + functionMaxDurationSeconds?: number; pluginSet?: JuniorPluginSet; plugins?: PluginCatalogConfig; pluginRuntimeRegistrations?: string[]; } = await import("#junior/config"); + const functionMaxDurationSeconds = Object.hasOwn( + mod, + "functionMaxDurationSeconds", + ) + ? mod.functionMaxDurationSeconds + : undefined; return { createDashboardApp: mod.createDashboardApp, dashboard: mod.dashboard, + functionMaxDurationSeconds, pluginSet: mod.pluginSet, plugins: mod.plugins, pluginRuntimeRegistrations: mod.pluginRuntimeRegistrations ?? [], @@ -530,6 +549,9 @@ function mountRoutes(app: Hono, routes: HostRouteRegistration[]): void { /** Create a Hono app with all Junior routes. */ export async function createApp(options?: JuniorAppOptions): Promise { const virtualConfig = await resolveVirtualConfig(); + const functionMaxDurationSeconds = + virtualConfig?.functionMaxDurationSeconds ?? + getChatConfig().functionMaxDurationSeconds; const dashboard = options?.dashboard ?? virtualConfig?.dashboard; const configuredPlugins = options?.plugins ?? virtualConfig?.pluginSet; const plugins = pluginRuntimeRegistrationsFromPluginSet(configuredPlugins); @@ -591,10 +613,16 @@ export async function createApp(options?: JuniorAppOptions): Promise { const agentRunner = createAgentRunner(executeAgentRun, { tracePropagation, }); + const turnLifecycle = new ConversationTurnLifecycleService( + getConversationEventStore(), + ); const runtimeServiceOverrides = { - replyExecutor: { agentRunner }, + replyExecutor: { agentRunner, turnLifecycle }, sandbox: { tracePropagation }, }; + const recoverableSlackDelivery = createProductionRecoverableSlackDelivery({ + getSlackAdapter: getProductionSlackAdapter, + }); const slackWebhookServices = createProductionSlackWebhookServices({ services: runtimeServiceOverrides, }); @@ -632,17 +660,26 @@ export async function createApp(options?: JuniorAppOptions): Promise { app.get("/api/oauth/callback/mcp/:provider", (c) => { return mcpOauthCallbackGET(c.req.raw, c.req.param("provider"), waitUntil, { agentRunner, + recoverableSlackDelivery, + turnLifecycle, }); }); app.get("/api/oauth/callback/:provider", (c) => { return oauthCallbackGET(c.req.raw, c.req.param("provider"), waitUntil, { agentRunner, + recoverableSlackDelivery, + turnLifecycle, }); }); - app.post("/api/internal/agent-dispatch", (c) => { - return agentDispatchPOST(c.req.raw, waitUntil, { agentRunner }); + app.post(JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE, (c) => { + return agentDispatchPOST(c.req.raw, waitUntil, { + agentRunner, + functionMaxDurationSeconds, + recoverableSlackDelivery, + turnLifecycle, + }); }); let agentContinuePOST: @@ -659,6 +696,8 @@ export async function createApp(options?: JuniorAppOptions): Promise { options?.conversationWork ?? createProductionConversationWorkOptions({ agentRunner, + recoverableSlackDelivery, + turnLifecycle, services: runtimeServiceOverrides, }); return conversationWorkOptions; @@ -681,7 +720,7 @@ export async function createApp(options?: JuniorAppOptions): Promise { }); app.get("/api/internal/heartbeat", (c) => { - return heartbeatGET(c.req.raw, waitUntil); + return heartbeatGET(c.req.raw, waitUntil, { recoverableSlackDelivery }); }); app.get("/api/internal/retention", (c) => { diff --git a/packages/junior/src/build/virtual-config.ts b/packages/junior/src/build/virtual-config.ts index d8fa71e00..0b17b76d8 100644 --- a/packages/junior/src/build/virtual-config.ts +++ b/packages/junior/src/build/virtual-config.ts @@ -38,6 +38,7 @@ function dashboardEnabled( /** Render the virtual config module consumed by createApp(). */ export function renderVirtualConfig(options: { dashboard?: JuniorDashboardOptions; + functionMaxDurationSeconds?: number; plugins?: PluginCatalogConfig; pluginModule?: RuntimePluginModule; pluginRuntimeRegistrations?: string[]; @@ -53,6 +54,7 @@ export function renderVirtualConfig(options: { `export const plugins = ${JSON.stringify(options.plugins ?? { packages: [] })};`, `export const pluginRuntimeRegistrations = ${JSON.stringify(options.pluginRuntimeRegistrations ?? [])};`, `export const dashboard = ${JSON.stringify(options.dashboard)};`, + `export const functionMaxDurationSeconds = ${JSON.stringify(options.functionMaxDurationSeconds)};`, ]; return lines.join("\n"); @@ -63,6 +65,7 @@ export function injectVirtualConfig( nitro: Nitro, options: { loadPluginSet?: () => Promise; + functionMaxDurationSeconds?: number; pluginModule?: RuntimePluginModule; plugins?: PluginCatalogConfig; pluginRuntimeRegistrations?: string[]; @@ -83,6 +86,7 @@ export function injectVirtualConfig( pluginSet, ).map((plugin) => plugin.manifest.name), dashboard: options.dashboard, + functionMaxDurationSeconds: options.functionMaxDurationSeconds, }); }; } diff --git a/packages/junior/src/chat/agent-dispatch/heartbeat.ts b/packages/junior/src/chat/agent-dispatch/heartbeat.ts index 5a88e92e6..850255373 100644 --- a/packages/junior/src/chat/agent-dispatch/heartbeat.ts +++ b/packages/junior/src/chat/agent-dispatch/heartbeat.ts @@ -3,11 +3,15 @@ import { logException, logInfo } from "@/chat/logging"; import { recoverConversationWork } from "@/chat/task-execution/heartbeat"; import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; import { getVercelConversationWorkQueue } from "@/chat/task-execution/vercel-queue"; +import type { RecoverableSlackDelivery } from "@/chat/slack/recoverable-delivery"; +import { recoverDueSlackDeliveries } from "@/chat/runtime/slack-delivery-recovery"; import { createHeartbeatContext } from "./context"; import { scheduleDispatchCallback } from "./signing"; import { + getDispatchConversationId, getDispatchStorageKey, getDispatchRecord, + getDispatchTurnId, isTerminalDispatchStatus, listIncompleteDispatchIds, parseDispatchRecord, @@ -15,6 +19,7 @@ import { withDispatchLock, } from "./store"; import type { DispatchRecord } from "./types"; +import { recoverAuthorizationCompletedAgentTurns } from "@/chat/services/agent-continue"; const DEFAULT_RECOVERY_LIMIT = 25; const DEFAULT_PLUGIN_LIMIT = 25; @@ -26,6 +31,8 @@ function isStaleDispatch(args: { record: { lastCallbackAtMs?: number; leaseExpiresAtMs?: number; + nextCallbackKind?: "delivery"; + nextCallbackAtMs?: number; status: string; }; }): boolean { @@ -35,6 +42,15 @@ function isStaleDispatch(args: { args.record.leaseExpiresAtMs <= args.nowMs ); } + if ( + args.record.status === "awaiting_resume" && + args.record.nextCallbackKind === "delivery" + ) { + return ( + typeof args.record.nextCallbackAtMs !== "number" || + args.record.nextCallbackAtMs <= args.nowMs + ); + } if (args.record.status === "awaiting_resume") { return ( typeof args.record.leaseExpiresAtMs !== "number" || @@ -94,6 +110,7 @@ async function runWithTimeout( export async function recoverStaleDispatches(args: { limit?: number; nowMs: number; + recoverableSlackDelivery?: RecoverableSlackDelivery; }): Promise { const ids = await listIncompleteDispatchIds(); let recovered = 0; @@ -106,26 +123,62 @@ export async function recoverStaleDispatches(args: { continue; } try { + let recoveryRecord = record; if (!isStaleDispatch({ record, nowMs: args.nowMs })) { continue; } - if (record.createdAtMs + DISPATCH_MAX_AGE_MS <= args.nowMs) { - await failDispatch({ - record, - errorMessage: "Dispatch expired before completion.", - }); - continue; - } - if (record.attempt >= record.maxAttempts) { - await failDispatch({ - record, - errorMessage: "Dispatch exceeded retry attempts.", + const canonicalTerminal = + await args.recoverableSlackDelivery?.loadTerminalOutcome({ + conversationId: getDispatchConversationId(record), + turnId: getDispatchTurnId(record.id), + acceptanceEvidence: "known_outbox_intent", }); - continue; + if (canonicalTerminal) { + const terminalRecovery = await withDispatchLock( + record.id, + async (state) => { + const current = parseDispatchRecord( + await state.get(getDispatchStorageKey(record.id)), + ); + if (!current || current.version !== record.version) { + return undefined; + } + return await updateDispatchRecord(state, { + ...current, + leaseExpiresAtMs: undefined, + nextCallbackAtMs: args.nowMs, + nextCallbackKind: "delivery", + status: "awaiting_resume", + }); + }, + ); + if (!terminalRecovery) continue; + recoveryRecord = terminalRecovery; + } else { + if (record.createdAtMs + DISPATCH_MAX_AGE_MS <= args.nowMs) { + await failDispatch({ + record, + errorMessage: "Dispatch expired before completion.", + }); + continue; + } + if ( + record.nextCallbackKind !== "delivery" && + record.attempt >= record.maxAttempts + ) { + await failDispatch({ + record, + errorMessage: "Dispatch exceeded retry attempts.", + }); + continue; + } } await scheduleDispatchCallback({ - id: record.id, - expectedVersion: record.version, + id: recoveryRecord.id, + expectedVersion: recoveryRecord.version, + ...(recoveryRecord.nextCallbackKind === "delivery" + ? { kind: "delivery" as const } + : {}), }); recovered += 1; } catch (error) { @@ -199,11 +252,27 @@ export async function runPluginHeartbeats(args: { export async function runHeartbeat(args: { conversationWorkQueue?: ConversationWorkQueue; nowMs: number; + recoverableSlackDelivery?: RecoverableSlackDelivery; }): Promise { + const conversationWorkQueue = + args.conversationWorkQueue ?? getVercelConversationWorkQueue(); + await recoverAuthorizationCompletedAgentTurns({ + nowMs: args.nowMs, + queue: conversationWorkQueue, + }); await recoverConversationWork({ nowMs: args.nowMs, - queue: args.conversationWorkQueue ?? getVercelConversationWorkQueue(), + queue: conversationWorkQueue, + }); + if (args.recoverableSlackDelivery) { + await recoverDueSlackDeliveries({ + delivery: args.recoverableSlackDelivery, + nowMs: args.nowMs, + }); + } + await recoverStaleDispatches({ + nowMs: args.nowMs, + recoverableSlackDelivery: args.recoverableSlackDelivery, }); - await recoverStaleDispatches({ nowMs: args.nowMs }); await runPluginHeartbeats({ nowMs: args.nowMs }); } diff --git a/packages/junior/src/chat/agent-dispatch/runner.ts b/packages/junior/src/chat/agent-dispatch/runner.ts index 0edb2bc67..25f2559cd 100644 --- a/packages/junior/src/chat/agent-dispatch/runner.ts +++ b/packages/junior/src/chat/agent-dispatch/runner.ts @@ -6,7 +6,15 @@ * calls the same agent boundary as Slack replies, persists visible result * state, and schedules follow-up slices when a turn needs to continue. */ -import { botConfig } from "@/chat/config"; +import { + botConfig, + FUNCTION_TIMEOUT_BUFFER_SECONDS, + getChatConfig, +} from "@/chat/config"; +import { + agentExecutionBudgetMs, + dispatchSliceLeaseMs, +} from "@/function-duration"; import type { AgentRunResult } from "@/chat/services/turn-result"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; import { logException } from "@/chat/logging"; @@ -37,23 +45,23 @@ import { persistThreadStateById, } from "@/chat/runtime/thread-state"; import { getStateAdapter } from "@/chat/state/adapter"; -import { - planSlackReplyPosts, - postSlackApiReplyPosts, -} from "@/chat/slack/reply"; +import { planSlackReplyPosts } from "@/chat/slack/reply"; import { buildSlackReplyFooter } from "@/chat/slack/footer"; +import { createSlackDeliveryLocator } from "@/chat/slack/outbound"; +import type { RecoverableSlackDelivery } from "@/chat/slack/recoverable-delivery"; +import { buildPendingSlackDeliveryCommandDraft } from "@/chat/slack/delivery-command"; +import { buildDeterministicAssistantMessageId } from "@/chat/state/turn-id"; import { finalizeFailedTurnReplyWithEvent } from "@/chat/services/turn-failure-response"; import { completeDeliveredTurn } from "@/chat/services/turn-session-record"; -import { - ConversationTurnLifecycleService, - type ConversationTurnLifecycle, -} from "@/chat/conversations/turn-lifecycle"; -import type { ConversationTurnFailureCode } from "@/chat/conversations/history"; -import { getConversationEventStore } from "@/chat/db"; +import { advanceSlackDeliveryWithTerminalRepair } from "@/chat/runtime/slack-delivery-recovery"; import { persistWithRetry } from "@/chat/services/persist-retry"; import { AuthorizationFlowDisabledError } from "@/chat/services/auth-pause"; import { PluginCredentialFailureError } from "@/chat/services/plugin-auth-orchestration"; import { scheduleSessionCompletedPluginTasks } from "@/chat/plugins/task-runner"; +import type { + ConversationTurnLifecycle, + StartConversationTurnInput, +} from "@/chat/conversations/turn-lifecycle"; import { scheduleDispatchCallback } from "./signing"; import { getDispatchConversationId, @@ -67,20 +75,30 @@ import { } from "./store"; import type { DispatchCallback, DispatchRecord } from "./types"; -const DISPATCH_SLICE_LEASE_MS = 5 * 60 * 1000; - export interface AgentDispatchRunnerDeps { agentRunner: AgentRunner; - turnLifecycle?: ConversationTurnLifecycle; + functionMaxDurationSeconds?: number; + recoverableSlackDelivery: RecoverableSlackDelivery; + turnLifecycle: ConversationTurnLifecycle; scheduleCallback?: typeof scheduleDispatchCallback; scheduleSessionCompletedPluginTasks?: typeof scheduleSessionCompletedPluginTasks; } +interface CanonicalDispatchTerminalProjection { + errorMessage?: string; + resultMessageTs?: string; + status: "completed" | "failed"; +} + function getUserMessageId(dispatch: DispatchRecord): string { return `dispatch:${dispatch.id}:user`; } function getAssistantMessageId(dispatch: DispatchRecord): string { + return buildDeterministicAssistantMessageId(getDispatchTurnId(dispatch.id)); +} + +function getLegacyAssistantMessageId(dispatch: DispatchRecord): string { return `dispatch:${dispatch.id}:assistant`; } @@ -149,6 +167,8 @@ async function persistRuntimePatch(args: { async function markDispatch(args: { dispatch: DispatchRecord; errorMessage?: string; + nextCallbackAtMs?: number; + nextCallbackKind?: "delivery"; resultMessageTs?: string; status: DispatchRecord["status"]; }): Promise { @@ -162,6 +182,8 @@ async function markDispatch(args: { return await updateDispatchRecord(state, { ...current, status: args.status, + nextCallbackAtMs: args.nextCallbackAtMs, + nextCallbackKind: args.nextCallbackKind, ...(args.errorMessage ? { errorMessage: args.errorMessage } : {}), ...(args.resultMessageTs ? { resultMessageTs: args.resultMessageTs } @@ -170,6 +192,26 @@ async function markDispatch(args: { }); } +async function parkPendingDelivery(args: { + dispatch: DispatchRecord; + retryAtMs: number; + schedule: typeof scheduleDispatchCallback; +}): Promise { + const pending = await markDispatch({ + dispatch: args.dispatch, + status: "awaiting_resume", + nextCallbackAtMs: args.retryAtMs, + nextCallbackKind: "delivery", + }); + if (args.retryAtMs <= Date.now()) { + await args.schedule({ + id: pending.id, + expectedVersion: pending.version, + kind: "delivery", + }); + } +} + function canClaimDispatch(record: DispatchRecord, nowMs: number): boolean { if (isTerminalDispatchStatus(record.status)) { return false; @@ -187,8 +229,8 @@ function canClaimDispatch(record: DispatchRecord, nowMs: number): boolean { return true; } -/** Run one serverless slice for a core-owned agent dispatch. */ -export async function runAgentDispatchSlice( +/** Process one authenticated callback as a serverless dispatch execution slice. */ +export async function processAgentDispatchCallback( callback: DispatchCallback, deps: AgentDispatchRunnerDeps, ): Promise { @@ -196,26 +238,42 @@ export async function runAgentDispatchSlice( const scheduleCompletedTasks = deps.scheduleSessionCompletedPluginTasks ?? scheduleSessionCompletedPluginTasks; - const turnLifecycle = - deps.turnLifecycle ?? - new ConversationTurnLifecycleService(getConversationEventStore()); const nowMs = Date.now(); + const functionMaxDurationSeconds = + deps.functionMaxDurationSeconds ?? + getChatConfig().functionMaxDurationSeconds; + const sliceLeaseMs = dispatchSliceLeaseMs( + functionMaxDurationSeconds, + FUNCTION_TIMEOUT_BUFFER_SECONDS, + ); + const turnTimeoutMs = agentExecutionBudgetMs( + functionMaxDurationSeconds, + FUNCTION_TIMEOUT_BUFFER_SECONDS, + ); + const turnDeadlineAtMs = nowMs + turnTimeoutMs; + const isDeliveryCallback = callback.kind === "delivery"; const claimedDispatch = await withDispatchLock(callback.id, async (state) => { const current = parseDispatchRecord( await state.get(getDispatchStorageKey(callback.id)), ); if ( !current || - !canClaimDispatch(current, nowMs) || - current.version !== callback.expectedVersion + current.version !== callback.expectedVersion || + (isDeliveryCallback + ? current.nextCallbackKind !== "delivery" + : !canClaimDispatch(current, nowMs)) ) { return undefined; } return await updateDispatchRecord(state, { ...current, lastCallbackAtMs: nowMs, - leaseExpiresAtMs: nowMs + DISPATCH_SLICE_LEASE_MS, - status: "running", + ...(isDeliveryCallback + ? {} + : { + leaseExpiresAtMs: nowMs + sliceLeaseMs, + status: "running" as const, + }), }); }); if (!claimedDispatch) { @@ -236,24 +294,137 @@ export async function runAgentDispatchSlice( }; const destinationLockId = getDispatchDestinationLockId(dispatch.destination); const stateAdapter = getStateAdapter(); - let lifecycleStarted = false; + let lifecycleStart: StartConversationTurnInput | undefined; let lifecycleTerminalized = false; - let failureCode: ConversationTurnFailureCode = "persistence_failed"; + let canonicalTerminalProjection: + | CanonicalDispatchTerminalProjection + | undefined; + let failureCode: + | "agent_run_failed" + | "delivery_failed" + | "persistence_failed" = "agent_run_failed"; await stateAdapter.connect(); const destinationLock = await stateAdapter.acquireLock( destinationLockId, - DISPATCH_SLICE_LEASE_MS, + sliceLeaseMs, ); if (!destinationLock) { - await markDispatch({ - dispatch, - status: "pending", - errorMessage: "Destination conversation is busy", - }); + if (isDeliveryCallback) { + await parkPendingDelivery({ + dispatch, + retryAtMs: Date.now() + 60_000, + schedule: scheduleCallback, + }); + } else { + await markDispatch({ + dispatch, + status: "pending", + errorMessage: "Destination conversation is busy", + }); + } return; } try { + const persisted = await getPersistedThreadState(conversationId); + const conversation = coerceThreadConversationState(persisted); + await hydrateConversationMessages({ conversation, conversationId }); + const pendingDelivery = await deps.recoverableSlackDelivery.loadByTurn({ + conversationId, + turnId, + }); + if (pendingDelivery) { + const recovered = await advanceSlackDeliveryWithTerminalRepair({ + delivery: deps.recoverableSlackDelivery, + intent: pendingDelivery, + }); + if (recovered.outcome === "pending") { + await parkPendingDelivery({ + dispatch, + retryAtMs: recovered.retryAtMs, + schedule: scheduleCallback, + }); + return; + } + await hydrateConversationMessages({ conversation, conversationId }); + } + const priorTerminal = + await deps.recoverableSlackDelivery.loadTerminalOutcome({ + conversationId, + turnId, + acceptanceEvidence: isDeliveryCallback + ? "known_outbox_intent" + : "visible_assistant", + }); + const deliveredMessage = conversation.messages.find( + (message) => + (message.id === getAssistantMessageId(dispatch) || + message.id === getLegacyAssistantMessageId(dispatch)) && + message.meta?.replied === true && + typeof message.meta.slackTs === "string", + ); + if (priorTerminal) { + canonicalTerminalProjection = { + status: priorTerminal.modelSucceeded ? "completed" : "failed", + ...(typeof deliveredMessage?.meta?.slackTs === "string" + ? { resultMessageTs: deliveredMessage.meta.slackTs } + : {}), + }; + lifecycleStart = { + conversationId, + turnId, + createdAtMs: nowMs, + inputMessageIds: [getUserMessageId(dispatch)], + surface: "api", + }; + await deps.turnLifecycle.start(lifecycleStart); + lifecycleTerminalized = true; + await persistRuntimePatch({ + threadId: conversationId, + conversation, + artifacts: coerceThreadArtifactsState(persisted), + }); + await markDispatch({ + dispatch, + ...canonicalTerminalProjection, + }); + return; + } + if (deliveredMessage) { + lifecycleStart = { + conversationId, + turnId, + createdAtMs: nowMs, + inputMessageIds: [getUserMessageId(dispatch)], + surface: "api", + }; + await deps.turnLifecycle.start(lifecycleStart); + await deps.turnLifecycle.fail({ + conversationId, + turnId, + createdAtMs: Date.now(), + failureCode: "persistence_failed", + }); + lifecycleTerminalized = true; + canonicalTerminalProjection = { + status: "completed", + resultMessageTs: deliveredMessage.meta!.slackTs!, + }; + await persistRuntimePatch({ + threadId: conversationId, + conversation, + artifacts: coerceThreadArtifactsState(persisted), + }); + await markDispatch({ + dispatch, + ...canonicalTerminalProjection, + }); + return; + } + if (isDeliveryCallback) { + throw new Error("Delivery callback has no pending delivery outcome"); + } + const startedDispatch = await withDispatchLock( dispatch.id, async (state) => { @@ -274,29 +445,9 @@ export async function runAgentDispatchSlice( }); }, ); - if (!startedDispatch) { - return; - } + if (!startedDispatch) return; dispatch = startedDispatch; - const persisted = await getPersistedThreadState(conversationId); - const conversation = coerceThreadConversationState(persisted); - await hydrateConversationMessages({ conversation, conversationId }); - const deliveredMessage = conversation.messages.find( - (message) => - message.id === getAssistantMessageId(dispatch) && - message.meta?.replied === true && - typeof message.meta.slackTs === "string", - ); - if (typeof deliveredMessage?.meta?.slackTs === "string") { - await markDispatch({ - dispatch, - status: "completed", - resultMessageTs: deliveredMessage.meta.slackTs, - }); - return; - } - let artifacts = coerceThreadArtifactsState(persisted); let sandboxId = typeof persisted.app_sandbox_id === "string" @@ -316,15 +467,14 @@ export async function runAgentDispatchSlice( nowMs, }); await persistConversationMessages({ conversation, conversationId }); - await turnLifecycle.start({ + lifecycleStart = { conversationId, turnId, createdAtMs: nowMs, inputMessageIds: [userMessageId], surface: "api", - }); - lifecycleStarted = true; - failureCode = "agent_run_failed"; + }; + await deps.turnLifecycle.start(lifecycleStart); const conversationContext = buildConversationContext(conversation, { excludeMessageId: userMessageId, }); @@ -366,6 +516,8 @@ export async function runAgentDispatchSlice( authorizationFlowMode: "disabled", configuration, channelConfiguration, + turnDeadlineAtMs, + turnTimeoutMs, }, state: { artifactState: artifacts, @@ -399,7 +551,7 @@ export async function runAgentDispatchSlice( }, }); if (outcome.status === "awaiting_auth") { - await turnLifecycle.fail({ + await deps.turnLifecycle.fail({ conversationId, turnId, createdAtMs: Date.now(), @@ -433,7 +585,7 @@ export async function runAgentDispatchSlice( ? undefined : (reply.diagnostics.errorMessage ?? `Agent turn ended with ${reply.diagnostics.outcome}.`); - let modelFailureEventId: string | undefined; + let finalizedFailureEventId: string | undefined; if (failure) { const finalized = finalizeFailedTurnReplyWithEvent({ reply, @@ -444,54 +596,103 @@ export async function runAgentDispatchSlice( }, }); reply = finalized.reply; - modelFailureEventId = finalized.eventId; + finalizedFailureEventId = finalized.eventId; } const deliveryReply = ensureVisibleDeliveryText(reply); - failureCode = "delivery_failed"; - const resultMessageTs = await postSlackApiReplyPosts({ - channelId: dispatch.destination.channelId, - posts: planSlackReplyPosts({ reply: deliveryReply }), - footer: buildSlackReplyFooter({ + const plannedPosts = planSlackReplyPosts({ reply: deliveryReply }).filter( + (post) => post.text.trim().length > 0, + ); + let resultMessageTs: string | undefined; + let deliveryTerminalized = false; + if (plannedPosts.length > 0) { + failureCode = "delivery_failed"; + const footer = buildSlackReplyFooter({ conversationId }); + const intent = await deps.recoverableSlackDelivery.createIntent({ conversationId, - }), - }); - failureCode = "persistence_failed"; - - // Slack accepted the reply: everything after this point serves duplicate - // suppression and bookkeeping, and must not turn into a failed dispatch - // that a retry would re-post. Persist the delivered marker - // (`meta.slackTs`, checked by the redelivery guard above) immediately and - // durably before the dispatch is marked terminal so the crash window - // between post and marker stays as small as possible. The retry-and-swallow - // `persistRuntimePatch` below appends canonical visible-message facts, so - // no separate transcript persist runs outside that guarded block. - markConversationMessage(conversation, userMessageId, { - replied: true, - skippedReason: undefined, - }); - if (reply.deliveryPlan?.postThreadText !== false) { - upsertConversationMessage(conversation, { - id: getAssistantMessageId(dispatch), - role: "assistant", - text: - normalizeConversationText(deliveryReply.text) || "[empty response]", - createdAtMs: nowMs, - author: { - userName: botConfig.userName, - isBot: true, - }, - meta: { - replied: true, - slackTs: resultMessageTs, - }, + turnId, + deliveryId: `slack:${turnId}`, + modelMessages: reply.piMessages ?? [], + command: buildPendingSlackDeliveryCommandDraft({ + route: { channelId: dispatch.destination.channelId }, + publicLocator: createSlackDeliveryLocator(), + session: { + surface: "api", + source: dispatch.source, + destination: dispatch.destination, + destinationVisibility: dispatch.destinationVisibility, + actor: dispatch.actor, + startedAtMs: dispatch.createdAtMs, + }, + posts: plannedPosts, + footer, + turnId, + inputMessageIds: [userMessageId], + assistantText: + normalizeConversationText(deliveryReply.text) || "[empty response]", + assistantCreatedAtMs: nowMs, + assistantUserName: botConfig.userName, + diagnostics: reply.diagnostics, + sliceId: 1, + failureEventId: finalizedFailureEventId, + }), + }); + const delivery = await advanceSlackDeliveryWithTerminalRepair({ + delivery: deps.recoverableSlackDelivery, + intent, + }); + if (delivery.outcome === "pending") { + await parkPendingDelivery({ + dispatch, + retryAtMs: delivery.retryAtMs, + schedule: scheduleCallback, + }); + return; + } + lifecycleTerminalized = true; + deliveryTerminalized = true; + canonicalTerminalProjection = + delivery.outcome === "failed" + ? { + status: "failed", + errorMessage: "Slack rejected the dispatched reply.", + } + : { + status: failure ? "failed" : "completed", + ...(failure ? { errorMessage: failure } : {}), + ...(delivery.messageTs + ? { resultMessageTs: delivery.messageTs } + : {}), + }; + await hydrateConversationMessages({ conversation, conversationId }); + if (delivery.outcome === "failed") { + await persistRuntimePatch({ + threadId: conversationId, + conversation, + artifacts, + sandboxId, + sandboxDependencyProfileHash, + }); + await markDispatch({ + dispatch, + ...canonicalTerminalProjection, + }); + return; + } + resultMessageTs = delivery.messageTs; + failureCode = "persistence_failed"; + } else { + markConversationMessage(conversation, userMessageId, { + replied: true, + skippedReason: undefined, }); } updateConversationStats(conversation); const nextArtifacts = reply.artifactStatePatch ? mergeArtifactsState(artifacts, reply.artifactStatePatch) : artifacts; - let statePersisted = false; + let postDeliveryPersistenceFailed = false; + let postDeliveryPersistenceEventId: string | undefined; try { await persistWithRetry(() => persistRuntimePatch({ @@ -503,25 +704,17 @@ export async function runAgentDispatchSlice( reply.sandboxDependencyProfileHash ?? sandboxDependencyProfileHash, }), ); - statePersisted = true; } catch (persistError) { - const eventId = logException( + postDeliveryPersistenceFailed = true; + postDeliveryPersistenceEventId = logException( persistError, "agent_dispatch_post_delivery_persist_failed", logContext, {}, "Failed to persist delivered dispatch state after Slack accepted the reply", ); - await turnLifecycle.fail({ - conversationId, - turnId, - createdAtMs: Date.now(), - failureCode: "persistence_failed", - ...(eventId ? { eventId } : {}), - }); - lifecycleTerminalized = true; } - if (reply.piMessages?.length) { + if (reply.piMessages?.length && !deliveryTerminalized) { // Destination acceptance is the completion boundary for the session // record too; this call swallows its own persistence failures. await completeDeliveredTurn({ @@ -546,32 +739,47 @@ export async function runAgentDispatchSlice( }, }); } - if (statePersisted) { - if (failure) { - await turnLifecycle.fail({ - conversationId, - turnId, - createdAtMs: Date.now(), - failureCode: "model_execution_failed", - ...(modelFailureEventId ? { eventId: modelFailureEventId } : {}), - }); - } else { - await turnLifecycle.complete({ - conversationId, - turnId, - createdAtMs: Date.now(), - outcome: "success", - }); - } + if (deliveryTerminalized) { lifecycleTerminalized = true; + } else if (postDeliveryPersistenceFailed) { + await deps.turnLifecycle.fail({ + conversationId, + turnId, + createdAtMs: Date.now(), + failureCode: "persistence_failed", + ...(postDeliveryPersistenceEventId + ? { eventId: postDeliveryPersistenceEventId } + : {}), + }); + } else if (failure) { + await deps.turnLifecycle.fail({ + conversationId, + turnId, + createdAtMs: Date.now(), + failureCode: "model_execution_failed", + ...(finalizedFailureEventId + ? { eventId: finalizedFailureEventId } + : {}), + }); + } else { + await deps.turnLifecycle.complete({ + conversationId, + turnId, + createdAtMs: Date.now(), + outcome: shouldPostDispatchReplyText(reply) ? "success" : "no_reply", + }); } - dispatch = await markDispatch({ - dispatch, + lifecycleTerminalized = true; + canonicalTerminalProjection = { status: failure ? "failed" : "completed", ...(failure ? { errorMessage: failure } : {}), - resultMessageTs, + ...(resultMessageTs ? { resultMessageTs } : {}), + }; + dispatch = await markDispatch({ + dispatch, + ...canonicalTerminalProjection, }); - if (!failure) { + if (!failure && !postDeliveryPersistenceFailed) { try { await scheduleCompletedTasks({ conversationId, @@ -588,9 +796,42 @@ export async function runAgentDispatchSlice( } } } catch (error) { + const unresolvedDelivery = await deps.recoverableSlackDelivery.loadByTurn({ + conversationId, + turnId, + }); + if (unresolvedDelivery) { + logException( + error, + "agent_dispatch_delivery_advance_failed", + logContext, + {}, + "Failed to advance durable dispatch delivery", + ); + await parkPendingDelivery({ + dispatch, + retryAtMs: Math.max( + unresolvedDelivery.nextAttemptAtMs, + Date.now() + 5_000, + ), + schedule: scheduleCallback, + }); + return; + } + if (canonicalTerminalProjection) { + logException( + error, + "agent_dispatch_terminal_projection_failed", + logContext, + {}, + "Failed to repair or project a canonically terminal dispatch", + ); + return; + } if (error instanceof AuthorizationFlowDisabledError) { - if (lifecycleStarted && !lifecycleTerminalized) { - await turnLifecycle.fail({ + if (lifecycleStart && !lifecycleTerminalized) { + await deps.turnLifecycle.start(lifecycleStart); + await deps.turnLifecycle.fail({ conversationId, turnId, createdAtMs: Date.now(), @@ -606,8 +847,9 @@ export async function runAgentDispatchSlice( return; } if (error instanceof PluginCredentialFailureError) { - if (lifecycleStarted && !lifecycleTerminalized) { - await turnLifecycle.fail({ + if (lifecycleStart && !lifecycleTerminalized) { + await deps.turnLifecycle.start(lifecycleStart); + await deps.turnLifecycle.fail({ conversationId, turnId, createdAtMs: Date.now(), @@ -632,8 +874,9 @@ export async function runAgentDispatchSlice( {}, "Agent dispatch failed", ); - if (lifecycleStarted && !lifecycleTerminalized) { - await turnLifecycle.fail({ + if (lifecycleStart && !lifecycleTerminalized) { + await deps.turnLifecycle.start(lifecycleStart); + await deps.turnLifecycle.fail({ conversationId, turnId, createdAtMs: Date.now(), diff --git a/packages/junior/src/chat/agent-dispatch/signing.ts b/packages/junior/src/chat/agent-dispatch/signing.ts index 78a3cc646..204e993d9 100644 --- a/packages/junior/src/chat/agent-dispatch/signing.ts +++ b/packages/junior/src/chat/agent-dispatch/signing.ts @@ -1,8 +1,8 @@ import { createHmac, timingSafeEqual } from "node:crypto"; import { resolveBaseUrl } from "@/chat/oauth-flow"; +import { JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE } from "@/deployment"; import type { DispatchCallback } from "./types"; -const DISPATCH_CALLBACK_PATH = "/api/internal/agent-dispatch"; const DISPATCH_HMAC_CONTEXT = "junior.agent_dispatch.v1"; const DISPATCH_SIGNATURE_VERSION = "v1"; const DISPATCH_MAX_SKEW_MS = 5 * 60 * 1000; @@ -41,13 +41,15 @@ function parseDispatchCallback(value: unknown): DispatchCallback | undefined { const record = value as Record; if ( typeof record.id !== "string" || - typeof record.expectedVersion !== "number" + typeof record.expectedVersion !== "number" || + (record.kind !== undefined && record.kind !== "delivery") ) { return undefined; } return { id: record.id, expectedVersion: record.expectedVersion, + ...(record.kind === "delivery" ? { kind: "delivery" as const } : {}), }; } @@ -71,16 +73,19 @@ export async function scheduleDispatchCallback( const body = JSON.stringify(callback); const timestamp = Date.now().toString(); - const response = await fetch(`${baseUrl}${DISPATCH_CALLBACK_PATH}`, { - method: "POST", - headers: { - "content-type": "application/json", - [DISPATCH_TIMESTAMP_HEADER]: timestamp, - [DISPATCH_SIGNATURE_HEADER]: signBody(secret, timestamp, body), + const response = await fetch( + `${baseUrl}${JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE}`, + { + method: "POST", + headers: { + "content-type": "application/json", + [DISPATCH_TIMESTAMP_HEADER]: timestamp, + [DISPATCH_SIGNATURE_HEADER]: signBody(secret, timestamp, body), + }, + signal: AbortSignal.timeout(DISPATCH_CALLBACK_TIMEOUT_MS), + body, }, - signal: AbortSignal.timeout(DISPATCH_CALLBACK_TIMEOUT_MS), - body, - }); + ); if (!response.ok) { throw new Error( `Agent dispatch callback failed with status ${response.status}`, diff --git a/packages/junior/src/chat/agent-dispatch/store.ts b/packages/junior/src/chat/agent-dispatch/store.ts index 626f69e78..8a5014b02 100644 --- a/packages/junior/src/chat/agent-dispatch/store.ts +++ b/packages/junior/src/chat/agent-dispatch/store.ts @@ -62,6 +62,8 @@ const dispatchRecordSchema = z leaseExpiresAtMs: z.number().finite().optional(), maxAttempts: z.number().int().positive(), metadata: z.record(z.string(), z.string()).optional(), + nextCallbackKind: z.literal("delivery").optional(), + nextCallbackAtMs: z.number().finite().optional(), plugin: nonEmptyExactStringSchema, resultMessageTs: z.string().optional(), source: sourceSchema, diff --git a/packages/junior/src/chat/agent-dispatch/types.ts b/packages/junior/src/chat/agent-dispatch/types.ts index 3ffe1ae34..1ded32888 100644 --- a/packages/junior/src/chat/agent-dispatch/types.ts +++ b/packages/junior/src/chat/agent-dispatch/types.ts @@ -44,6 +44,8 @@ export interface DispatchRecord { maxAttempts: number; metadata?: Record; plugin: string; + nextCallbackKind?: "delivery"; + nextCallbackAtMs?: number; resultMessageTs?: string; source: Source; status: DispatchStatus; @@ -61,6 +63,7 @@ export interface DispatchProjection { export interface DispatchCallback { expectedVersion: number; id: string; + kind?: "delivery"; } export interface DispatchCreateResult { diff --git a/packages/junior/src/chat/agent/index.ts b/packages/junior/src/chat/agent/index.ts index 101cfa922..8d52dfee8 100644 --- a/packages/junior/src/chat/agent/index.ts +++ b/packages/junior/src/chat/agent/index.ts @@ -191,7 +191,14 @@ async function executeAgentRunInPrivacyContext( assertCorrelationDestinationMatch(routing); const replyStartedAtMs = Date.now(); - const configuredTurnDeadlineAtMs = replyStartedAtMs + botConfig.turnTimeoutMs; + const policyTurnTimeoutMs = + typeof policy.turnTimeoutMs === "number" && + Number.isFinite(policy.turnTimeoutMs) && + policy.turnTimeoutMs >= 0 + ? Math.floor(policy.turnTimeoutMs) + : undefined; + const configuredTurnDeadlineAtMs = + replyStartedAtMs + (policyTurnTimeoutMs ?? botConfig.turnTimeoutMs); const policyTurnDeadlineAtMs = typeof policy.turnDeadlineAtMs === "number" && Number.isFinite(policy.turnDeadlineAtMs) diff --git a/packages/junior/src/chat/agent/request.ts b/packages/junior/src/chat/agent/request.ts index b4fe5226f..3f26009b7 100644 --- a/packages/junior/src/chat/agent/request.ts +++ b/packages/junior/src/chat/agent/request.ts @@ -109,6 +109,8 @@ export interface AgentRunRouting { /** Carries execution limits and dependency overrides for one run slice. */ export interface AgentRunPolicy { + /** Runtime-owned soft timeout for this agent slice, in milliseconds. */ + turnTimeoutMs?: number; /** Absolute wall-clock deadline for this host request, in milliseconds. */ turnDeadlineAtMs?: number; /** Cancels provider work when the owning host request is abandoned. */ diff --git a/packages/junior/src/chat/app/factory.ts b/packages/junior/src/chat/app/factory.ts index cd5457dcd..bb5ca2306 100644 --- a/packages/junior/src/chat/app/factory.ts +++ b/packages/junior/src/chat/app/factory.ts @@ -100,7 +100,10 @@ function upsertSkippedConversationMessage( export function createSlackRuntime( options: CreateSlackRuntimeOptions, ): SlackTurnRuntime { - const services = createJuniorRuntimeServices(options.services); + const services = createJuniorRuntimeServices( + { getSlackAdapter: options.getSlackAdapter }, + options.services, + ); const prepareTurnState = createPrepareTurnState({ compactConversationIfNeeded: services.conversationMemory.compactConversationIfNeeded, diff --git a/packages/junior/src/chat/app/production.ts b/packages/junior/src/chat/app/production.ts index 3d699b1e5..bd0055ab0 100644 --- a/packages/junior/src/chat/app/production.ts +++ b/packages/junior/src/chat/app/production.ts @@ -18,6 +18,8 @@ import { resumeAwaitingSlackContinuation } from "@/chat/runtime/agent-continue-r import type { JuniorRuntimeServiceOverrides } from "@/chat/app/services"; import { getConversationStore } from "@/chat/db"; import type { ConversationStore } from "@/chat/conversations/store"; +import type { ConversationTurnLifecycle } from "@/chat/conversations/turn-lifecycle"; +import type { RecoverableSlackDelivery } from "@/chat/slack/recoverable-delivery"; let productionSlackAdapter: SlackAdapter | undefined; let productionSlackRuntime: ReturnType | undefined; @@ -95,6 +97,8 @@ export function getProductionSlackWebhookServices(): SlackWebhookServices { /** Return the production queue callback options for conversation work. */ export function createProductionConversationWorkOptions(options: { agentRunner: AgentRunner; + recoverableSlackDelivery: RecoverableSlackDelivery; + turnLifecycle: ConversationTurnLifecycle; services?: JuniorRuntimeServiceOverrides; }): VercelConversationWorkCallbackOptions { const conversationStore = getProductionConversationStore(); @@ -121,6 +125,8 @@ export function createProductionConversationWorkOptions(options: { resumeAwaitingContinuation: async (conversationId) => await resumeAwaitingSlackContinuation(conversationId, { agentRunner, + recoverableSlackDelivery: options.recoverableSlackDelivery, + turnLifecycle: options.turnLifecycle, scheduleSessionCompletedPluginTasks: services.replyExecutor?.scheduleSessionCompletedPluginTasks, }), diff --git a/packages/junior/src/chat/app/services.ts b/packages/junior/src/chat/app/services.ts index 651120d86..d4360d4ba 100644 --- a/packages/junior/src/chat/app/services.ts +++ b/packages/junior/src/chat/app/services.ts @@ -1,3 +1,4 @@ +import type { SlackAdapter } from "@chat-adapter/slack"; import { completeObject, completeText } from "@/chat/pi/client"; import { executeAgentRun as executeAgentRunImpl } from "@/chat/agent"; import type { SandboxEgressTracePropagationConfig } from "@/chat/sandbox/egress/tracing"; @@ -31,8 +32,19 @@ import { type VisionContextService, } from "@/chat/slack/vision-context"; import { createAgentRunner } from "@/chat/runtime/agent-runner"; +import { getSqlExecutor } from "@/chat/db"; +import { + postRecoverableSlackMessage, + reconcileRecoverableSlackMessage, +} from "@/chat/slack/outbound"; +import { RecoverableSlackDeliveryService } from "@/chat/slack/recoverable-delivery"; +import type { + RecoverableSlackDelivery, + RecoverableSlackDeliveryPort, +} from "@/chat/slack/recoverable-delivery"; import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; import { getConversationEventStore } from "@/chat/db"; +import { runWithSlackInstallation } from "@/chat/slack/adapter-context"; export interface JuniorRuntimeServices { conversationMemory: ConversationMemoryService; @@ -53,7 +65,38 @@ export interface JuniorRuntimeServiceOverrides { visionContext?: Partial; } +/** Bind recoverable provider calls to the persisted destination installation. */ +export function createInstallationBoundRecoverableSlackDeliveryPort(args: { + getSlackAdapter: () => SlackAdapter; +}): RecoverableSlackDeliveryPort { + return { + post: async ({ teamId, ...input }) => + await runWithSlackInstallation({ + adapter: args.getSlackAdapter(), + installation: { teamId }, + task: async () => await postRecoverableSlackMessage(input), + }), + reconcile: async ({ teamId, ...input }) => + await runWithSlackInstallation({ + adapter: args.getSlackAdapter(), + installation: { teamId }, + task: async () => await reconcileRecoverableSlackMessage(input), + }), + }; +} + +/** Compose the production SQL and Slack ports for durable reply delivery. */ +export function createProductionRecoverableSlackDelivery(args: { + getSlackAdapter: () => SlackAdapter; +}): RecoverableSlackDelivery { + return new RecoverableSlackDeliveryService( + getSqlExecutor(), + createInstallationBoundRecoverableSlackDeliveryPort(args), + ); +} + export function createJuniorRuntimeServices( + options: { getSlackAdapter: () => SlackAdapter }, overrides: JuniorRuntimeServiceOverrides = {}, ): JuniorRuntimeServices { const conversationMemory = createConversationMemoryService({ @@ -95,6 +138,9 @@ export function createJuniorRuntimeServices( (async (params) => { await scheduleSessionCompletedPluginTasks(params); }), + recoverableSlackDelivery: + overrides.replyExecutor?.recoverableSlackDelivery ?? + createProductionRecoverableSlackDelivery(options), turnLifecycle: overrides.replyExecutor?.turnLifecycle ?? new ConversationTurnLifecycleService(getConversationEventStore()), diff --git a/packages/junior/src/chat/capabilities/factory.ts b/packages/junior/src/chat/capabilities/factory.ts index baf062e31..2b580822c 100644 --- a/packages/junior/src/chat/capabilities/factory.ts +++ b/packages/junior/src/chat/capabilities/factory.ts @@ -17,7 +17,7 @@ const sandboxEgressRouters = new WeakMap< >(); /** Create the user token store used by OAuth-backed credential brokers. */ -export function createUserTokenStore(): UserTokenStore { +export function createUserTokenStore(): StateAdapterTokenStore { return new StateAdapterTokenStore(getStateAdapter()); } diff --git a/packages/junior/src/chat/config.ts b/packages/junior/src/chat/config.ts index 6ec9652c8..9ee975784 100644 --- a/packages/junior/src/chat/config.ts +++ b/packages/junior/src/chat/config.ts @@ -11,11 +11,11 @@ import { modelProfileSchema, STANDARD_MODEL_PROFILE, } from "@/chat/model-profile"; +import { resolveConfiguredFunctionMaxDurationSeconds } from "@/function-duration"; const MIN_AGENT_TURN_TIMEOUT_MS = 10 * 1000; const DEFAULT_AGENT_TURN_TIMEOUT_MS = 12 * 60 * 1000; const MAX_SLICES_PER_TURN = 100; -const DEFAULT_FUNCTION_MAX_DURATION_SECONDS = 300; const DEFAULT_SLACK_SLASH_COMMAND = "/jr"; const DEFAULT_PROCESSING_REACTION_EMOJI = "eyes"; const DEFAULT_COMPLETED_REACTION_EMOJI = "white_check_mark"; @@ -92,17 +92,6 @@ function parseAgentTurnTimeoutMs( return Math.max(MIN_AGENT_TURN_TIMEOUT_MS, Math.min(value, maxTimeoutMs)); } -function resolveFunctionMaxDurationSeconds(env: NodeJS.ProcessEnv): number { - const raw = - env.FUNCTION_MAX_DURATION_SECONDS ?? - env.QUEUE_CALLBACK_MAX_DURATION_SECONDS; - const value = Number.parseInt(raw ?? "", 10); - if (Number.isNaN(value) || value <= 0) { - return DEFAULT_FUNCTION_MAX_DURATION_SECONDS; - } - return value; -} - function resolveMaxTurnTimeoutMs(functionMaxDurationSeconds: number): number { const budgetSeconds = functionMaxDurationSeconds - FUNCTION_TIMEOUT_BUFFER_SECONDS; @@ -248,7 +237,8 @@ function parseReactionEmoji( } function readBotConfig(env: NodeJS.ProcessEnv): BotConfig { - const functionMaxDurationSeconds = resolveFunctionMaxDurationSeconds(env); + const functionMaxDurationSeconds = + resolveConfiguredFunctionMaxDurationSeconds(env); const maxTurnTimeoutMs = resolveMaxTurnTimeoutMs(functionMaxDurationSeconds); const modelId = validateGatewayModelId(env.AI_MODEL) ?? DEFAULT_MODEL_ID; const reasoningLevel = toOptionalTrimmed(env.AI_REASONING_LEVEL); @@ -326,7 +316,8 @@ export function readChatConfig( const databaseUrl = readDatabaseUrl(env); return { bot: readBotConfig(env), - functionMaxDurationSeconds: resolveFunctionMaxDurationSeconds(env), + functionMaxDurationSeconds: + resolveConfiguredFunctionMaxDurationSeconds(env), sql: { databaseUrl, driver: readSqlDriver(env, databaseUrl), diff --git a/packages/junior/src/chat/conversations/README.md b/packages/junior/src/chat/conversations/README.md index cb6627600..e3ad3c5f0 100644 --- a/packages/junior/src/chat/conversations/README.md +++ b/packages/junior/src/chat/conversations/README.md @@ -116,20 +116,42 @@ conversation storage component tests. Local, ordinary and resumed Slack, and dispatch runtimes write lifecycle events, and detail reporting reduces `turn_failed` to one privacy-safe error -marker. Crash-safe Slack delivery and continuation recovery remain follow-up -work. +marker. Ordinary Slack replies now use the durable delivery boundary; resumed +and dispatched recovery are separate runtime integrations. The structural failure marker never exposes failure code or event ID. An independently delivered fallback remains ordinary visible content, so a public conversation preserves its approved `event_id` reference while private detail redaction removes the fallback text and retains only structural metadata. -Lifecycle appends have stable idempotency keys so explicitly retried calls are -safe, but they are not an outbox transaction with an external destination. A -process death after destination acceptance and before the -visible/session/terminal writes can still leave a started turn without a -terminal event. The next delivery slice must add durable intent/receipt -reconciliation for Slack before claiming crash-safe terminality. +Lifecycle appends have stable idempotency keys. Ordinary concrete Slack final +replies additionally close the external-acceptance gap through the pending +delivery outbox; private authorization, continuation, and nonstandard notice +paths are not yet covered by that outbox. + +`junior_pending_deliveries` is deletable control state for closing that gap; it +is not a second history API. Each row retains one strict finalized Slack reply +command, one retry schedule, fenced lease state, ordered provider receipts, and +the current part's posting or reconciliation state. At most one unresolved row +exists per conversation, so newer input cannot bypass older delivery. The +durable contract currently admits only ordinary Slack assistant replies +correlated to their owning turn; other delivery families stay in their owning +paths until they have a live producer and recovery design. A stale `posting` +state becomes `uncertain` and cannot be posted again until reconciliation +explicitly marks it repostable after its grace period. Terminalization persists +the canonical turn terminal and deletes the pending row in one transaction. +That turn terminal is also the post-commit delivery authority: only +`turn_failed` with `delivery_failed` means Slack rejected delivery; every other +terminal outcome means Slack accepted the reply while a known pending intent is +being advanced. +Startup recovery, where the intent row may already be gone, additionally +requires the atomically finalized visible assistant-message fact before it +classifies a non-delivery-failure terminal as accepted. The ordinary Slack +reply executor keeps the original inbox record until terminalization and +resumes multipart or ambiguous delivery without rerunning the model. +Permanent Slack reconciliation rejections remain uncertain and cannot authorize +a repost. They emit an actionable delivery error and use a long retry interval +so an operator can restore provider access without a five-second retry loop. Imported historical advisor executions own separate child event streams. The parent records start/end references and the child stores only its local events. diff --git a/packages/junior/src/chat/conversations/model-message.ts b/packages/junior/src/chat/conversations/model-message.ts new file mode 100644 index 000000000..0d9f4606e --- /dev/null +++ b/packages/junior/src/chat/conversations/model-message.ts @@ -0,0 +1,12 @@ +import { z } from "zod"; + +/** Junior-owned model-continuity shape; provider validation belongs to adapters. */ +export const conversationModelMessageSchema = z + .object({ role: z.string() }) + .passthrough() + .transform((value) => value as { role: string }); + +/** Opaque model-continuity message stored by Junior-owned durable state. */ +export type ConversationModelMessage = z.output< + typeof conversationModelMessageSchema +>; diff --git a/packages/junior/src/chat/conversations/sql/purge.ts b/packages/junior/src/chat/conversations/sql/purge.ts index a6d8942a9..b7c9ce903 100644 --- a/packages/junior/src/chat/conversations/sql/purge.ts +++ b/packages/junior/src/chat/conversations/sql/purge.ts @@ -6,6 +6,7 @@ import { juniorConversationMessages, juniorConversations, juniorDestinations, + juniorPendingDeliveries, } from "@/db/schema"; import { withConversationEventLock } from "./event-lock"; import { resolveRootVisibility } from "./privacy"; @@ -256,6 +257,10 @@ export async function purgeConversationTree( return { purged: false, conversations: 0 }; } } + await executor + .db() + .delete(juniorPendingDeliveries) + .where(inArray(juniorPendingDeliveries.conversationId, ids)); await executor .db() .delete(juniorConversationEvents) diff --git a/packages/junior/src/chat/conversations/turn-failure.ts b/packages/junior/src/chat/conversations/turn-failure.ts new file mode 100644 index 000000000..beaef689d --- /dev/null +++ b/packages/junior/src/chat/conversations/turn-failure.ts @@ -0,0 +1,14 @@ +import { z } from "zod"; + +/** Stable, privacy-safe classification for a failed turn. */ +export const conversationTurnFailureCodeSchema = z.enum([ + "agent_run_failed", + "delivery_failed", + "model_execution_failed", + "persistence_failed", +]); + +/** Failure classification persisted without raw provider or exception data. */ +export type ConversationTurnFailureCode = z.output< + typeof conversationTurnFailureCodeSchema +>; diff --git a/packages/junior/src/chat/credentials/state-adapter-token-store.ts b/packages/junior/src/chat/credentials/state-adapter-token-store.ts index 4d484480e..793429540 100644 --- a/packages/junior/src/chat/credentials/state-adapter-token-store.ts +++ b/packages/junior/src/chat/credentials/state-adapter-token-store.ts @@ -13,6 +13,11 @@ const LONG_LIVED_TTL_MS = 365 * 24 * 60 * 60 * 1000; const REFRESH_LOCK_WAIT_MS = 30_000; const REFRESH_LOCK_RETRY_MS = 100; +interface AuthorizationCompletedTokenRecord { + authorizationCompletionId: string; + tokens: StoredTokens; +} + function tokenKey(userId: string, provider: string): string { return `${KEY_PREFIX}:${userId}:${provider}`; } @@ -21,6 +26,28 @@ function refreshLockKey(userId: string, provider: string): string { return `${tokenKey(userId, provider)}:refresh`; } +function parseStoredTokenRecord(value: unknown): { + authorizationCompletionId?: string; + tokens: StoredTokens; +} { + if ( + typeof value === "object" && + value !== null && + "tokens" in value && + "authorizationCompletionId" in value + ) { + const record = value as Record; + if (typeof record.authorizationCompletionId !== "string") { + throw new Error("Invalid OAuth authorization completion receipt"); + } + return { + authorizationCompletionId: record.authorizationCompletionId, + tokens: storedTokensSchema.parse(record.tokens), + }; + } + return { tokens: storedTokensSchema.parse(value) }; +} + export class StateAdapterTokenStore implements UserTokenStore { private readonly state: StateAdapter; @@ -35,7 +62,7 @@ export class StateAdapterTokenStore implements UserTokenStore { const stored = await this.state.get(tokenKey(userId, provider)); return stored === null || stored === undefined ? undefined - : storedTokensSchema.parse(stored); + : parseStoredTokenRecord(stored).tokens; } async set( @@ -51,6 +78,42 @@ export class StateAdapterTokenStore implements UserTokenStore { await this.state.set(tokenKey(userId, provider), parsed, ttlMs); } + /** Commit tokens and the opaque receipt proving one callback exchanged its code. */ + async setForAuthorizationCompletion( + userId: string, + provider: string, + tokens: StoredTokens, + authorizationCompletionId: string, + ): Promise { + const parsed = storedTokensSchema.parse(tokens); + const expiresAt = parsed.refreshTokenExpiresAt ?? parsed.expiresAt; + const ttlMs = expiresAt + ? Math.max(expiresAt - Date.now() + BUFFER_MS, BUFFER_MS) + : LONG_LIVED_TTL_MS; + await this.state.set( + tokenKey(userId, provider), + { + authorizationCompletionId, + tokens: parsed, + } satisfies AuthorizationCompletedTokenRecord, + ttlMs, + ); + } + + /** Check whether the credential slot carries one exact callback receipt. */ + async hasAuthorizationCompletion( + userId: string, + provider: string, + authorizationCompletionId: string, + ): Promise { + const stored = await this.state.get(tokenKey(userId, provider)); + if (stored === null || stored === undefined) return false; + return ( + parseStoredTokenRecord(stored).authorizationCompletionId === + authorizationCompletionId + ); + } + async delete(userId: string, provider: string): Promise { await this.state.delete(tokenKey(userId, provider)); } diff --git a/packages/junior/src/chat/mcp/auth-store.ts b/packages/junior/src/chat/mcp/auth-store.ts index 312b41eaa..d61cb72c0 100644 --- a/packages/junior/src/chat/mcp/auth-store.ts +++ b/packages/junior/src/chat/mcp/auth-store.ts @@ -43,6 +43,7 @@ export interface McpAuthSessionState { } export interface McpStoredOAuthCredentials { + authorizationCompletionId?: string; clientInformation?: OAuthClientInformationMixed; discoveryState?: OAuthDiscoveryState; tokens?: OAuthTokens; @@ -204,6 +205,9 @@ function parseStoredCredentials( } return { + ...(typeof parsed.authorizationCompletionId === "string" + ? { authorizationCompletionId: parsed.authorizationCompletionId } + : {}), ...(isRecord(parsed.clientInformation) ? { clientInformation: diff --git a/packages/junior/src/chat/mcp/oauth-provider.ts b/packages/junior/src/chat/mcp/oauth-provider.ts index abc97f050..8d06b8be5 100644 --- a/packages/junior/src/chat/mcp/oauth-provider.ts +++ b/packages/junior/src/chat/mcp/oauth-provider.ts @@ -59,6 +59,7 @@ export class StateBackedMcpOAuthClientProvider implements OAuthClientProvider { private readonly runCredentialMutation?: ( mutation: () => Promise, ) => Promise, + private readonly authorizationCompletionId?: string, ) { this.clientMetadata = createClientMetadata(callbackUrl); } @@ -124,6 +125,9 @@ export class StateBackedMcpOAuthClientProvider implements OAuthClientProvider { await putMcpStoredOAuthCredentials(session.userId, session.provider, { ...credentials, tokens, + ...(this.authorizationCompletionId + ? { authorizationCompletionId: this.authorizationCompletionId } + : {}), }); }); } @@ -193,6 +197,14 @@ export class StateBackedMcpOAuthClientProvider implements OAuthClientProvider { )) ?? {}; await putMcpStoredOAuthCredentials(session.userId, session.provider, { + ...(scope === "tokens" || scope === "all" + ? {} + : credentials.authorizationCompletionId + ? { + authorizationCompletionId: + credentials.authorizationCompletionId, + } + : {}), ...(scope === "tokens" || scope === "all" ? {} : credentials.tokens diff --git a/packages/junior/src/chat/mcp/oauth.ts b/packages/junior/src/chat/mcp/oauth.ts index cccc75f01..ffda7cae8 100644 --- a/packages/junior/src/chat/mcp/oauth.ts +++ b/packages/junior/src/chat/mcp/oauth.ts @@ -72,6 +72,7 @@ export async function finalizeMcpAuthorization( authSessionId: string, authorizationCode: string, runCredentialMutation?: (mutation: () => Promise) => Promise, + authorizationCompletionId?: string, ): Promise { const plugin = requirePluginWithMcp(provider); const mcp = plugin.manifest.mcp; @@ -101,6 +102,7 @@ export async function finalizeMcpAuthorization( callbackUrl, undefined, runCredentialMutation, + authorizationCompletionId, ); const requestInit: RequestInit = {}; if (mcp.headers && Object.keys(mcp.headers).length > 0) { diff --git a/packages/junior/src/chat/runtime/README.md b/packages/junior/src/chat/runtime/README.md index 4e60aca3b..8721cee6c 100644 --- a/packages/junior/src/chat/runtime/README.md +++ b/packages/junior/src/chat/runtime/README.md @@ -32,10 +32,25 @@ this directory owns product orchestration around it. idempotency key, so the first committed outcome wins. - Intentional silence is a `turn_completed` `no_reply` outcome and does not create a synthetic visible assistant message. -- Stable lifecycle keys make explicit retries idempotent; they do not cover - process death between external delivery and persistence. Slack needs a - durable delivery outbox/receipt reconciler before terminal events are - crash-safe across that boundary. +- Ordinary finalized Slack thread replies use a durable intent before the + first post, opaque per-part metadata, one-attempt writes, and receipt + reconciliation. Creating the intent atomically commits model continuity to + conversation events and stores only event cursors in the outbox. A redelivered + inbox record advances that outbox without rerunning Pi; accepted delivery + commits visible facts and the turn terminal before the inbox is acknowledged, + while definitive rejection before any accepted part rolls live Pi context + back to the pre-intent cursor. A partial multipart rejection records only the + accepted Slack prefix as visible, retains full model/tool continuity, and + terminalizes as `delivery_failed`. +- Final replies from OAuth and continuation resumes use the same outbox. + Heartbeat advances due intents without rerunning Pi, and repairs session and + derived thread state before terminalization can delete the intent. +- Authorization/private notices, canvas recovery, and generic Chat SDK + fallback notices retain their owning delivery semantics. +- Recovery repairs canonical conversation state and terminal session metadata; + exact reconstruction of artifact-only Redis scratch after process loss + remains a separate bounded follow-up rather than widening the durable reply + command with arbitrary artifact payloads. ## Prompt Ownership diff --git a/packages/junior/src/chat/runtime/agent-continue-runner.ts b/packages/junior/src/chat/runtime/agent-continue-runner.ts index b142c25b8..b2699dd45 100644 --- a/packages/junior/src/chat/runtime/agent-continue-runner.ts +++ b/packages/junior/src/chat/runtime/agent-continue-runner.ts @@ -20,10 +20,12 @@ import { hydrateConversationMessages } from "@/chat/conversations/visible-messag import { loadProjection, loadConversationProjection, + recordAuthorizationCompleted, } from "@/chat/conversations/projection"; import { failAgentTurnSessionRecord, getAgentTurnSessionRecord, + isAgentTurnAuthorizationRecoveryActive, listAgentTurnSessionSummariesForConversation, type AgentTurnSessionRecord, type AgentTurnSessionSummary, @@ -34,7 +36,10 @@ import { persistThreadStateById, getChannelConfigurationServiceById, } from "@/chat/runtime/thread-state"; -import { buildDeliveredTurnStatePatch } from "@/chat/runtime/delivered-turn-state"; +import { + buildDeliveredTurnStatePatch, + buildRecoveredDeliveredTurnStatePatch, +} from "@/chat/runtime/delivered-turn-state"; import { getTurnUserMessage, getTurnUserReplyAttachmentContext, @@ -53,7 +58,7 @@ import { type AgentContinueRequest, } from "@/chat/services/agent-continue"; import { parseSlackThreadId } from "@/chat/slack/context"; -import { postSlackMessage } from "@/chat/slack/outbound"; +import { createSlackDeliveryLocator } from "@/chat/slack/outbound"; import { getStateAdapter } from "@/chat/state/adapter"; import { acquireActiveLock } from "@/chat/state/locks"; import { persistYieldSessionRecord } from "@/chat/services/turn-session-record"; @@ -81,12 +86,22 @@ import { stripRuntimeTurnContext, } from "@/chat/pi/transcript"; import { latestReportedProgress } from "@/chat/runtime/report-progress"; +import type { ConversationTurnLifecycle } from "@/chat/conversations/turn-lifecycle"; +import type { ConversationTurnFailureCode } from "@/chat/conversations/history"; +import type { RecoverableSlackDelivery } from "@/chat/slack/recoverable-delivery"; +import { + advanceOwnedSlackDeliveryWithTerminalRepair, + recoverOwnedSlackDeliveryForTurn, +} from "@/chat/runtime/slack-delivery-recovery"; +import { buildDeterministicAssistantMessageId } from "@/chat/state/turn-id"; const AGENT_CONTINUE_LOCK_RETRY_DELAYS_MS = [250, 1_000, 2_000] as const; /** Runtime ports for agent continuation scheduling. */ export interface AgentContinueRunnerOptions { agentRunner: AgentRunner; + recoverableSlackDelivery?: RecoverableSlackDelivery; + turnLifecycle: ConversationTurnLifecycle; resumeTurn?: typeof resumeSlackTurn; scheduleAgentContinue?: (request: AgentContinueRequest) => Promise; scheduleSessionCompletedPluginTasks?: (params: { @@ -98,7 +113,7 @@ export interface AgentContinueRunnerOptions { /** Persist a delivered continuation reply as the terminal thread state. */ async function persistCompletedReplyState(args: { sessionRecord: AgentTurnSessionRecord; - reply: AgentRunResult; + reply?: AgentRunResult; }): Promise { const currentState = await getPersistedThreadState( args.sessionRecord.conversationId, @@ -113,13 +128,19 @@ async function persistCompletedReplyState(args: { conversation, args.sessionRecord.sessionId, ); - const statePatch = buildDeliveredTurnStatePatch({ - artifacts, - conversation, - reply: args.reply, - sessionId: args.sessionRecord.sessionId, - userMessageId: userMessage?.id, - }); + const statePatch = args.reply + ? buildDeliveredTurnStatePatch({ + artifacts, + conversation, + reply: args.reply, + sessionId: args.sessionRecord.sessionId, + userMessageId: userMessage?.id, + }) + : buildRecoveredDeliveredTurnStatePatch({ + conversation, + sessionId: args.sessionRecord.sessionId, + inputMessageIds: userMessage ? [userMessage.id] : [], + }); await persistThreadStateById(args.sessionRecord.conversationId, { ...statePatch, @@ -264,18 +285,58 @@ function isContinuationResume(summary: AgentTurnSessionSummary): boolean { ); } -async function failUnresumableContinuation(args: { +interface TerminalContinuationFailure { conversationId: string; errorMessage: string; - expectedVersion?: number; - summary: AgentTurnSessionSummary; -}): Promise { - await failAgentTurnSessionRecord({ + expectedVersion: number; + failureCode: ConversationTurnFailureCode; + sessionId: string; + turnLifecycle: ConversationTurnLifecycle; +} + +async function terminallyFailOwnedContinuation( + args: TerminalContinuationFailure, +): Promise { + const current = await getAgentTurnSessionRecord( + args.conversationId, + args.sessionId, + ); + if ( + !current || + current.version !== args.expectedVersion || + current.state === "completed" || + current.state === "failed" || + current.state === "abandoned" + ) { + return false; + } + await args.turnLifecycle.fail({ conversationId: args.conversationId, - expectedVersion: args.expectedVersion ?? args.summary.version, - sessionId: args.summary.sessionId, + turnId: args.sessionId, + createdAtMs: Date.now(), + failureCode: args.failureCode, + }); + const failed = await failAgentTurnSessionRecord({ + conversationId: args.conversationId, + expectedVersion: args.expectedVersion, + sessionId: args.sessionId, errorMessage: args.errorMessage, }); + return failed !== undefined; +} + +async function terminallyFailContinuation( + args: TerminalContinuationFailure, +): Promise { + const state = getStateAdapter(); + await state.connect(); + const lock = await acquireActiveLock(state, args.conversationId); + if (!lock) return false; + try { + return await terminallyFailOwnedContinuation(args); + } finally { + await state.releaseLock(lock); + } } /** @@ -302,25 +363,65 @@ export async function continueSlackAgentRun( channelId: thread.channelId, threadTs: thread.threadTs, lockKey: payload.conversationId, + agentRunner: options.agentRunner, + recoverableSlackDelivery: options.recoverableSlackDelivery, lifecycleCorrelation: { conversationId: payload.conversationId, turnId: payload.sessionId, }, - agentRunner: options.agentRunner, + turnLifecycle: options.turnLifecycle, scheduleSessionCompletedPluginTasks: options.scheduleSessionCompletedPluginTasks, beforeStart: async () => { let sessionRecord: AgentTurnSessionRecord | undefined; try { + if ( + await recoverOwnedSlackDeliveryForTurn({ + conversationId: payload.conversationId, + delivery: options.recoverableSlackDelivery, + turnId: payload.sessionId, + }) + ) { + return false; + } sessionRecord = await getAgentTurnSessionRecord( payload.conversationId, payload.sessionId, ); + if ( + sessionRecord?.state === "completed" || + sessionRecord?.state === "failed" || + sessionRecord?.state === "abandoned" + ) { + const terminalState = await getPersistedThreadState( + payload.conversationId, + ); + const terminalConversation = + coerceThreadConversationState(terminalState); + clearPendingAuth(terminalConversation, payload.sessionId); + if ( + terminalConversation.processing.activeTurnId === payload.sessionId + ) { + terminalConversation.processing.activeTurnId = undefined; + } + await persistThreadStateById(payload.conversationId, { + conversation: terminalConversation, + }); + return false; + } + const authorizationCompleted = + sessionRecord?.resumeReason === "auth" && + (await isAgentTurnAuthorizationRecoveryActive({ + conversationId: payload.conversationId, + expectedVersion: payload.expectedVersion, + sessionId: payload.sessionId, + })); if ( !sessionRecord || sessionRecord.state !== "awaiting_resume" || (sessionRecord.resumeReason !== "timeout" && - sessionRecord.resumeReason !== "yield") || + sessionRecord.resumeReason !== "yield" && + !authorizationCompleted) || sessionRecord.version !== payload.expectedVersion ) { return false; @@ -342,10 +443,27 @@ export async function continueSlackAgentRun( `Unable to locate the persisted user message for agent continuation session "${payload.sessionId}"`, ); } - if (conversation.processing.activeTurnId !== payload.sessionId) { + const completedAuth = authorizationCompleted + ? conversation.processing.pendingAuth + : undefined; + if ( + completedAuth + ? completedAuth.sessionId !== payload.sessionId + : conversation.processing.activeTurnId !== payload.sessionId + ) { return false; } + if (completedAuth) { + await recordAuthorizationCompleted({ + conversationId: payload.conversationId, + kind: completedAuth.kind, + provider: completedAuth.provider, + actorId: completedAuth.actorId, + authorizationId: `${payload.sessionId}:${completedAuth.kind}:${completedAuth.provider}`, + }); + } + const channelConfiguration = getChannelConfigurationServiceById( thread.channelId, ); @@ -376,6 +494,7 @@ export async function continueSlackAgentRun( await failStrandedSessionWithFallback({ conversationId: payload.conversationId, errorMessage: "Stored Slack actor missing for continuation", + recoverableSlackDelivery: options.recoverableSlackDelivery, sessionRecord: activeSessionRecord, }); return false; @@ -388,11 +507,13 @@ export async function continueSlackAgentRun( }; } if (!activeSessionRecord.source) { - await failAgentTurnSessionRecord({ + await terminallyFailOwnedContinuation({ conversationId: payload.conversationId, expectedVersion: activeSessionRecord.version, + failureCode: "persistence_failed", sessionId: payload.sessionId, errorMessage: "Stored Slack source missing for continuation", + turnLifecycle: options.turnLifecycle, }); return false; } @@ -406,8 +527,9 @@ export async function continueSlackAgentRun( return { messageText: userMessage.text, - messageTs: getTurnUserSlackMessageTs(userMessage), inputMessageIds: [userMessage.id], + sliceId: activeSessionRecord.sliceId, + messageTs: getTurnUserSlackMessageTs(userMessage), initialStatus: latestReportedProgress(turnMessages), replyContext: { input: { @@ -457,6 +579,11 @@ export async function continueSlackAgentRun( reply, }); }, + onRecoveredSuccess: async () => { + await persistCompletedReplyState({ + sessionRecord: activeSessionRecord, + }); + }, onFailure: async () => { await persistFailedReplyState(activeSessionRecord); }, @@ -505,40 +632,49 @@ export async function continueSlackAgentRun( }); } -/** Terminally fail a stranded session and post the standard visible fallback. */ +/** Durably deliver the standard fallback before a stranded session closes. */ async function failStrandedSessionWithFallback(args: { conversationId: string; errorMessage: string; + recoverableSlackDelivery?: RecoverableSlackDelivery; sessionRecord: AgentTurnSessionRecord; }): Promise { - await failAgentTurnSessionRecord({ - conversationId: args.conversationId, - expectedVersion: args.sessionRecord.version, - sessionId: args.sessionRecord.sessionId, - errorMessage: args.errorMessage, - }); + if (!args.recoverableSlackDelivery) { + throw new TypeError("Stranded Slack fallback requires durable delivery"); + } + const current = await getAgentTurnSessionRecord( + args.conversationId, + args.sessionRecord.sessionId, + ); + if ( + !current || + current.version !== args.sessionRecord.version || + current.state === "completed" || + current.state === "failed" || + current.state === "abandoned" + ) { + return; + } const currentState = await getPersistedThreadState(args.conversationId); const conversation = coerceThreadConversationState(currentState); await hydrateConversationMessages({ conversation, conversationId: args.conversationId, }); - markTurnFailed({ - conversation, - nowMs: Date.now(), - sessionId: args.sessionRecord.sessionId, - userMessageId: getTurnUserMessage( - conversation, - args.sessionRecord.sessionId, - )?.id, - markConversationMessage, - updateConversationStats, - }); - await persistThreadStateById(args.conversationId, { conversation }); - const thread = parseSlackThreadId(args.conversationId); - if (!thread) { - return; + const userMessage = getTurnUserMessage( + conversation, + args.sessionRecord.sessionId, + ); + if ( + !thread || + !userMessage || + !current.destination || + current.destination.platform !== "slack" || + !current.source || + current.source.platform !== "slack" + ) { + throw new Error("Stranded Slack fallback metadata is incomplete"); } const eventName = "agent_turn_stranded_session_failed"; const eventId = logException( @@ -551,12 +687,57 @@ async function failStrandedSessionWithFallback(args: { }, "Stranded running agent session terminally failed", ); - await postSlackMessage({ - channelId: thread.channelId, - threadTs: thread.threadTs, - text: buildTurnFailureResponse( - requireTurnFailureEventId(eventId, eventName), - ), + const failureEventId = requireTurnFailureEventId(eventId, eventName); + const text = buildTurnFailureResponse(failureEventId); + const projection = await loadConversationProjection({ + conversationId: args.conversationId, + }); + const intent = await args.recoverableSlackDelivery.createIntent({ + conversationId: args.conversationId, + deliveryId: `slack:${current.sessionId}`, + turnId: current.sessionId, + modelMessages: projection.messages, + command: { + route: { + channelId: thread.channelId, + threadTs: thread.threadTs, + }, + publicLocator: createSlackDeliveryLocator(), + session: { + surface: "slack", + source: current.source, + destination: current.destination, + ...(current.actor ? { actor: current.actor } : {}), + ...(current.channelName ? { channelName: current.channelName } : {}), + startedAtMs: current.startedAtMs ?? Date.now(), + }, + parts: [{ text }], + completion: { + turnId: current.sessionId, + inputMessageIds: [userMessage.id], + assistantMessage: { + messageId: buildDeterministicAssistantMessageId(current.sessionId), + text, + createdAtMs: Date.now(), + author: { userName: botConfig.userName, isBot: true }, + }, + model: { + modelId: + current.modelId ?? + modelIdForProfile(botConfig, projection.modelProfile), + }, + sliceId: current.sliceId, + terminal: { + outcome: "failed", + failureCode: "persistence_failed", + eventId: failureEventId, + }, + }, + }, + }); + await advanceOwnedSlackDeliveryWithTerminalRepair({ + delivery: args.recoverableSlackDelivery, + intent, }); } @@ -581,75 +762,83 @@ async function recoverStrandedRunningSession(args: { if (!probe) { return false; } - await stateAdapter.releaseLock(probe); - - const sessionRecord = await getAgentTurnSessionRecord( - args.conversationId, - args.summary.sessionId, - ); - if (!sessionRecord || sessionRecord.state !== "running") { - return false; - } + let request: AgentContinueRequest | undefined; + try { + const sessionRecord = await getAgentTurnSessionRecord( + args.conversationId, + args.summary.sessionId, + ); + if (!sessionRecord || sessionRecord.state !== "running") { + return false; + } - const recoveryProjection = await loadConversationProjection({ - conversationId: args.conversationId, - }); - const modelProfile = recoveryProjection.modelProfile; - const modelId = modelIdForProfile(botConfig, modelProfile); - const recoveryMessages = - modelProfile !== STANDARD_MODEL_PROFILE - ? [ - ...stripRuntimeTurnContext(recoveryProjection.messages), - ...retainRuntimeTurnContext(sessionRecord.piMessages), - ] - : sessionRecord.piMessages; - - const parked = await persistYieldSessionRecord({ - channelName: sessionRecord.channelName, - conversationId: args.conversationId, - sessionId: sessionRecord.sessionId, - currentSliceId: sessionRecord.sliceId, - destination: sessionRecord.destination, - source: sessionRecord.source, - messages: recoveryMessages, - errorMessage: "Recovered running session after hard worker death", - logContext: {}, - modelId, - actor: sessionRecord.actor, - surface: sessionRecord.surface, - }); - if (!parked) { - await failStrandedSessionWithFallback({ + const recoveryProjection = await loadConversationProjection({ conversationId: args.conversationId, - errorMessage: - "Stranded running session had no resumable boundary after worker death", - sessionRecord, }); - return false; - } + const modelProfile = recoveryProjection.modelProfile; + const modelId = modelIdForProfile(botConfig, modelProfile); + const recoveryMessages = + modelProfile !== STANDARD_MODEL_PROFILE + ? [ + ...stripRuntimeTurnContext(recoveryProjection.messages), + ...retainRuntimeTurnContext(sessionRecord.piMessages), + ] + : sessionRecord.piMessages; + + const parked = await persistYieldSessionRecord({ + channelName: sessionRecord.channelName, + conversationId: args.conversationId, + sessionId: sessionRecord.sessionId, + currentSliceId: sessionRecord.sliceId, + destination: sessionRecord.destination, + source: sessionRecord.source, + messages: recoveryMessages, + errorMessage: "Recovered running session after hard worker death", + logContext: {}, + modelId, + actor: sessionRecord.actor, + surface: sessionRecord.surface, + }); + if (!parked) { + await failStrandedSessionWithFallback({ + conversationId: args.conversationId, + errorMessage: + "Stranded running session had no resumable boundary after worker death", + recoverableSlackDelivery: args.options.recoverableSlackDelivery, + sessionRecord, + }); + return false; + } - const request = await getAwaitingAgentContinueRequest({ - conversationId: args.conversationId, - sessionId: sessionRecord.sessionId, - }); - if (!request) { - await failStrandedSessionWithFallback({ + request = await getAwaitingAgentContinueRequest({ conversationId: args.conversationId, - errorMessage: - "Stranded running session could not materialize continuation metadata", - sessionRecord: parked, + sessionId: sessionRecord.sessionId, }); - return false; + if (!request) { + await failStrandedSessionWithFallback({ + conversationId: args.conversationId, + errorMessage: + "Stranded running session could not materialize continuation metadata", + recoverableSlackDelivery: args.options.recoverableSlackDelivery, + sessionRecord: parked, + }); + return false; + } + } finally { + await stateAdapter.releaseLock(probe); } + if (!request) return false; if (await continueSlackAgentRunWithLockRetry(request, args.options)) { return true; } - await failUnresumableContinuation({ + await terminallyFailContinuation({ conversationId: args.conversationId, expectedVersion: request.expectedVersion, - summary: args.summary, + sessionId: args.summary.sessionId, + failureCode: "agent_run_failed", errorMessage: "Awaiting agent continuation was stale before it could run", + turnLifecycle: args.options.turnLifecycle, }); return false; } @@ -675,7 +864,9 @@ export async function resumeAwaitingSlackContinuation( } for (const summary of summaries) { - if (!isContinuationResume(summary)) { + const authorizationCandidate = + summary.state === "awaiting_resume" && summary.resumeReason === "auth"; + if (!isContinuationResume(summary) && !authorizationCandidate) { continue; } @@ -684,11 +875,15 @@ export async function resumeAwaitingSlackContinuation( sessionId: summary.sessionId, }); if (!request) { - await failUnresumableContinuation({ + if (authorizationCandidate) continue; + await terminallyFailContinuation({ conversationId, - summary, + expectedVersion: summary.version, + sessionId: summary.sessionId, + failureCode: "persistence_failed", errorMessage: "Awaiting agent continuation metadata could not be materialized", + turnLifecycle: options.turnLifecycle, }); continue; } @@ -697,11 +892,13 @@ export async function resumeAwaitingSlackContinuation( return true; } - await failUnresumableContinuation({ + await terminallyFailContinuation({ conversationId, expectedVersion: request.expectedVersion, - summary, + sessionId: summary.sessionId, + failureCode: "agent_run_failed", errorMessage: "Awaiting agent continuation was stale before it could run", + turnLifecycle: options.turnLifecycle, }); } diff --git a/packages/junior/src/chat/runtime/delivered-turn-state.ts b/packages/junior/src/chat/runtime/delivered-turn-state.ts index 216e3f8d0..f66e886f1 100644 --- a/packages/junior/src/chat/runtime/delivered-turn-state.ts +++ b/packages/junior/src/chat/runtime/delivered-turn-state.ts @@ -6,7 +6,7 @@ import { mergeArtifactsState, type ThreadStatePatch, } from "@/chat/runtime/thread-state"; -import { markTurnCompleted } from "@/chat/runtime/turn"; +import { markTurnCompleted, markTurnFailed } from "@/chat/runtime/turn"; import { markConversationMessage, normalizeConversationText, @@ -72,3 +72,68 @@ export function buildDeliveredTurnStatePatch(args: { sandboxDependencyProfileHash: args.reply.sandboxDependencyProfileHash, }; } + +/** Repair derived thread state after canonical SQL proves delivery completed. */ +export function buildRecoveredDeliveredTurnStatePatch(args: { + assistantMessage?: { + author: { isBot: true; userName: string }; + createdAtMs: number; + messageId: string; + text: string; + }; + conversation: ThreadConversationState; + sessionId: string; + inputMessageIds?: readonly string[]; +}): { conversation: ThreadConversationState } { + const conversation = structuredClone(args.conversation); + clearPendingAuth(conversation, args.sessionId); + for (const messageId of args.inputMessageIds ?? []) { + markConversationMessage(conversation, messageId, { + replied: true, + skippedReason: undefined, + }); + } + if (args.assistantMessage) { + upsertConversationMessage(conversation, { + id: args.assistantMessage.messageId, + role: "assistant", + text: + normalizeConversationText(args.assistantMessage.text) || + "[empty response]", + createdAtMs: args.assistantMessage.createdAtMs, + author: args.assistantMessage.author, + meta: { replied: true }, + }); + } + markTurnCompleted({ + conversation, + nowMs: Date.now(), + sessionId: args.sessionId, + updateConversationStats, + }); + return { conversation }; +} + +/** Repair derived thread state after canonical SQL proves delivery failed. */ +export function buildRecoveredFailedDeliveryStatePatch(args: { + conversation: ThreadConversationState; + sessionId: string; + inputMessageIds?: readonly string[]; +}): { conversation: ThreadConversationState } { + const conversation = structuredClone(args.conversation); + clearPendingAuth(conversation, args.sessionId); + for (const messageId of args.inputMessageIds ?? []) { + markConversationMessage(conversation, messageId, { + replied: false, + skippedReason: "reply failed", + }); + } + markTurnFailed({ + conversation, + nowMs: Date.now(), + sessionId: args.sessionId, + markConversationMessage, + updateConversationStats, + }); + return { conversation }; +} diff --git a/packages/junior/src/chat/runtime/reply-executor.ts b/packages/junior/src/chat/runtime/reply-executor.ts index e8a354da4..f469d2e75 100644 --- a/packages/junior/src/chat/runtime/reply-executor.ts +++ b/packages/junior/src/chat/runtime/reply-executor.ts @@ -46,6 +46,7 @@ import { import { persistThreadRuntimeState, persistThreadState, + type ThreadStatePatch, } from "@/chat/runtime/thread-state"; import { buildDeliveredTurnStatePatch } from "@/chat/runtime/delivered-turn-state"; import { getTurnRequestDeadline } from "@/chat/runtime/request-deadline"; @@ -94,6 +95,7 @@ import { ConversationTurnBoundaryError, CooperativeTurnYieldError, getConversationTurnBoundaryError, + isTurnInputCommitLostError, TurnInputDeferredError, } from "@/chat/runtime/turn"; import { buildDeterministicTurnId } from "@/chat/runtime/turn"; @@ -105,6 +107,7 @@ import { getAgentTurnDiagnosticsAttributes, } from "@/chat/services/turn-failure-response"; import { buildAuthPauseResponse } from "@/chat/services/auth-pause-response"; +import { clearPendingAuth } from "@/chat/services/pending-auth"; import { maybeApplyProviderDefaultConfigRequest } from "@/chat/services/provider-default-config"; import type { PiMessage } from "@/chat/pi/messages"; import { @@ -136,6 +139,11 @@ import { requireSlackDestination } from "@/chat/destination"; import { escapeXml } from "@/chat/xml"; import { persistConversationMessages } from "@/chat/conversations/visible-messages"; import { modelIdForProfile } from "@/chat/model-profile"; +import type { RecoverableSlackDelivery } from "@/chat/slack/recoverable-delivery"; +import { advanceSlackDeliveryWithTerminalRepair } from "@/chat/runtime/slack-delivery-recovery"; +import type { PendingConversationDelivery } from "@/chat/slack/delivery-outbox"; +import { buildPendingSlackDeliveryCommandDraft } from "@/chat/slack/delivery-command"; +import { createSlackDeliveryLocator } from "@/chat/slack/outbound"; import type { ConversationTurnLifecycle } from "@/chat/conversations/turn-lifecycle"; /** @@ -384,6 +392,7 @@ export interface ReplyExecutorServices { sessionId: string; }) => Promise; lookupSlackUser: typeof lookupSlackUser; + recoverableSlackDelivery: RecoverableSlackDelivery; turnLifecycle: ConversationTurnLifecycle; scheduleAgentContinue: (request: AgentContinueRequest) => Promise; scheduleSessionCompletedPluginTasks: (params: { @@ -426,6 +435,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { destination: Destination; explicitMention?: boolean; ack?: () => Promise; + ackMessageIds?: (messageIds: readonly string[]) => Promise; onToolInvocation?: (invocation: TurnToolInvocation) => void; onTurnCompleted?: () => Promise; onTurnStatePersisted?: () => Promise; @@ -611,6 +621,55 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { } }; let activeTurnId = preparedState.conversation.processing.activeTurnId; + const terminalizeParkedTurn = async (args: { + expectedVersion: number; + failureCode: "agent_run_failed" | "persistence_failed"; + mode: "abandoned" | "failed"; + sessionId: string; + errorMessage: string; + }): Promise => { + if (!conversationId) return false; + const stateAdapter = getStateAdapter(); + await stateAdapter.connect(); + const lock = await acquireActiveLock(stateAdapter, conversationId); + if (!lock) return false; + try { + const current = await getAgentTurnSessionRecord( + conversationId, + args.sessionId, + ); + if ( + !current || + current.version !== args.expectedVersion || + current.state !== "awaiting_resume" + ) { + return false; + } + await deps.services.turnLifecycle.fail({ + conversationId, + turnId: args.sessionId, + createdAtMs: Date.now(), + failureCode: args.failureCode, + }); + const terminal = + args.mode === "abandoned" + ? await abandonAgentTurnSessionRecord({ + conversationId, + expectedVersion: args.expectedVersion, + sessionId: args.sessionId, + errorMessage: args.errorMessage, + }) + : await failAgentTurnSessionRecord({ + conversationId, + expectedVersion: args.expectedVersion, + sessionId: args.sessionId, + errorMessage: args.errorMessage, + }); + return terminal !== undefined; + } finally { + await stateAdapter.releaseLock(lock); + } + }; const resolveSteeringMessages = async ( queuedMessages: QueuedTurnMessage[], ): Promise => { @@ -749,6 +808,129 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { })); return drainParkedInputToEventLog(parkedPairs); }; + /** Finish mailbox and plugin work after centralized terminal repair. */ + const finishCanonicalTerminal = async (args: { + ack: + | { kind: "all" } + | { kind: "messages"; messageIds: readonly string[] }; + deliveryId: string; + deliveryOutcome: "accepted" | "failed"; + modelSucceeded: boolean; + turnId: string; + runtimePatch?: Omit; + }): Promise => { + const attributes = { "app.delivery.id": args.deliveryId }; + if (args.runtimePatch) { + await persistThreadRuntimeStateWithRetry(thread, args.runtimePatch); + } + try { + await options.onTurnStatePersisted?.(); + } catch (repairError) { + logException( + repairError, + "slack_delivery_state_callback_repair_failed", + turnTraceContext, + attributes, + "Failed to confirm repaired Slack delivery runtime state", + ); + } + if ( + args.deliveryOutcome === "accepted" && + args.modelSucceeded && + conversationId + ) { + try { + await deps.services.scheduleSessionCompletedPluginTasks({ + conversationId, + sessionId: args.turnId, + }); + } catch (repairError) { + logException( + repairError, + "slack_delivery_plugin_repair_failed", + turnTraceContext, + attributes, + "Failed to schedule terminal Slack delivery plugin tasks", + ); + } + } + try { + await options.onTurnCompleted?.(); + } catch (repairError) { + logException( + repairError, + "slack_delivery_completion_callback_repair_failed", + turnTraceContext, + attributes, + "Failed to complete repaired Slack delivery callbacks", + ); + } + if (args.ack.kind === "messages") { + await options.ackMessageIds?.(args.ack.messageIds); + } else { + await options.ack?.(); + } + }; + const pendingDelivery = conversationId + ? await deps.services.recoverableSlackDelivery.loadByConversation({ + conversationId, + }) + : undefined; + if (pendingDelivery) { + const recovery = await advanceSlackDeliveryWithTerminalRepair({ + delivery: deps.services.recoverableSlackDelivery, + intent: pendingDelivery, + }); + if (recovery.outcome === "pending") { + throw new TurnInputDeferredError( + "Slack delivery is awaiting retry or reconciliation", + Math.max(0, recovery.retryAtMs - Date.now()), + ); + } + const recoveredSuccessfully = + recovery.outcome === "accepted" && + pendingDelivery.command.completion.terminal.outcome === "success"; + await finishCanonicalTerminal({ + ack: + pendingDelivery.turnId === turnId + ? { kind: "all" } + : { + kind: "messages", + messageIds: + pendingDelivery.command.completion.inputMessageIds, + }, + deliveryId: pendingDelivery.deliveryId, + deliveryOutcome: recovery.outcome, + modelSucceeded: recoveredSuccessfully, + turnId: pendingDelivery.turnId, + }); + if (pendingDelivery.turnId !== turnId) { + throw new TurnInputDeferredError( + "Recovered an older Slack delivery before newer input", + 0, + true, + ); + } + return; + } + const priorTerminal = conversationId + ? await deps.services.recoverableSlackDelivery.loadTerminalOutcome({ + conversationId, + turnId, + acceptanceEvidence: "visible_assistant", + }) + : undefined; + if (priorTerminal && conversationId) { + const modelSucceeded = priorTerminal.modelSucceeded; + await finishCanonicalTerminal({ + ack: { kind: "all" }, + deliveryId: `slack:${turnId}`, + deliveryOutcome: priorTerminal.deliveryOutcome, + modelSucceeded, + turnId, + }); + return; + } if (preparedState.userMessageAlreadyReplied) { await persistThreadState(thread, { conversation: preparedState.conversation, @@ -809,31 +991,42 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { // A user follow-up supersedes the auth-parked run: answer it // now as a fresh turn instead of consuming it into a pause that // may never resume. The parked prompt stays model-visible via - // the event-log projection, pendingAuth state keeps the - // authorization link reusable, and the abandoned record turns a - // late OAuth callback into a stale no-op instead of a competing - // run. - await abandonAgentTurnSessionRecord({ - conversationId, - sessionId: activeTurnId, - errorMessage: - "Auth-parked session superseded by a new user message", - }); + // the event-log projection. The abandoned record turns a late + // OAuth callback into a stale no-op, and its exact pending-auth + // marker must be cleared because that session cannot resume. + if ( + !(await terminalizeParkedTurn({ + expectedVersion: sessionRecord.version, + failureCode: "agent_run_failed", + mode: "abandoned", + sessionId: activeTurnId, + errorMessage: + "Auth-parked session superseded by a new user message", + })) + ) { + throw new TurnInputDeferredError(); + } markTurnClosed({ conversation: preparedState.conversation, nowMs: Date.now(), sessionId: activeTurnId, updateConversationStats, }); + clearPendingAuth(preparedState.conversation, activeTurnId); activeTurnId = undefined; } else { - await failAgentTurnSessionRecord({ - conversationId, - expectedVersion: sessionRecord.version, - sessionId: activeTurnId, - errorMessage: - "Awaiting agent continuation metadata could not be materialized", - }); + if ( + !(await terminalizeParkedTurn({ + expectedVersion: sessionRecord.version, + failureCode: "persistence_failed", + mode: "failed", + sessionId: activeTurnId, + errorMessage: + "Awaiting agent continuation metadata could not be materialized", + })) + ) { + throw new TurnInputDeferredError(); + } markTurnFailed({ conversation: preparedState.conversation, nowMs: Date.now(), @@ -1018,15 +1211,16 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { // destination accepted the final posts, later errors in the same turn // must not mark it failed or trigger the fallback failure reply. let finalReplyDelivered = false; - let lifecycleTerminalized = false; let turnCompletionNotified = false; + let recoverableDeliveryTerminalized = false; + let lifecycleTerminalized = false; let latestArtifacts = preparedState.artifacts; let assistantTitleArtifacts: Partial = {}; let agentContinueScheduleError: unknown; let boundaryFailureCode: "agent_run_failed" | "delivery_failed" = "agent_run_failed"; - let boundaryFailureEventId: string | undefined; let finalizedFailureEventId: string | undefined; + let durableDeliveryIntent: PendingConversationDelivery | undefined; const hasVisibleSlackDelivery = (post: { text: string }) => post.text.trim().length > 0; const notifyTurnCompleted = async (): Promise => { @@ -1281,7 +1475,10 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { onToolInvocation: options.onToolInvocation, }, durability: { - onInputCommitted: options.ack, + // Pi still receives an explicit safe input-commit callback, but + // the queue inbox remains the recovery trigger until delivery + // terminalization or another durable handoff below. + onInputCommitted: async () => undefined, drainSteeringMessages, shouldYield: options.shouldYield, onSandboxAcquired: async (sandbox) => { @@ -1368,6 +1565,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { } persistedAtLeastOnce = true; shouldPersistFailureState = false; + await options.ack?.(); return; } await postAuthPauseNotice(outcome.providerDisplayName); @@ -1380,6 +1578,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { }); persistedAtLeastOnce = true; shouldPersistFailureState = false; + await options.ack?.(); return; } if (outcome.status === "suspended") { @@ -1403,6 +1602,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { sessionId: turnId, expectedVersion: outcome.resumeVersion, }); + await options.ack?.(); shouldPersistFailureState = false; } catch (scheduleError) { logException( @@ -1451,9 +1651,8 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { const replyFooter = buildSlackReplyFooter({ conversationId, }); - const shouldUseSlackFooter = - Boolean(replyFooter) && - Boolean(channelId && threadTs) && + const shouldUseRecoverableSlackDelivery = + Boolean(channelId && threadTs && conversationId) && (thread.adapter as { name?: string } | undefined)?.name === "slack"; boundaryFailureCode = "delivery_failed"; @@ -1469,37 +1668,92 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { "Slack final reply plan did not contain visible delivery", ); } - if (shouldUseSlackFooter) { + if (shouldUseRecoverableSlackDelivery) { const slackChannelId = channelId; const slackThreadTs = threadTs; if (!slackChannelId || !slackThreadTs) { throw new Error( - "Slack footer delivery requires a concrete channel and thread timestamp", + "Recoverable Slack delivery requires a concrete channel and thread timestamp", ); } - - await postSlackApiReplyPosts({ - beforePost: beforeFirstResponsePost, - channelId: slackChannelId, - threadTs: slackThreadTs, - posts: plannedPosts, - footer: replyFooter, - onPostError: ({ error, messageTs, stage }) => { - boundaryFailureEventId = logException( - error, - "slack_thread_post_failed", - turnTraceContext, - { - "app.slack.reply_stage": stage, - ...(messageTs - ? { "messaging.message.id": messageTs } + const visiblePosts = plannedPosts.filter(hasVisibleSlackDelivery); + const assistantCreatedAtMs = Date.now(); + const publicLocator = createSlackDeliveryLocator(); + const delivery = + await deps.services.recoverableSlackDelivery.createIntent({ + conversationId: conversationId!, + turnId, + deliveryId: `slack:${turnId}`, + // A reply without a replacement transcript preserves the + // model input boundary; absence must not mean "erase it". + modelMessages: reply.piMessages ?? piMessages ?? [], + command: buildPendingSlackDeliveryCommandDraft({ + publicLocator, + route: { + channelId: slackChannelId, + threadTs: slackThreadTs, + }, + session: { + surface: "slack", + source, + destination, + ...(destinationVisibility + ? { destinationVisibility } : {}), - ...getSlackErrorObservabilityAttributes(error), + ...(executionActor ? { actor: executionActor } : {}), + ...(channelName ? { channelName } : {}), + startedAtMs: message.metadata.dateSent.getTime(), }, - "Failed to post Slack thread reply", - ); - }, - }); + posts: visiblePosts, + footer: replyFooter, + turnId, + inputMessageIds: [ + ...new Set([ + ...(options.queuedMessages ?? []).map( + (queued) => queued.message.id, + ), + ...(preparedState.userMessageId + ? [preparedState.userMessageId] + : []), + ]), + ], + assistantText: normalizeConversationText(reply.text), + assistantCreatedAtMs, + assistantUserName: botConfig.userName, + diagnostics: reply.diagnostics, + sliceId: 1, + failureEventId: finalizedFailureEventId, + }), + }); + durableDeliveryIntent = delivery; + await beforeFirstResponsePost(); + const deliveryOutcome = + await advanceSlackDeliveryWithTerminalRepair({ + delivery: deps.services.recoverableSlackDelivery, + intent: delivery, + }); + if (deliveryOutcome.outcome === "pending") { + shouldPersistFailureState = false; + throw new TurnInputDeferredError( + "Slack delivery is awaiting retry or reconciliation", + Math.max(0, deliveryOutcome.retryAtMs - Date.now()), + ); + } + if (deliveryOutcome.outcome === "failed") { + lifecycleTerminalized = true; + persistedAtLeastOnce = true; + shouldPersistFailureState = false; + await finishCanonicalTerminal({ + ack: { kind: "all" }, + deliveryId: delivery.deliveryId, + deliveryOutcome: "failed", + modelSucceeded: false, + turnId, + }); + return; + } + recoverableDeliveryTerminalized = true; + lifecycleTerminalized = true; } else { for (const post of plannedPosts) { if (!hasVisibleSlackDelivery(post)) { @@ -1537,71 +1791,74 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { if (completedState.artifacts) { latestArtifacts = completedState.artifacts; } + preparedState.conversation = completedState.conversation; try { // Commit the durable delivery record first so recovery cannot // regenerate an accepted reply. Persist canonical visible-message // facts next, then update Redis runtime scratch independently. - if (conversationId && reply.piMessages?.length) { - await completeDeliveredTurn({ - channelName, - conversationId, - durationMs: reply.diagnostics.durationMs, - usage: reply.diagnostics.usage, - reasoningLevel: reply.diagnostics.reasoningLevel, - destination, - destinationVisibility, - source, - sessionId: turnId, - sliceId: 1, - messages: reply.piMessages, - modelId: reply.diagnostics.modelId, - logContext: { - threadId, - actorId: slackActorId, - channelId, - runId, - assistantUserName: botConfig.userName, - }, - actor: executionActor, - surface: "slack", - }); - } else if (conversationId) { - await recordAgentTurnSessionSummary({ - channelName, - conversationId, - cumulativeDurationMs: reply.diagnostics.durationMs, - cumulativeUsage: reply.diagnostics.usage, - sessionId: turnId, - sliceId: 1, - startedAtMs: message.metadata.dateSent.getTime(), - state: "completed", - actor: executionActor, - destination, - destinationVisibility, - source, - traceId: getActiveTraceId(), - }); - } - await persistWithRetry(() => - persistConversationMessages({ - conversation: completedState.conversation, - conversationId, - }), - ); - await persistThreadRuntimeStateWithRetry(thread, completedState); - if ( - completedState.artifacts && - (assistantTitleArtifacts.assistantTitle !== undefined || - assistantTitleArtifacts.assistantTitleSourceMessageId !== - undefined) && - (completedState.artifacts.assistantTitle !== - assistantTitleArtifacts.assistantTitle || - completedState.artifacts.assistantTitleSourceMessageId !== - assistantTitleArtifacts.assistantTitleSourceMessageId) - ) { - await persistThreadRuntimeStateWithRetry(thread, { - artifacts: latestArtifacts, - }); + if (!recoverableDeliveryTerminalized) { + if (conversationId && reply.piMessages?.length) { + await completeDeliveredTurn({ + channelName, + conversationId, + durationMs: reply.diagnostics.durationMs, + usage: reply.diagnostics.usage, + reasoningLevel: reply.diagnostics.reasoningLevel, + destination, + destinationVisibility, + source, + sessionId: turnId, + sliceId: 1, + messages: reply.piMessages, + modelId: reply.diagnostics.modelId, + logContext: { + threadId, + actorId: slackActorId, + channelId, + runId, + assistantUserName: botConfig.userName, + }, + actor: executionActor, + surface: "slack", + }); + } else if (conversationId) { + await recordAgentTurnSessionSummary({ + channelName, + conversationId, + cumulativeDurationMs: reply.diagnostics.durationMs, + cumulativeUsage: reply.diagnostics.usage, + sessionId: turnId, + sliceId: 1, + startedAtMs: message.metadata.dateSent.getTime(), + state: "completed", + actor: executionActor, + destination, + destinationVisibility, + source, + traceId: getActiveTraceId(), + }); + } + await persistWithRetry(() => + persistConversationMessages({ + conversation: completedState.conversation, + conversationId, + }), + ); + await persistThreadRuntimeStateWithRetry(thread, completedState); + if ( + completedState.artifacts && + (assistantTitleArtifacts.assistantTitle !== undefined || + assistantTitleArtifacts.assistantTitleSourceMessageId !== + undefined) && + (completedState.artifacts.assistantTitle !== + assistantTitleArtifacts.assistantTitle || + completedState.artifacts.assistantTitleSourceMessageId !== + assistantTitleArtifacts.assistantTitleSourceMessageId) + ) { + await persistThreadRuntimeStateWithRetry(thread, { + artifacts: latestArtifacts, + }); + } } } catch (commitError) { // The user already saw the reply; keep the turn successful and @@ -1624,8 +1881,21 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { lifecycleTerminalized = true; } } - preparedState.conversation = completedState.conversation; persistedAtLeastOnce = true; + if (recoverableDeliveryTerminalized && conversationId) { + const { conversation: _conversation, ...runtimePatch } = + completedState; + await finishCanonicalTerminal({ + ack: { kind: "all" }, + deliveryId: + durableDeliveryIntent?.deliveryId ?? `slack:${turnId}`, + deliveryOutcome: "accepted", + modelSucceeded: reply.diagnostics.outcome === "success", + turnId, + runtimePatch, + }); + return; + } if (!lifecycleTerminalized && conversationId) { if (reply.diagnostics.outcome === "success") { await deps.services.turnLifecycle.complete({ @@ -1659,6 +1929,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { "Agent turn completed", ); } + await options.ack?.(); await notifyTurnCompleted(); if (reply.diagnostics.outcome === "success" && conversationId) { try { @@ -1677,6 +1948,68 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { } } } catch (error) { + if (error instanceof TurnInputDeferredError) { + shouldPersistFailureState = false; + throw error; + } + if (isTurnInputCommitLostError(error)) { + shouldPersistFailureState = false; + throw error; + } + if ( + durableDeliveryIntent && + conversationId && + !recoverableDeliveryTerminalized + ) { + logException( + error, + "slack_delivery_intent_interrupted", + turnTraceContext, + { + "app.delivery.id": durableDeliveryIntent.deliveryId, + "app.delivery.turn_id": durableDeliveryIntent.turnId, + }, + "Slack delivery attempt was interrupted after durable intent", + ); + const stillPending = + await deps.services.recoverableSlackDelivery.loadByTurn({ + conversationId, + turnId: durableDeliveryIntent.turnId, + }); + if (stillPending) { + shouldPersistFailureState = false; + throw new TurnInputDeferredError( + "Slack delivery intent remains pending after an interrupted attempt", + Math.max( + 0, + Math.max( + stillPending.nextAttemptAtMs, + stillPending.lease?.expiresAtMs ?? 0, + ) - Date.now(), + ), + ); + } + const terminal = + await deps.services.recoverableSlackDelivery.loadTerminalOutcome({ + conversationId, + turnId: durableDeliveryIntent.turnId, + acceptanceEvidence: "known_outbox_intent", + }); + if (terminal) { + shouldPersistFailureState = false; + finalReplyDelivered = terminal.deliveryOutcome === "accepted"; + persistedAtLeastOnce = true; + const modelSucceeded = terminal.modelSucceeded; + await finishCanonicalTerminal({ + ack: { kind: "all" }, + deliveryId: durableDeliveryIntent.deliveryId, + deliveryOutcome: terminal.deliveryOutcome, + modelSucceeded, + turnId: durableDeliveryIntent.turnId, + }); + return; + } + } if (finalReplyDelivered) { // Delivered-turn guard: errors after Slack accepted the final // reply (redundant-ack cleanup, completion callbacks) must not @@ -1723,7 +2056,6 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { classifiedFailure?.failureCode ?? boundaryFailureCode; const failureEventId = classifiedFailure?.eventId ?? - boundaryFailureEventId ?? logException( failureCause, "slack_turn_execution_failed", @@ -1784,6 +2116,7 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { } persistedAtLeastOnce = true; shouldPersistFailureState = false; + await options.ack?.(); return; } throw new ConversationTurnBoundaryError({ @@ -1805,6 +2138,15 @@ export function createReplyToThread(deps: ReplyExecutorDeps) { }); if (conversationId) { try { + if (!lifecycleTerminalized) { + await deps.services.turnLifecycle.fail({ + conversationId, + turnId, + createdAtMs: Date.now(), + failureCode: "agent_run_failed", + }); + lifecycleTerminalized = true; + } await recordAgentTurnSessionSummary({ channelName, conversationId, diff --git a/packages/junior/src/chat/runtime/slack-delivery-recovery.ts b/packages/junior/src/chat/runtime/slack-delivery-recovery.ts new file mode 100644 index 000000000..1ed76d1ac --- /dev/null +++ b/packages/junior/src/chat/runtime/slack-delivery-recovery.ts @@ -0,0 +1,319 @@ +/** Repair non-SQL state while a durable Slack delivery intent still exists. */ +import { botConfig } from "@/chat/config"; +import { logException } from "@/chat/logging"; +import { scheduleSessionCompletedPluginTasks } from "@/chat/plugins/task-runner"; +import { loadTurnProjection } from "@/chat/conversations/projection"; +import { hydrateConversationMessages } from "@/chat/conversations/visible-messages"; +import { + buildRecoveredDeliveredTurnStatePatch, + buildRecoveredFailedDeliveryStatePatch, +} from "@/chat/runtime/delivered-turn-state"; +import { + getPersistedThreadState, + persistThreadStateById, +} from "@/chat/runtime/thread-state"; +import { coerceThreadConversationState } from "@/chat/state/conversation"; +import { + getAgentTurnSessionRecord, + upsertAgentTurnSessionRecord, +} from "@/chat/state/turn-session"; +import { completeDeliveredTurn } from "@/chat/services/turn-session-record"; +import { markTurnFailed } from "@/chat/runtime/turn"; +import { + markConversationMessage, + updateConversationStats, +} from "@/chat/services/conversation-memory"; +import type { + RecoverableSlackDelivery, + RecoverableSlackDeliveryOutcome, + RecoverableSlackDeliveryTerminalizingInput, +} from "@/chat/slack/recoverable-delivery"; +import type { PendingConversationDelivery } from "@/chat/slack/delivery-outbox"; +import { getStateAdapter } from "@/chat/state/adapter"; +import { acquireActiveLock } from "@/chat/state/locks"; +import { addAgentTurnUsage } from "@/chat/usage"; + +const DEFAULT_RECOVERY_LIMIT = 25; + +/** Repair session and derived thread state before terminal delivery deletion. */ +async function repairOwnedTerminalizingSlackDelivery( + input: RecoverableSlackDeliveryTerminalizingInput, +): Promise { + const command = input.command; + const conversationId = input.conversationId; + const sessionId = input.turnId; + const sessionRecord = await getAgentTurnSessionRecord( + conversationId, + sessionId, + ); + + const turnCompleted = + input.deliveryOutcome === "accepted" && + command.completion.terminal.outcome === "success"; + if (turnCompleted) { + if (sessionRecord?.state !== "completed") { + const projection = await loadTurnProjection({ + conversationId, + committedSeq: command.completion.model.committedSeq, + includeTail: false, + }); + if (!projection) { + throw new Error("Delivered turn projection is unavailable"); + } + await completeDeliveredTurn({ + conversationId, + destination: command.session.destination, + destinationVisibility: command.session.destinationVisibility, + durationMs: command.completion.durationMs, + messages: projection.messages, + modelId: projection.modelId ?? command.completion.model.modelId, + actor: command.session.actor, + reasoningLevel: command.completion.reasoningLevel, + sessionId, + sliceId: command.completion.sliceId, + source: command.session.source, + surface: command.session.surface, + usage: command.completion.usage, + channelName: command.session.channelName, + logContext: { + threadId: conversationId, + actorId: + command.session.actor?.platform === "slack" + ? command.session.actor.userId + : undefined, + channelId: command.session.destination.channelId, + assistantUserName: botConfig.userName, + }, + }); + } + } else { + const errorMessage = + input.deliveryOutcome === "accepted" + ? "Delivered terminal failure reply" + : "Slack rejected the final turn reply"; + if ( + !sessionRecord || + (sessionRecord.state !== "completed" && + sessionRecord.state !== "failed" && + sessionRecord.state !== "abandoned") + ) { + const projection = await loadTurnProjection({ + conversationId, + committedSeq: command.completion.model.committedSeq, + includeTail: false, + }); + if (!projection) { + throw new Error("Terminal turn projection is unavailable"); + } + await upsertAgentTurnSessionRecord({ + ...(command.session.channelName + ? { channelName: command.session.channelName } + : {}), + conversationId, + ...(sessionRecord?.cumulativeDurationMs !== undefined || + command.completion.durationMs !== undefined + ? { + cumulativeDurationMs: + (sessionRecord?.cumulativeDurationMs ?? 0) + + (command.completion.durationMs ?? 0), + } + : {}), + cumulativeUsage: addAgentTurnUsage( + sessionRecord?.cumulativeUsage, + command.completion.usage, + ), + destination: command.session.destination, + destinationVisibility: command.session.destinationVisibility, + source: command.session.source, + sessionId, + sliceId: command.completion.sliceId, + state: "failed", + surface: command.session.surface, + piMessages: projection.messages, + modelId: projection.modelId ?? command.completion.model.modelId, + actor: command.session.actor ?? sessionRecord?.actor, + reasoningLevel: + command.completion.reasoningLevel ?? sessionRecord?.reasoningLevel, + loadedSkillNames: sessionRecord?.loadedSkillNames, + errorMessage, + }); + } + } + + const currentState = await getPersistedThreadState(conversationId); + const conversation = coerceThreadConversationState(currentState); + await hydrateConversationMessages({ conversation, conversationId }); + const patch = + input.deliveryOutcome === "accepted" + ? buildRecoveredDeliveredTurnStatePatch({ + assistantMessage: command.completion.assistantMessage, + conversation, + inputMessageIds: command.completion.inputMessageIds, + sessionId, + }) + : buildRecoveredFailedDeliveryStatePatch({ + conversation, + inputMessageIds: command.completion.inputMessageIds, + sessionId, + }); + if ( + input.deliveryOutcome === "accepted" && + command.completion.terminal.outcome === "failed" + ) { + markTurnFailed({ + conversation: patch.conversation, + nowMs: Date.now(), + sessionId, + markConversationMessage, + updateConversationStats, + }); + } + await persistThreadStateById(conversationId, patch); +} + +interface AdvanceSlackDeliveryArgs { + delivery: RecoverableSlackDelivery; + intent: PendingConversationDelivery; + beforeRepair?: ( + input: RecoverableSlackDeliveryTerminalizingInput, + ) => Promise; +} + +/** Advance a delivery while the caller owns the active conversation lock. */ +export async function advanceOwnedSlackDeliveryWithTerminalRepair( + args: AdvanceSlackDeliveryArgs, +): Promise { + return await args.delivery.advance(args.intent, { + beforeTerminalize: async (input) => { + await args.beforeRepair?.(input); + await repairOwnedTerminalizingSlackDelivery(input); + }, + }); +} + +/** Acquire turn ownership before advancing and repairing a Slack delivery. */ +export async function advanceSlackDeliveryWithTerminalRepair( + args: AdvanceSlackDeliveryArgs, +): Promise { + const state = getStateAdapter(); + await state.connect(); + const lock = await acquireActiveLock(state, args.intent.conversationId); + if (!lock) { + return { outcome: "pending", retryAtMs: Date.now() + 5_000 }; + } + try { + return await advanceOwnedSlackDeliveryWithTerminalRepair(args); + } finally { + await state.releaseLock(lock); + } +} + +async function scheduleCompletedDeliveryPlugins(args: { + command: RecoverableSlackDeliveryTerminalizingInput["command"]; + conversationId: string; + outcome: RecoverableSlackDeliveryOutcome; +}): Promise { + if ( + args.outcome.outcome === "accepted" && + args.command.completion.terminal.outcome === "success" + ) { + await scheduleSessionCompletedPluginTasks({ + conversationId: args.conversationId, + sessionId: args.command.completion.turnId, + }); + } +} + +/** Advance due Slack delivery intents without starting an agent run. */ +export async function recoverDueSlackDeliveries(args: { + delivery: RecoverableSlackDelivery; + limit?: number; + nowMs: number; +}): Promise { + const pending = await args.delivery.listDue({ + limit: args.limit ?? DEFAULT_RECOVERY_LIMIT, + nowMs: args.nowMs, + }); + let recovered = 0; + for (const intent of pending) { + try { + const outcome = await advanceSlackDeliveryWithTerminalRepair({ + delivery: args.delivery, + intent, + }); + await scheduleCompletedDeliveryPlugins({ + command: intent.command, + conversationId: intent.conversationId, + outcome, + }); + recovered += 1; + } catch (error) { + logException( + error, + "slack_delivery_recovery_failed", + { conversationId: intent.conversationId }, + { + "app.delivery.id": intent.deliveryId, + "app.turn.id": intent.turnId, + }, + "Heartbeat Slack delivery recovery failed", + ); + } + } + return recovered; +} + +async function recoverSlackDeliveryForTurnWithOwnership(args: { + conversationId: string; + delivery?: RecoverableSlackDelivery; + ownsActiveLock: boolean; + turnId: string; +}): Promise { + if (!args.delivery) return undefined; + const pending = await args.delivery.loadByTurn(args); + if (pending) { + const advance = args.ownsActiveLock + ? advanceOwnedSlackDeliveryWithTerminalRepair + : advanceSlackDeliveryWithTerminalRepair; + const outcome = await advance({ + delivery: args.delivery, + intent: pending, + }); + await scheduleCompletedDeliveryPlugins({ + command: pending.command, + conversationId: pending.conversationId, + outcome, + }); + return outcome; + } + const terminal = await args.delivery.loadTerminalOutcome({ + conversationId: args.conversationId, + turnId: args.turnId, + acceptanceEvidence: "visible_assistant", + }); + return terminal ? { outcome: terminal.deliveryOutcome } : undefined; +} + +/** Recover one known turn after acquiring active conversation ownership. */ +export async function recoverSlackDeliveryForTurn(args: { + conversationId: string; + delivery?: RecoverableSlackDelivery; + turnId: string; +}): Promise { + return await recoverSlackDeliveryForTurnWithOwnership({ + ...args, + ownsActiveLock: false, + }); +} + +/** Recover one known turn while the caller owns active conversation state. */ +export async function recoverOwnedSlackDeliveryForTurn(args: { + conversationId: string; + delivery?: RecoverableSlackDelivery; + turnId: string; +}): Promise { + return await recoverSlackDeliveryForTurnWithOwnership({ + ...args, + ownsActiveLock: true, + }); +} diff --git a/packages/junior/src/chat/runtime/slack-resume.ts b/packages/junior/src/chat/runtime/slack-resume.ts index 57e8562a5..9a9752633 100644 --- a/packages/junior/src/chat/runtime/slack-resume.ts +++ b/packages/junior/src/chat/runtime/slack-resume.ts @@ -20,12 +20,6 @@ import { finalizeFailedTurnReplyWithEvent, requireTurnFailureEventId, } from "@/chat/services/turn-failure-response"; -import { - ConversationTurnLifecycleService, - type ConversationTurnLifecycle, -} from "@/chat/conversations/turn-lifecycle"; -import type { ConversationTurnFailureCode } from "@/chat/conversations/history"; -import { getConversationEventStore } from "@/chat/db"; import { persistCompletedSessionRecord } from "@/chat/services/turn-session-record"; import { persistThreadStateById } from "@/chat/runtime/thread-state"; import { @@ -43,6 +37,9 @@ import { } from "@/chat/slack/reply"; import { isUserActor, type Actor } from "@/chat/actor"; import { postSlackMessage as postSlackApiMessage } from "@/chat/slack/outbound"; +import { createSlackDeliveryLocator } from "@/chat/slack/outbound"; +import type { RecoverableSlackDelivery } from "@/chat/slack/recoverable-delivery"; +import { buildPendingSlackDeliveryCommandDraft } from "@/chat/slack/delivery-command"; import { getStateAdapter } from "@/chat/state/adapter"; import { acquireActiveLock } from "@/chat/state/locks"; import { @@ -52,10 +49,13 @@ import { import type { SlackMessageTs } from "@/chat/slack/timestamp"; import { buildAuthPauseResponse } from "@/chat/services/auth-pause-response"; import { getTurnRequestDeadline } from "@/chat/runtime/request-deadline"; +import type { ConversationTurnLifecycle } from "@/chat/conversations/turn-lifecycle"; +import { loadConversationProjection } from "@/chat/conversations/projection"; import { TurnSliceLimitExceededError, buildTurnLimitResponse, } from "@/chat/services/turn-limit"; +import { advanceOwnedSlackDeliveryWithTerminalRepair } from "@/chat/runtime/slack-delivery-recovery"; function resolveReplyTimeoutMs(explicitTimeoutMs?: number): number | undefined { if (typeof explicitTimeoutMs === "number" && explicitTimeoutMs > 0) { @@ -139,6 +139,14 @@ export class ResumeTurnBusyError extends Error { } } +/** Leaves a durable delivery intent for a later callback or conversation turn. */ +export class ResumeDeliveryPendingError extends Error { + constructor(readonly retryAtMs: number) { + super("Resumed Slack delivery is awaiting retry or reconciliation"); + this.name = "ResumeDeliveryPendingError"; + } +} + interface ResumeSlackTurnArgs { messageText: string; channelId: string; @@ -149,7 +157,9 @@ interface ResumeSlackTurnArgs { initialText?: string; initialStatus?: AssistantStatusSpec; agentRunner: AgentRunner; + recoverableSlackDelivery?: RecoverableSlackDelivery; inputMessageIds?: string[]; + sliceId?: number; lifecycleCorrelation?: ResumeLifecycleCorrelation; turnLifecycle?: ConversationTurnLifecycle; scheduleSessionCompletedPluginTasks?: (params: { @@ -157,6 +167,7 @@ interface ResumeSlackTurnArgs { sessionId: string; }) => Promise; onSuccess?: (reply: AgentRunResult) => Promise; + onRecoveredSuccess?: () => Promise; onFailure?: (error: unknown) => Promise; onAuthPause?: (pause: { providerDisplayName: string }) => Promise; onTimeoutPause?: (resume: { resumeVersion: number }) => Promise; @@ -171,37 +182,6 @@ type ResumeReplyContext = Omit & { input?: Omit; }; -interface ResumeLifecycleCorrelation { - conversationId: string; - turnId: string; -} - -interface ResumeLifecycleContext extends ResumeLifecycleCorrelation { - inputMessageIds?: string[]; - service: ConversationTurnLifecycle; -} - -function getResumeLifecycleContext( - args: ResumeSlackTurnArgs, - request?: AgentRunRequest, -): ResumeLifecycleContext | undefined { - const correlation = request?.routing.correlation; - const conversationId = - correlation?.conversationId ?? args.lifecycleCorrelation?.conversationId; - const turnId = correlation?.turnId ?? args.lifecycleCorrelation?.turnId; - if (!conversationId || !turnId) return undefined; - return { - conversationId, - turnId, - ...(args.inputMessageIds?.length - ? { inputMessageIds: [...new Set(args.inputMessageIds)] } - : {}), - service: - args.turnLifecycle ?? - new ConversationTurnLifecycleService(getConversationEventStore()), - }; -} - function getDefaultLockKey(channelId: string, threadTs: string): string { return `slack:${channelId}:${threadTs}`; } @@ -227,6 +207,31 @@ function getResumeLogContext( }; } +async function scheduleResumeCompletedPluginTasks(args: { + conversationId: string; + logContext: LogContext; + schedule: (params: { + conversationId: string; + sessionId: string; + }) => Promise; + sessionId: string; +}): Promise { + try { + await args.schedule({ + conversationId: args.conversationId, + sessionId: args.sessionId, + }); + } catch (error) { + logException( + error, + "plugin_session_completed_task_schedule_failed", + args.logContext, + {}, + "Plugin session.completed task scheduling failed", + ); + } +} + /** Resolve the conversation identifier used by resumed-turn logs and Slack footers. */ function getResumeConversationId( args: ResumeSlackTurnArgs, @@ -251,16 +256,63 @@ async function postResumeFailureReply(args: { }); } +interface ResumeLifecycleCorrelation { + conversationId: string; + turnId: string; +} + +interface ResumeLifecycleContext extends ResumeLifecycleCorrelation { + inputMessageIds?: string[]; +} + +interface StartedResumeLifecycleContext extends ResumeLifecycleCorrelation { + inputMessageIds: string[]; +} + +function getResumeLifecycleContext( + args: ResumeSlackTurnArgs, + request: AgentRunRequest, +): StartedResumeLifecycleContext | undefined { + const conversationId = request.routing.correlation?.conversationId; + const turnId = request.routing.correlation?.turnId; + if (!conversationId || !turnId) { + return undefined; + } + if (!args.inputMessageIds?.length) { + throw new TypeError( + "Correlated Slack resume requires durable input message IDs", + ); + } + if (!args.turnLifecycle) { + throw new TypeError("Correlated Slack resume requires turn lifecycle"); + } + return { + conversationId, + turnId, + inputMessageIds: [...new Set(args.inputMessageIds)], + }; +} + async function handleResumeFailure(args: { body: string; error: unknown; eventName: string; lockKey: string; lifecycle?: ResumeLifecycleContext; - failureCode: ConversationTurnFailureCode; resumeArgs: ResumeSlackTurnArgs; }): Promise { const logContext = getResumeLogContext(args.resumeArgs, args.lockKey); + if (args.lifecycle) { + if (args.lifecycle.inputMessageIds?.length) { + await args.resumeArgs.turnLifecycle!.start({ + conversationId: args.lifecycle.conversationId, + turnId: args.lifecycle.turnId, + inputMessageIds: args.lifecycle.inputMessageIds, + createdAtMs: Date.now(), + surface: "slack", + }); + } + } const capturedEventId = logException( args.error, args.eventName, @@ -281,13 +333,14 @@ async function handleResumeFailure(args: { "Failed to persist resumed turn failure state", ); try { - await args.lifecycle?.service.fail({ - conversationId: args.lifecycle.conversationId, - turnId: args.lifecycle.turnId, - createdAtMs: Date.now(), - failureCode: "persistence_failed", - ...(persistEventId ? { eventId: persistEventId } : {}), - }); + if (args.lifecycle) { + await args.resumeArgs.turnLifecycle!.fail({ + ...args.lifecycle, + createdAtMs: Date.now(), + failureCode: "persistence_failed", + ...(persistEventId ? { eventId: persistEventId } : {}), + }); + } } catch (lifecycleError) { logException( lifecycleError, @@ -309,19 +362,20 @@ async function handleResumeFailure(args: { } catch (deliveryError) { const deliveryEventId = logException( deliveryError, - "slack_resume_failure_delivery_failed", + "slack_resume_failure_reply_post_failed", logContext, { "app.error.original_event_id": eventId }, - "Failed to deliver resumed turn failure state", + "Failed to post resumed turn failure reply", ); try { - await args.lifecycle?.service.fail({ - conversationId: args.lifecycle.conversationId, - turnId: args.lifecycle.turnId, - createdAtMs: Date.now(), - failureCode: "delivery_failed", - ...(deliveryEventId ? { eventId: deliveryEventId } : {}), - }); + if (args.lifecycle) { + await args.resumeArgs.turnLifecycle!.fail({ + ...args.lifecycle, + createdAtMs: Date.now(), + failureCode: "delivery_failed", + ...(deliveryEventId ? { eventId: deliveryEventId } : {}), + }); + } } catch (lifecycleError) { logException( lifecycleError, @@ -336,13 +390,14 @@ async function handleResumeFailure(args: { if (failureStatePersistError) { throw failureStatePersistError; } - await args.lifecycle?.service.fail({ - conversationId: args.lifecycle.conversationId, - turnId: args.lifecycle.turnId, - createdAtMs: Date.now(), - failureCode: args.failureCode, - eventId, - }); + if (args.lifecycle) { + await args.resumeArgs.turnLifecycle!.fail({ + ...args.lifecycle, + createdAtMs: Date.now(), + failureCode: "agent_run_failed", + eventId, + }); + } } function createResumeReplyContext( @@ -357,6 +412,9 @@ function createResumeReplyContext( throw new TypeError("Slack resume requires a reply context source"); } const source = replyContext.routing.source; + if (source.platform !== "slack") { + throw new TypeError("Slack resume requires a Slack source"); + } if (replyContext.routing.destination.platform !== "slack") { throw new TypeError("Slack resume requires a Slack destination"); } @@ -452,9 +510,9 @@ export async function resumeSlackTurn( let deferredPauseHandler: (() => Promise) | undefined; let deferredFailureHandler: (() => Promise) | undefined; let finalReplyDelivered = false; + let recoverableDeliveryTerminalized = false; let postDeliveryCommitError: unknown; - let lifecycle = getResumeLifecycleContext(args); - let failureCode: ConversationTurnFailureCode = "agent_run_failed"; + let lifecycle: ResumeLifecycleContext | undefined = args.lifecycleCorrelation; let runArgs = args; try { const preparedArgs = await args.beforeStart?.(); @@ -516,15 +574,108 @@ export async function resumeSlackTurn( status.update(runArgs.initialStatus); const replyContext = createResumeReplyContext(runArgs, status); - lifecycle = getResumeLifecycleContext(runArgs, replyContext) ?? lifecycle; - if (lifecycle?.inputMessageIds?.length) { - await lifecycle.service.start({ - conversationId: lifecycle.conversationId, - turnId: lifecycle.turnId, + const lifecycleCandidate = getResumeLifecycleContext(runArgs, replyContext); + if (lifecycleCandidate) { + lifecycle = lifecycleCandidate; + await runArgs.turnLifecycle!.start({ + ...lifecycleCandidate, createdAtMs: Date.now(), - inputMessageIds: lifecycle.inputMessageIds, surface: "slack", }); + const pendingDelivery = + await runArgs.recoverableSlackDelivery?.loadByTurn({ + conversationId: lifecycleCandidate.conversationId, + turnId: lifecycleCandidate.turnId, + }); + if (pendingDelivery) { + let repairedBeforeTerminal = false; + const recovery = await advanceOwnedSlackDeliveryWithTerminalRepair({ + delivery: runArgs.recoverableSlackDelivery!, + intent: pendingDelivery, + beforeRepair: async (input) => { + if (input.deliveryOutcome === "accepted") { + await runArgs.onRecoveredSuccess?.(); + } else { + await runArgs.onFailure?.( + new Error("Slack rejected the resumed turn reply"), + ); + } + repairedBeforeTerminal = true; + }, + }); + if (recovery.outcome === "pending") { + throw new ResumeDeliveryPendingError(recovery.retryAtMs); + } + if (repairedBeforeTerminal) { + finalReplyDelivered = recovery.outcome === "accepted"; + recoverableDeliveryTerminalized = true; + if ( + recovery.outcome === "accepted" && + pendingDelivery.command.completion.terminal.outcome === "success" + ) { + await scheduleResumeCompletedPluginTasks({ + conversationId: lifecycleCandidate.conversationId, + sessionId: lifecycleCandidate.turnId, + schedule: + runArgs.scheduleSessionCompletedPluginTasks ?? + scheduleSessionCompletedPluginTasks, + logContext: getResumeLogContext(runArgs, lockKey), + }); + } + return true; + } + } + const priorTerminal = + await runArgs.recoverableSlackDelivery?.loadTerminalOutcome({ + conversationId: lifecycleCandidate.conversationId, + turnId: lifecycleCandidate.turnId, + acceptanceEvidence: "visible_assistant", + }); + if (priorTerminal) { + if (priorTerminal.deliveryOutcome === "failed") { + await runArgs.onFailure?.( + new Error("Slack rejected the resumed turn reply"), + ); + } else { + const projection = await loadConversationProjection({ + conversationId: lifecycleCandidate.conversationId, + }); + await persistCompletedSessionRecord({ + conversationId: lifecycleCandidate.conversationId, + sessionId: lifecycleCandidate.turnId, + allMessages: projection.messages, + modelId: projection.modelId ?? botConfig.modelId, + destination: replyContext.routing.destination, + destinationVisibility: replyContext.routing.destinationVisibility, + source: replyContext.routing.source, + actor: resumeActor, + surface: "slack", + logContext: { + threadId: replyContext.routing.correlation?.threadId, + actorId: isUserActor(replyContext.routing.actor) + ? replyContext.routing.actor.userId + : undefined, + channelId: runArgs.channelId, + runId: replyContext.routing.correlation?.runId, + assistantUserName: botConfig.userName, + }, + }); + finalReplyDelivered = true; + recoverableDeliveryTerminalized = true; + await runArgs.onRecoveredSuccess?.(); + if (priorTerminal.modelSucceeded) { + await scheduleResumeCompletedPluginTasks({ + conversationId: lifecycleCandidate.conversationId, + sessionId: lifecycleCandidate.turnId, + schedule: + runArgs.scheduleSessionCompletedPluginTasks ?? + scheduleSessionCompletedPluginTasks, + logContext: getResumeLogContext(runArgs, lockKey), + }); + } + } + return true; + } } const replyPromise = runArgs.agentRunner.run(replyContext); const replyTimeoutMs = resolveReplyTimeoutMs(runArgs.replyTimeoutMs); @@ -575,9 +726,8 @@ export async function resumeSlackTurn( `Resumed run ended ${outcome.status} without a pause handler`, ), eventName: "slack_resume_turn_failed", - failureCode: "agent_run_failed", - lifecycle, lockKey, + lifecycle, resumeArgs: runArgs, }); }; @@ -594,16 +744,106 @@ export async function resumeSlackTurn( const footer = buildSlackReplyFooter({ conversationId: getResumeConversationId(runArgs, lockKey), }); - const plannedPosts = planSlackReplyPosts({ reply }); - failureCode = "delivery_failed"; - await postSlackApiReplyPosts({ - channelId: runArgs.channelId, - threadTs: runArgs.threadTs, - posts: plannedPosts, - footer, - }); + const plannedPosts = planSlackReplyPosts({ reply }).filter( + (post) => post.text.trim().length > 0, + ); + if ( + lifecycle && + plannedPosts.length > 0 && + runArgs.recoverableSlackDelivery + ) { + const deliverySource = replyContext.routing.source; + const deliveryDestination = replyContext.routing.destination; + if ( + deliverySource.platform !== "slack" || + deliveryDestination.platform !== "slack" + ) { + throw new TypeError( + "Slack resume delivery requires Slack coordinates", + ); + } + const intent = await runArgs.recoverableSlackDelivery.createIntent({ + conversationId: lifecycle.conversationId, + turnId: lifecycle.turnId, + deliveryId: `slack:${lifecycle.turnId}`, + modelMessages: + reply.piMessages ?? replyContext.input.piMessages ?? [], + command: buildPendingSlackDeliveryCommandDraft({ + route: { + channelId: runArgs.channelId, + threadTs: runArgs.threadTs, + }, + publicLocator: createSlackDeliveryLocator(), + session: { + surface: "slack", + source: deliverySource, + destination: deliveryDestination, + destinationVisibility: replyContext.routing.destinationVisibility, + actor: resumeActor, + startedAtMs: Date.now(), + }, + posts: plannedPosts, + footer, + turnId: lifecycle.turnId, + inputMessageIds: lifecycle.inputMessageIds ?? [], + assistantText: reply.text, + assistantCreatedAtMs: Date.now(), + assistantUserName: botConfig.userName, + diagnostics: reply.diagnostics, + sliceId: runArgs.sliceId ?? 1, + failureEventId: finalized.eventId, + }), + }); + let repairedBeforeTerminal = false; + const delivery = await advanceOwnedSlackDeliveryWithTerminalRepair({ + delivery: runArgs.recoverableSlackDelivery, + intent, + beforeRepair: async (input) => { + if (input.deliveryOutcome === "accepted") { + await runArgs.onSuccess?.(reply); + } else { + await runArgs.onFailure?.( + new Error("Slack rejected the resumed turn reply"), + ); + } + repairedBeforeTerminal = true; + }, + }); + if (delivery.outcome === "pending") { + throw new ResumeDeliveryPendingError(delivery.retryAtMs); + } + recoverableDeliveryTerminalized = true; + if (delivery.outcome === "failed") { + if (!repairedBeforeTerminal) { + await runArgs.onFailure?.( + new Error("Slack rejected the resumed turn reply"), + ); + } + return true; + } + if (repairedBeforeTerminal) { + finalReplyDelivered = true; + if (reply.diagnostics.outcome === "success") { + await scheduleResumeCompletedPluginTasks({ + conversationId: lifecycle.conversationId, + sessionId: lifecycle.turnId, + schedule: + runArgs.scheduleSessionCompletedPluginTasks ?? + scheduleSessionCompletedPluginTasks, + logContext: getResumeLogContext(runArgs, lockKey), + }); + } + return true; + } + } else { + await postSlackApiReplyPosts({ + channelId: runArgs.channelId, + threadTs: runArgs.threadTs, + posts: plannedPosts, + footer, + }); + } finalReplyDelivered = true; - failureCode = "persistence_failed"; // Destination acceptance is the completion boundary: only now commit the // final assistant messages and the terminal completed session record. // Persistence is retried and any remaining failure reaches this runtime @@ -636,18 +876,19 @@ export async function resumeSlackTurn( }); } await runArgs.onSuccess?.(reply); - if (lifecycle) { + if (lifecycle && !recoverableDeliveryTerminalized) { if (reply.diagnostics.outcome === "success") { - await lifecycle.service.complete({ + await runArgs.turnLifecycle!.complete({ conversationId: lifecycle.conversationId, turnId: lifecycle.turnId, createdAtMs: Date.now(), - outcome: plannedPosts.some((post) => post.text.trim().length > 0) - ? "success" - : "no_reply", + outcome: + planSlackReplyPosts({ reply }).length === 0 + ? "no_reply" + : "success", }); } else { - await lifecycle.service.fail({ + await runArgs.turnLifecycle!.fail({ conversationId: lifecycle.conversationId, turnId: lifecycle.turnId, createdAtMs: Date.now(), @@ -661,30 +902,40 @@ export async function resumeSlackTurn( replyContext.routing.correlation?.conversationId && replyContext.routing.correlation.turnId ) { - try { - const params = { - conversationId: replyContext.routing.correlation.conversationId, - sessionId: replyContext.routing.correlation.turnId, - }; - if (runArgs.scheduleSessionCompletedPluginTasks) { - await runArgs.scheduleSessionCompletedPluginTasks(params); - } else { - await scheduleSessionCompletedPluginTasks(params); - } - } catch (scheduleError) { - logException( - scheduleError, - "plugin_session_completed_task_schedule_failed", - getResumeLogContext(runArgs, lockKey), - {}, - "Plugin session.completed task scheduling failed", - ); - } + await scheduleResumeCompletedPluginTasks({ + conversationId: replyContext.routing.correlation.conversationId, + sessionId: replyContext.routing.correlation.turnId, + schedule: + runArgs.scheduleSessionCompletedPluginTasks ?? + scheduleSessionCompletedPluginTasks, + logContext: getResumeLogContext(runArgs, lockKey), + }); } } } catch (error) { await status.clear(); + if (error instanceof ResumeDeliveryPendingError) { + return true; + } + + if ( + lifecycle && + (await runArgs.recoverableSlackDelivery?.loadByTurn({ + conversationId: lifecycle.conversationId, + turnId: lifecycle.turnId, + })) + ) { + logException( + error, + "slack_resume_delivery_repair_deferred", + getResumeLogContext(runArgs, lockKey), + {}, + "Deferred resumed delivery repair to heartbeat", + ); + return true; + } + if (finalReplyDelivered) { postDeliveryCommitError = error; try { @@ -698,15 +949,30 @@ export async function resumeSlackTurn( "Failed to terminalize resumed turn after post-delivery commit failure", ); } + const eventId = logException( + error, + "slack_resume_success_handler_failed", + getResumeLogContext(runArgs, lockKey), + {}, + "Failed to persist resumed turn state after final reply delivery", + ); + if (lifecycle && !recoverableDeliveryTerminalized) { + await runArgs.turnLifecycle!.fail({ + conversationId: lifecycle.conversationId, + turnId: lifecycle.turnId, + createdAtMs: Date.now(), + failureCode: "persistence_failed", + ...(eventId ? { eventId } : {}), + }); + } } else { deferredFailureHandler = async () => { await handleResumeFailure({ body: "Failed to resume Slack turn", error, eventName: "slack_resume_turn_failed", - failureCode, - lifecycle, lockKey, + lifecycle, resumeArgs: runArgs, }); }; @@ -721,20 +987,6 @@ export async function resumeSlackTurn( } if (postDeliveryCommitError) { - const eventId = logException( - postDeliveryCommitError, - "slack_resume_success_handler_failed", - getResumeLogContext(runArgs, lockKey), - {}, - "Failed to persist resumed turn state after final reply delivery", - ); - await lifecycle?.service.fail({ - conversationId: lifecycle.conversationId, - turnId: lifecycle.turnId, - createdAtMs: Date.now(), - failureCode: "persistence_failed", - ...(eventId ? { eventId } : {}), - }); throw postDeliveryCommitError; } @@ -761,9 +1013,8 @@ export async function resumeSlackTurn( body: "Failed to handle resumed turn pause", error: pauseError, eventName: "slack_resume_pause_handler_failed", - failureCode: "persistence_failed", - lifecycle, lockKey, + lifecycle, resumeArgs: runArgs, }); return true; @@ -787,10 +1038,13 @@ export async function resumeAuthorizedRequest(args: { replyContext?: ResumeReplyContext; lockKey?: string; agentRunner: AgentRunner; + recoverableSlackDelivery?: RecoverableSlackDelivery; inputMessageIds?: string[]; + sliceId?: number; lifecycleCorrelation?: ResumeLifecycleCorrelation; turnLifecycle?: ConversationTurnLifecycle; onSuccess?: (reply: AgentRunResult) => Promise; + onRecoveredSuccess?: () => Promise; onFailure?: (error: unknown) => Promise; onAuthPause?: (pause: { providerDisplayName: string }) => Promise; onTimeoutPause?: (resume: { resumeVersion: number }) => Promise; @@ -807,10 +1061,13 @@ export async function resumeAuthorizedRequest(args: { lockKey: args.lockKey, initialText: args.connectedText, agentRunner: args.agentRunner, + recoverableSlackDelivery: args.recoverableSlackDelivery, inputMessageIds: args.inputMessageIds, + sliceId: args.sliceId, lifecycleCorrelation: args.lifecycleCorrelation, turnLifecycle: args.turnLifecycle, onSuccess: args.onSuccess, + onRecoveredSuccess: args.onRecoveredSuccess, onFailure: args.onFailure, onAuthPause: args.onAuthPause, onTimeoutPause: args.onTimeoutPause, diff --git a/packages/junior/src/chat/runtime/slack-runtime.ts b/packages/junior/src/chat/runtime/slack-runtime.ts index 088276dfe..bd77e7b85 100644 --- a/packages/junior/src/chat/runtime/slack-runtime.ts +++ b/packages/junior/src/chat/runtime/slack-runtime.ts @@ -73,6 +73,7 @@ export interface ReplyHooks { ) => Promise; messageContext?: MessageContext; ack?: () => Promise; + ackMessageIds?: (messageIds: readonly string[]) => Promise; onToolInvocation?: (invocation: TurnToolInvocation) => void; onTurnStatePersisted?: () => Promise; isFinalAttempt?: boolean; @@ -196,6 +197,7 @@ export interface SlackTurnRuntimeDependencies { destination: Destination; explicitMention?: boolean; ack?: () => Promise; + ackMessageIds?: (messageIds: readonly string[]) => Promise; onToolInvocation?: (invocation: TurnToolInvocation) => void; onTurnCompleted?: () => Promise; onTurnStatePersisted?: () => Promise; @@ -808,6 +810,7 @@ export function createSlackTurnRuntime< destination: hooks.destination, queuedMessages, ack, + ackMessageIds: hooks.ackMessageIds, onToolInvocation: toolInvocationHook, onTurnCompleted, drainSteeringMessages, @@ -1128,6 +1131,7 @@ export function createSlackTurnRuntime< beforeFirstResponsePost: hooks.beforeFirstResponsePost, queuedMessages, ack, + ackMessageIds: hooks.ackMessageIds, onToolInvocation: toolInvocationHook, onTurnCompleted, drainSteeringMessages, diff --git a/packages/junior/src/chat/runtime/turn.ts b/packages/junior/src/chat/runtime/turn.ts index 4565f50f5..b6a333069 100644 --- a/packages/junior/src/chat/runtime/turn.ts +++ b/packages/junior/src/chat/runtime/turn.ts @@ -76,10 +76,18 @@ export function isTurnInputCommitLostError( /** Error indicating durable turn input should stay pending for a later worker. */ export class TurnInputDeferredError extends Error { readonly code = "turn_input_deferred"; - - constructor(message = "Turn input is deferred until the active resume ends") { + readonly retryAfterMs?: number; + readonly immediate: boolean; + + constructor( + message = "Turn input is deferred until the active resume ends", + retryAfterMs?: number, + immediate = false, + ) { super(message); this.name = "TurnInputDeferredError"; + this.retryAfterMs = retryAfterMs; + this.immediate = immediate; } } diff --git a/packages/junior/src/chat/services/agent-continue.ts b/packages/junior/src/chat/services/agent-continue.ts index d6f5edd08..0e2ff3a6e 100644 --- a/packages/junior/src/chat/services/agent-continue.ts +++ b/packages/junior/src/chat/services/agent-continue.ts @@ -6,13 +6,33 @@ */ import type { StateAdapter } from "chat"; import type { Destination } from "@sentry/junior-plugin-api"; -import { getAgentTurnSessionRecord } from "@/chat/state/turn-session"; +import { logException, logWarn } from "@/chat/logging"; +import { + activateAgentTurnAuthorizationRecovery, + getAgentTurnSessionRecord, + isAgentTurnAuthorizationRecoveryActive, + prepareAgentTurnAuthorizationRecovery, + type AgentTurnSessionRecord, +} from "@/chat/state/turn-session"; +import { getStateAdapter } from "@/chat/state/adapter"; +import { acquireActiveLock } from "@/chat/state/locks"; +import { createUserTokenStore } from "@/chat/capabilities/factory"; +import { getMcpStoredOAuthCredentials } from "@/chat/mcp/auth-store"; import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; import { ensureConversationWake, requestConversationWork, } from "@/chat/task-execution/store"; import { getVercelConversationWorkQueue } from "@/chat/task-execution/vercel-queue"; +import { sleep } from "@/chat/sleep"; +import { + AUTHORIZATION_RECOVERY_MISSING_RECORD_GRACE_MS, + listAuthorizationRecoveries, + removeAuthorizationRecovery, + type AuthorizationRecoveryIndexEntry, +} from "@/chat/state/authorization-recovery-index"; + +const AUTHORIZATION_RECOVERY_LOCK_RETRY_MS = 250; export interface AgentContinueRequest { conversationId: string; @@ -27,6 +47,266 @@ export interface ScheduleAgentContinueOptions { state?: StateAdapter; } +interface AuthorizationRecoveryIdentity { + authorizationCompletionId: string; + conversationId: string; + expectedVersion: number; + sessionId: string; +} + +async function removeAuthorizationRecoveryBestEffort( + entry: Pick< + AuthorizationRecoveryIndexEntry, + "authorizationCompletionId" | "conversationId" | "sessionId" + >, + state: StateAdapter, +): Promise { + try { + await removeAuthorizationRecovery(state, entry); + } catch (error) { + logException( + error, + "authorization_recovery_index_cleanup_failed", + { conversationId: entry.conversationId }, + { "app.ai.session_id": entry.sessionId }, + "Failed to remove a completed authorization recovery index entry", + ); + } +} + +async function acquireAuthorizationRecoveryLock( + conversationId: string, +): Promise>> { + const state = getStateAdapter(); + const immediate = await acquireActiveLock(state, conversationId); + if (immediate) return immediate; + await sleep(AUTHORIZATION_RECOVERY_LOCK_RETRY_MS); + return await acquireActiveLock(state, conversationId); +} + +/** Prepare an auth recovery without racing a mailbox-owned terminal transition. */ +export async function prepareAgentTurnAuthorizationRecoveryUnderActiveLock( + args: Parameters[0], +): Promise { + const state = getStateAdapter(); + await state.connect(); + const lock = await acquireAuthorizationRecoveryLock(args.conversationId); + if (!lock) return undefined; + try { + return await prepareAgentTurnAuthorizationRecovery(args); + } finally { + await state.releaseLock(lock); + } +} + +/** + * Activate an exact parked auth session and durably wake it under turn ownership. + */ +export async function activateAndScheduleAgentTurnAuthorizationRecovery( + args: AuthorizationRecoveryIdentity, + options: ScheduleAgentContinueOptions = {}, +): Promise { + const state = getStateAdapter(); + await state.connect(); + const lock = await acquireAuthorizationRecoveryLock(args.conversationId); + if (!lock) return undefined; + try { + const current = await getAgentTurnSessionRecord( + args.conversationId, + args.sessionId, + ); + if ( + !current || + current.state !== "awaiting_resume" || + current.resumeReason !== "auth" || + current.version !== args.expectedVersion || + current.authorizationRecovery?.authorizationCompletionId !== + args.authorizationCompletionId + ) { + return undefined; + } + + const activated = current.authorizationRecovery.active + ? current + : await activateAgentTurnAuthorizationRecovery(args); + if (!activated?.destination) return undefined; + await scheduleAgentContinue( + { + conversationId: activated.conversationId, + destination: activated.destination, + expectedVersion: activated.version, + sessionId: activated.sessionId, + }, + options, + ); + await removeAuthorizationRecoveryBestEffort(args, state); + return activated; + } finally { + await state.releaseLock(lock); + } +} + +/** Mark an exact auth callback complete, then durably wake its paused session. */ +export async function wakeAuthorizationCompletedAgentTurn( + args: { + conversationId: string; + provider: string; + sessionId: string; + }, + options: ScheduleAgentContinueOptions = {}, +): Promise { + try { + const sessionRecord = await getAgentTurnSessionRecord( + args.conversationId, + args.sessionId, + ); + if (!sessionRecord?.destination) { + logWarn( + "authorization_completed_resume_not_schedulable", + { conversationId: args.conversationId }, + { "app.ai.session_id": args.sessionId }, + "Authorization callback found no durable session destination to wake", + ); + return; + } + const recovery = sessionRecord.authorizationRecovery; + if (!recovery?.active) return; + await activateAndScheduleAgentTurnAuthorizationRecovery( + { + authorizationCompletionId: recovery.authorizationCompletionId, + conversationId: sessionRecord.conversationId, + expectedVersion: sessionRecord.version, + sessionId: sessionRecord.sessionId, + }, + options, + ); + } catch (error) { + logException( + error, + "authorization_completed_resume_schedule_failed", + { conversationId: args.conversationId }, + { + "app.ai.session_id": args.sessionId, + "app.credential.provider": args.provider, + }, + "Failed to schedule an authorized turn after its callback found the conversation busy", + ); + } +} + +/** Re-drive completed auth callbacks from indexed turn-session recovery state. */ +export async function recoverAuthorizationCompletedAgentTurns( + options: ScheduleAgentContinueOptions = {}, +): Promise { + const state = options.state ?? getStateAdapter(); + const nowMs = options.nowMs ?? Date.now(); + const recoveries = await listAuthorizationRecoveries(state, nowMs); + let recovered = 0; + for (const indexed of recoveries) { + try { + const session = await getAgentTurnSessionRecord( + indexed.conversationId, + indexed.sessionId, + ); + const missingRecordIsStale = + nowMs - indexed.registeredAtMs >= + AUTHORIZATION_RECOVERY_MISSING_RECORD_GRACE_MS; + if (!session) { + if (missingRecordIsStale) { + await removeAuthorizationRecoveryBestEffort(indexed, state); + } + continue; + } + if ( + session.state === "completed" || + session.state === "failed" || + session.state === "abandoned" + ) { + await removeAuthorizationRecoveryBestEffort(indexed, state); + continue; + } + const recovery = session.authorizationRecovery; + if (!recovery) { + if (missingRecordIsStale) { + await removeAuthorizationRecoveryBestEffort(indexed, state); + } + continue; + } + if ( + recovery.authorizationCompletionId !== indexed.authorizationCompletionId + ) { + await removeAuthorizationRecoveryBestEffort(indexed, state); + continue; + } + if ( + session.state !== "awaiting_resume" || + session.resumeReason !== "auth" || + !session.destination + ) { + continue; + } + if (!recovery.active) { + const committed = + recovery.authorizationKind === "plugin" + ? await createUserTokenStore().hasAuthorizationCompletion( + recovery.userId, + recovery.provider, + recovery.authorizationCompletionId, + ) + : ( + await getMcpStoredOAuthCredentials( + recovery.userId, + recovery.provider, + ) + )?.authorizationCompletionId === + recovery.authorizationCompletionId; + if (!committed) continue; + } + const completed = await activateAndScheduleAgentTurnAuthorizationRecovery( + { + authorizationCompletionId: recovery.authorizationCompletionId, + conversationId: indexed.conversationId, + expectedVersion: session.version, + sessionId: indexed.sessionId, + }, + options, + ); + if (!completed) continue; + recovered += 1; + } catch (error) { + logException( + error, + "authorization_completed_resume_recovery_failed", + { conversationId: indexed.conversationId }, + { "app.ai.session_id": indexed.sessionId }, + "Failed to recover an authorized turn awaiting a durable wake", + ); + } + } + return recovered; +} + +async function ensureAgentContinueWake(args: { + nowMs: number; + options: ScheduleAgentContinueOptions; + request: AgentContinueRequest; +}): Promise { + const queue = args.options.queue ?? getVercelConversationWorkQueue(); + await ensureConversationWake({ + conversationId: args.request.conversationId, + idempotencyKey: [ + "agent-continue", + args.request.conversationId, + args.request.sessionId, + args.request.expectedVersion, + args.nowMs, + ].join(":"), + nowMs: args.nowMs, + queue, + state: args.options.state, + }); +} + /** Build the queue request for an awaiting automatic agent continuation. */ export async function getAwaitingAgentContinueRequest(args: { conversationId: string; @@ -36,11 +316,20 @@ export async function getAwaitingAgentContinueRequest(args: { args.conversationId, args.sessionId, ); + if (!sessionRecord || sessionRecord.state !== "awaiting_resume") { + return undefined; + } + const authorizationCompleted = + sessionRecord.resumeReason === "auth" && + (await isAgentTurnAuthorizationRecoveryActive({ + conversationId: args.conversationId, + expectedVersion: sessionRecord.version, + sessionId: args.sessionId, + })); if ( - !sessionRecord || - sessionRecord.state !== "awaiting_resume" || (sessionRecord.resumeReason !== "timeout" && - sessionRecord.resumeReason !== "yield") || + sessionRecord.resumeReason !== "yield" && + !authorizationCompleted) || (sessionRecord.resumeReason === "timeout" && sessionRecord.sliceId < 2) ) { return undefined; @@ -69,18 +358,9 @@ export async function scheduleAgentContinue( nowMs, state: options.state, }); - const queue = options.queue ?? getVercelConversationWorkQueue(); - await ensureConversationWake({ - conversationId: request.conversationId, - idempotencyKey: [ - "agent-continue", - request.conversationId, - request.sessionId, - request.expectedVersion, - nowMs, - ].join(":"), + await ensureAgentContinueWake({ nowMs, - queue, - state: options.state, + options, + request, }); } diff --git a/packages/junior/src/chat/slack/README.md b/packages/junior/src/chat/slack/README.md index 6d2589518..eaec669db 100644 --- a/packages/junior/src/chat/slack/README.md +++ b/packages/junior/src/chat/slack/README.md @@ -23,6 +23,51 @@ runtime orchestration. model-provided paths or destinations. - Reactions and status messages are progress UI, not completion contracts. - OAuth links and other private authorization material use private delivery. +- Recoverable reply writes carry only an opaque `junior_delivery` marker. They + are attempted once; ambiguous writes are reconciled through one + `conversations.replies` page per invocation so durable orchestration can + persist cursors and honor Slack's rate limit without risking duplicate posts. +- Reconciliation validates the marker and the current app/bot identity. Read + errors remain unresolved and never authorize a repost. Permanent API + rejections use a long operator-recovery backoff; rate limits continue to + honor Slack's `Retry-After`. + +### Recoverable Delivery Ownership + +Finalized assistant replies, including OAuth and continuation resumes, close +the external-acceptance gap through the Slack-owned pending-delivery outbox. +Private authorization and nonstandard notice paths are not covered by it. + +`delivery-command.ts` owns the strict persisted Slack reply command and progress +schemas. `delivery-outbox.ts` owns fenced SQL control state and atomic +terminalization. `recoverable-delivery.ts` advances or reconciles that state +without rerunning the model, while `outbound.ts` keeps Slack API results and +error classification inside the adapter. + +Creating an intent first commits the generated model-continuity boundary to the +canonical conversation event log. The outbox stores only its committed and +rollback event cursors, never a Pi transcript. Slack acceptance still gates the +visible assistant-message fact; a definitive rejection opens a rollback epoch +when no part was accepted, so the wholly undelivered generation remains audit +history but not live Pi context. A partial multipart delivery retains the model +generation because its tool calls and side effects happened, records the +accepted Slack prefix as the visible assistant fact, and closes with a +`delivery_failed` turn terminal so the undelivered tail is explicit in history. + +At most one unresolved delivery exists per conversation, so newer input cannot +bypass older delivery. A stale `posting` state becomes `uncertain` and cannot +be posted again until reconciliation explicitly marks it repostable after its +grace period. Terminalization persists canonical visible-message and turn facts +and deletes the pending row in one transaction. Only `turn_failed` with +`delivery_failed` means Slack rejected delivery; recovery without a live intent +also requires the atomically finalized visible assistant message before it +classifies another terminal as accepted. Permanent reconciliation rejections +remain uncertain and use an operator-recovery backoff rather than authorizing a +duplicate post. + +The authenticated heartbeat scans due intents oldest-first. It never invokes +Pi, and repairs session/thread projections before terminalization deletes the +last durable retry marker. `outbound.ts` owns Slack API calls and retry classification. `mrkdwn.ts` owns format conversion. `assistant-thread/` owns assistant-thread lifecycle and diff --git a/packages/junior/src/chat/slack/delivery-command.ts b/packages/junior/src/chat/slack/delivery-command.ts new file mode 100644 index 000000000..90d3ea92c --- /dev/null +++ b/packages/junior/src/chat/slack/delivery-command.ts @@ -0,0 +1,297 @@ +/** Exact persisted command and progress schemas for recoverable Slack replies. */ +import { z } from "zod"; +import { + actorSchema, + slackDestinationSchema, + sourceSchema, +} from "@sentry/junior-plugin-api"; +import { agentTurnUsageSchema } from "@/chat/usage"; +import { buildDeterministicAssistantMessageId } from "@/chat/state/turn-id"; +import { conversationTurnFailureCodeSchema } from "@/chat/conversations/turn-failure"; +import type { AgentTurnDiagnostics } from "@/chat/services/turn-result"; +import { + buildSlackReplyBlocks, + type SlackReplyFooter, +} from "@/chat/slack/footer"; + +/** Stable identifier shared by delivery control state and canonical facts. */ +export const conversationDeliveryIdSchema = z + .string() + .min(1) + .max(160) + .regex(/^[A-Za-z0-9:_-]+$/); + +/** Privacy-safe reason that an intended delivery cannot be completed. */ +export const conversationDeliveryFailureCodeSchema = + z.literal("provider_rejected"); + +const slackMrkdwnTextSchema = z + .object({ type: z.literal("mrkdwn"), text: z.string() }) + .strict(); + +const durableSlackBlockSchema = z.union([ + z.object({ type: z.literal("markdown"), text: z.string() }).strict(), + z + .object({ type: z.literal("section"), text: slackMrkdwnTextSchema }) + .strict(), + z + .object({ + type: z.literal("context"), + elements: z.array(slackMrkdwnTextSchema).min(1), + }) + .strict(), +]); + +const durableDeliveryPartSchema = z + .object({ + text: z.string().min(1).max(40_000), + blocks: z.array(durableSlackBlockSchema).min(1).optional(), + }) + .strict(); + +/** + * Immutable command needed to finish one Slack assistant reply without + * rerunning the model. This deliberately excludes private authorization + * delivery, arbitrary Slack metadata, credentials, and provider responses. + */ +export const pendingConversationDeliveryCommandSchema = z + .object({ + route: z + .object({ + channelId: z.string().regex(/^[CDG][A-Z0-9]+$/), + threadTs: z + .string() + .regex(/^\d+(?:\.\d+)?$/) + .optional(), + }) + .strict(), + publicLocator: z.string().regex(/^[A-Za-z0-9_-]{22}$/), + session: z + .object({ + surface: z.enum(["slack", "api"]), + source: sourceSchema, + destination: slackDestinationSchema, + destinationVisibility: z.enum(["public", "private"]).optional(), + actor: actorSchema.optional(), + channelName: z.string().min(1).optional(), + startedAtMs: z.number().finite(), + }) + .strict(), + parts: z.array(durableDeliveryPartSchema).min(1), + completion: z + .object({ + inputMessageIds: z.array(z.string().min(1)).min(1), + assistantMessage: z + .object({ + messageId: z.string().min(1), + text: z.string(), + createdAtMs: z.number().finite(), + author: z + .object({ userName: z.string().min(1), isBot: z.literal(true) }) + .strict(), + }) + .strict(), + turnId: z.string().min(1), + model: z + .object({ + modelId: z.string().min(1), + committedSeq: z.number().int().min(-1), + rollbackSeq: z.number().int().min(-1), + }) + .strict(), + durationMs: z.number().int().nonnegative().optional(), + usage: agentTurnUsageSchema.optional(), + reasoningLevel: z.string().min(1).optional(), + sliceId: z.number().int().positive(), + terminal: z.discriminatedUnion("outcome", [ + z.object({ outcome: z.literal("success") }).strict(), + z + .object({ + outcome: z.literal("failed"), + failureCode: conversationTurnFailureCodeSchema, + eventId: z + .string() + .regex(/^[a-f0-9]{32}$/i) + .optional(), + }) + .strict(), + ]), + }) + .strict(), + }) + .strict() + .superRefine((command, ctx) => { + if ( + new Set(command.completion.inputMessageIds).size !== + command.completion.inputMessageIds.length + ) { + ctx.addIssue({ + code: "custom", + message: "delivery input message ids must be unique", + path: ["completion", "inputMessageIds"], + }); + } + if ( + command.completion.assistantMessage.messageId !== + buildDeterministicAssistantMessageId(command.completion.turnId) + ) { + ctx.addIssue({ + code: "custom", + message: "delivery assistant message id must match its turn", + path: ["completion", "assistantMessage", "messageId"], + }); + } + if ( + command.completion.model.rollbackSeq > + command.completion.model.committedSeq + ) { + ctx.addIssue({ + code: "custom", + message: "delivery rollback cursor cannot exceed its commit cursor", + path: ["completion", "model", "rollbackSeq"], + }); + } + if (command.route.channelId !== command.session.destination.channelId) { + ctx.addIssue({ + code: "custom", + message: "delivery route must match the session destination", + path: ["route"], + }); + } + }); + +/** Exact validated command retained only while delivery is unresolved. */ +export type PendingConversationDeliveryCommand = z.output< + typeof pendingConversationDeliveryCommandSchema +>; + +/** Delivery command before canonical model-event cursors are assigned. */ +export type PendingConversationDeliveryCommandDraft = Omit< + PendingConversationDeliveryCommand, + "completion" +> & { + completion: Omit< + PendingConversationDeliveryCommand["completion"], + "model" + > & { + model: { modelId: string }; + }; +}; + +export interface BuildPendingSlackDeliveryCommandDraftArgs { + assistantCreatedAtMs: number; + assistantText: string; + assistantUserName: string; + diagnostics: AgentTurnDiagnostics; + failureEventId?: string; + footer?: SlackReplyFooter; + inputMessageIds: string[]; + posts: readonly { text: string }[]; + publicLocator: string; + route: PendingConversationDeliveryCommandDraft["route"]; + session: PendingConversationDeliveryCommandDraft["session"]; + sliceId: number; + turnId: string; +} + +/** Build the shared persisted command for one finalized Slack reply. */ +export function buildPendingSlackDeliveryCommandDraft( + args: BuildPendingSlackDeliveryCommandDraftArgs, +): PendingConversationDeliveryCommandDraft { + return { + publicLocator: args.publicLocator, + route: args.route, + session: args.session, + parts: args.posts.map((post, index) => { + const blocks = buildSlackReplyBlocks( + post.text, + index === args.posts.length - 1 ? args.footer : undefined, + ); + return { + text: post.text, + ...(blocks ? { blocks } : {}), + }; + }), + completion: { + turnId: args.turnId, + inputMessageIds: args.inputMessageIds, + assistantMessage: { + messageId: buildDeterministicAssistantMessageId(args.turnId), + text: args.assistantText, + createdAtMs: args.assistantCreatedAtMs, + author: { userName: args.assistantUserName, isBot: true }, + }, + model: { modelId: args.diagnostics.modelId }, + ...(args.diagnostics.durationMs !== undefined + ? { durationMs: args.diagnostics.durationMs } + : {}), + ...(args.diagnostics.usage ? { usage: args.diagnostics.usage } : {}), + ...(args.diagnostics.reasoningLevel + ? { reasoningLevel: args.diagnostics.reasoningLevel } + : {}), + sliceId: args.sliceId, + terminal: + args.diagnostics.outcome === "success" + ? { outcome: "success" } + : { + outcome: "failed", + failureCode: "model_execution_failed", + ...(args.failureEventId ? { eventId: args.failureEventId } : {}), + }, + }, + }; +} + +const pendingDeliveryReadyStateSchema = z + .object({ status: z.literal("pending") }) + .strict(); +const pendingDeliveryPostingStateSchema = z + .object({ + status: z.literal("posting"), + attemptedAtMs: z.number().finite(), + }) + .strict(); +const pendingDeliveryUncertainStateSchema = z + .object({ + status: z.literal("uncertain"), + attemptedAtMs: z.number().finite(), + reconciliationCursor: z.string().min(1).max(512).optional(), + confirmedAbsentAtMs: z.number().finite().optional(), + }) + .strict(); +const pendingDeliveryFailedStateSchema = z + .object({ + status: z.literal("failed"), + failureCode: conversationDeliveryFailureCodeSchema, + }) + .strict(); + +/** Mutable state for the current ordered delivery part. */ +export const pendingConversationDeliveryCurrentStateSchema = z.union([ + pendingDeliveryReadyStateSchema, + pendingDeliveryPostingStateSchema, + pendingDeliveryUncertainStateSchema, + pendingDeliveryFailedStateSchema, +]); + +export type PendingConversationDeliveryCurrentState = z.output< + typeof pendingConversationDeliveryCurrentStateSchema +>; + +/** Accepted part count plus the mutable state of the next ordered part. */ +export const pendingConversationDeliveryProgressSchema = z + .object({ + acceptedPartCount: z.number().int().nonnegative(), + acceptedMessageTs: z.array(z.string().regex(/^\d+(?:\.\d+)?$/)).default([]), + currentState: pendingConversationDeliveryCurrentStateSchema, + }) + .strict() + .refine( + (progress) => + progress.acceptedMessageTs.length === progress.acceptedPartCount, + "accepted Slack receipts must align with accepted parts", + ); + +export type PendingConversationDeliveryProgress = z.output< + typeof pendingConversationDeliveryProgressSchema +>; diff --git a/packages/junior/src/chat/slack/delivery-outbox.ts b/packages/junior/src/chat/slack/delivery-outbox.ts new file mode 100644 index 000000000..0ada8f82c --- /dev/null +++ b/packages/junior/src/chat/slack/delivery-outbox.ts @@ -0,0 +1,876 @@ +/** + * SQL control state for recoverable Slack delivery. + * + * Fenced leases own every mutable transition; terminalization commits canonical + * conversation facts and deletes the outbox row under the conversation lock. + */ +import { isDeepStrictEqual } from "node:util"; +import { and, asc, eq, gt, isNull, lte, or } from "drizzle-orm"; +import { z } from "zod"; +import type { JuniorSqlDatabase } from "@/db/db"; +import { juniorConversationEvents, juniorPendingDeliveries } from "@/db/schema"; +import { sanitizePostgresJson } from "@/db/postgres-json"; +import { buildDeterministicAssistantMessageId } from "@/chat/state/turn-id"; +import { + conversationDeliveryFailureCodeSchema, + conversationDeliveryIdSchema, + pendingConversationDeliveryCommandSchema, + pendingConversationDeliveryProgressSchema, + type PendingConversationDeliveryCommand, + type PendingConversationDeliveryProgress, +} from "./delivery-command"; +import { conversationEventDataSchema } from "@/chat/conversations/history"; +import { ensureConversationRow } from "@/chat/conversations/sql/conversation-row"; +import { withConversationEventLock } from "@/chat/conversations/sql/event-lock"; + +type PendingDeliveryRow = typeof juniorPendingDeliveries.$inferSelect; + +const leaseOwnerSchema = z.string().min(1).max(160); + +/** A fenced claim that must accompany every delivery-state mutation. */ +export interface PendingDeliveryLease { + owner: string; + version: number; + expiresAtMs: number; +} + +/** Validated unresolved delivery control state. */ +export interface PendingConversationDelivery { + deliveryId: string; + conversationId: string; + turnId: string; + command: PendingConversationDeliveryCommand; + progress: PendingConversationDeliveryProgress; + nextPartIndex: number; + nextAttemptAtMs: number; + lease?: PendingDeliveryLease; +} + +/** Canonical turn-terminal interpretation after outbox finalization commits. */ +export interface PendingDeliveryTerminalOutcome { + deliveryOutcome: "accepted" | "failed"; + modelSucceeded: boolean; +} + +/** Evidence available when interpreting a canonical turn terminal. */ +export type PendingDeliveryAcceptanceEvidence = + | "known_outbox_intent" + | "visible_assistant"; + +/** Raised when a stale worker tries to mutate state after losing its fence. */ +export class PendingDeliveryLeaseLostError extends Error { + constructor() { + super("Pending delivery lease is no longer valid"); + this.name = "PendingDeliveryLeaseLostError"; + } +} + +function initialProgress(): PendingConversationDeliveryProgress { + return { + acceptedPartCount: 0, + acceptedMessageTs: [], + currentState: { status: "pending" }, + }; +} + +function parseRow(row: PendingDeliveryRow): PendingConversationDelivery { + const command = pendingConversationDeliveryCommandSchema.parse(row.command); + const progress = pendingConversationDeliveryProgressSchema.parse( + row.progress, + ); + if (progress.acceptedPartCount > command.parts.length) { + throw new Error("Pending delivery progress exceeds its command"); + } + if ( + progress.acceptedPartCount === command.parts.length && + progress.currentState.status !== "pending" + ) { + throw new Error("Completed delivery progress has an active current state"); + } + if ((row.leaseOwner === null) !== (row.leaseExpiresAt === null)) { + throw new Error("Pending delivery has a partial lease"); + } + return { + deliveryId: conversationDeliveryIdSchema.parse(row.deliveryId), + conversationId: row.conversationId, + turnId: row.turnId, + command, + progress, + nextPartIndex: progress.acceptedPartCount, + nextAttemptAtMs: row.nextAttemptAt.getTime(), + ...(row.leaseOwner && row.leaseExpiresAt + ? { + lease: { + owner: row.leaseOwner, + version: row.leaseVersion, + expiresAtMs: row.leaseExpiresAt.getTime(), + }, + } + : {}), + }; +} + +function validateTime(value: number, label: string): number { + if (!Number.isFinite(value)) throw new Error(`${label} must be finite`); + return value; +} + +async function lockedRow( + executor: JuniorSqlDatabase, + deliveryId: string, +): Promise { + const rows = await executor + .db() + .select() + .from(juniorPendingDeliveries) + .where(eq(juniorPendingDeliveries.deliveryId, deliveryId)) + .for("update"); + return rows[0]; +} + +function requireFence( + row: PendingDeliveryRow, + lease: PendingDeliveryLease, + nowMs: number, +): void { + if ( + row.leaseOwner !== lease.owner || + row.leaseVersion !== lease.version || + row.leaseExpiresAt === null || + row.leaseExpiresAt.getTime() !== lease.expiresAtMs || + row.leaseExpiresAt.getTime() <= nowMs + ) { + throw new PendingDeliveryLeaseLostError(); + } +} + +/** Create or validate immutable unresolved delivery control state. */ +export async function createPendingConversationDelivery( + executor: JuniorSqlDatabase, + args: { + conversationId: string; + turnId: string; + deliveryId: string; + command: PendingConversationDeliveryCommand; + nowMs: number; + }, +): Promise { + const deliveryId = conversationDeliveryIdSchema.parse(args.deliveryId); + const command = pendingConversationDeliveryCommandSchema.parse(args.command); + const nowMs = validateTime(args.nowMs, "nowMs"); + if (!args.conversationId || !args.turnId) { + throw new Error("Pending delivery requires conversation and turn ids"); + } + if (command.completion.turnId !== args.turnId) { + throw new Error("Pending delivery command turn does not match its row"); + } + return withConversationEventLock(executor, args.conversationId, async () => + executor.transaction(async () => { + await ensureConversationRow(executor, args.conversationId, nowMs); + if ( + await turnTerminalOutcome( + executor, + args.conversationId, + args.turnId, + "known_outbox_intent", + ) + ) { + throw new Error("Delivery is already terminalized"); + } + await executor + .db() + .insert(juniorPendingDeliveries) + .values({ + deliveryId, + conversationId: args.conversationId, + turnId: args.turnId, + command: sanitizePostgresJson(command), + progress: sanitizePostgresJson(initialProgress()), + nextAttemptAt: new Date(nowMs), + }) + .onConflictDoNothing(); + const rows = await executor + .db() + .select() + .from(juniorPendingDeliveries) + .where(eq(juniorPendingDeliveries.conversationId, args.conversationId)); + const existing = rows[0]; + if (!existing) throw new Error("Pending delivery insert was lost"); + const parsed = parseRow(existing); + if ( + parsed.deliveryId !== deliveryId || + !isDeepStrictEqual( + JSON.parse(JSON.stringify(sanitizePostgresJson(parsed.command))), + JSON.parse(JSON.stringify(sanitizePostgresJson(command))), + ) + ) { + throw new Error( + "Conversation already has a different pending delivery", + ); + } + return parsed; + }), + ); +} + +/** Load unresolved control state by its deterministic conversation turn. */ +export async function loadPendingDeliveryByTurn( + executor: JuniorSqlDatabase, + args: { conversationId: string; turnId: string }, +): Promise { + const rows = await executor + .db() + .select() + .from(juniorPendingDeliveries) + .where( + and( + eq(juniorPendingDeliveries.conversationId, args.conversationId), + eq(juniorPendingDeliveries.turnId, args.turnId), + ), + ); + return rows[0] ? parseRow(rows[0]) : undefined; +} + +/** Load the conversation's only unresolved delivery. */ +export async function loadPendingDeliveryByConversation( + executor: JuniorSqlDatabase, + args: { conversationId: string }, +): Promise { + const rows = await executor + .db() + .select() + .from(juniorPendingDeliveries) + .where(eq(juniorPendingDeliveries.conversationId, args.conversationId)) + .limit(1); + return rows[0] ? parseRow(rows[0]) : undefined; +} + +/** List due, unclaimed delivery intents for bounded background recovery. */ +export async function listDuePendingDeliveries( + executor: JuniorSqlDatabase, + args: { limit: number; nowMs: number }, +): Promise { + const nowMs = validateTime(args.nowMs, "nowMs"); + if (!Number.isInteger(args.limit) || args.limit <= 0) { + throw new Error("Pending delivery recovery limit must be positive"); + } + const now = new Date(nowMs); + const rows = await executor + .db() + .select() + .from(juniorPendingDeliveries) + .where( + and( + lte(juniorPendingDeliveries.nextAttemptAt, now), + or( + isNull(juniorPendingDeliveries.leaseExpiresAt), + lte(juniorPendingDeliveries.leaseExpiresAt, now), + ), + ), + ) + .orderBy( + asc(juniorPendingDeliveries.nextAttemptAt), + asc(juniorPendingDeliveries.deliveryId), + ) + .limit(args.limit); + return rows.map(parseRow); +} + +/** Claim a due delivery; stale `posting` state becomes uncertain, never pending. */ +export async function claimPendingConversationDelivery( + executor: JuniorSqlDatabase, + args: { + deliveryId: string; + leaseOwner: string; + nowMs: number; + leaseDurationMs: number; + }, +): Promise { + const deliveryId = conversationDeliveryIdSchema.parse(args.deliveryId); + const owner = leaseOwnerSchema.parse(args.leaseOwner); + const nowMs = validateTime(args.nowMs, "nowMs"); + if (!Number.isFinite(args.leaseDurationMs) || args.leaseDurationMs <= 0) { + throw new Error("leaseDurationMs must be positive"); + } + return executor.transaction(async () => { + const row = await lockedRow(executor, deliveryId); + if ( + !row || + row.nextAttemptAt.getTime() > nowMs || + (row.leaseExpiresAt !== null && row.leaseExpiresAt.getTime() > nowMs) + ) { + return undefined; + } + const current = parseRow(row); + const progress: PendingConversationDeliveryProgress = { + ...current.progress, + currentState: + current.progress.currentState.status === "posting" + ? { + status: "uncertain", + attemptedAtMs: current.progress.currentState.attemptedAtMs, + } + : current.progress.currentState, + }; + const expiresAtMs = nowMs + args.leaseDurationMs; + const rows = await executor + .db() + .update(juniorPendingDeliveries) + .set({ + progress: sanitizePostgresJson(progress), + leaseOwner: owner, + leaseVersion: row.leaseVersion + 1, + leaseExpiresAt: new Date(expiresAtMs), + }) + .where(eq(juniorPendingDeliveries.deliveryId, deliveryId)) + .returning(); + return rows[0] ? parseRow(rows[0]) : undefined; + }); +} + +/** Extend an unexpired claim without changing its fencing version. */ +export async function renewPendingDeliveryLease( + executor: JuniorSqlDatabase, + args: { + deliveryId: string; + lease: PendingDeliveryLease; + nowMs: number; + leaseDurationMs: number; + }, +): Promise { + const deliveryId = conversationDeliveryIdSchema.parse(args.deliveryId); + const nowMs = validateTime(args.nowMs, "nowMs"); + if (!Number.isFinite(args.leaseDurationMs) || args.leaseDurationMs <= 0) { + throw new Error("leaseDurationMs must be positive"); + } + const expiresAtMs = nowMs + args.leaseDurationMs; + const rows = await executor + .db() + .update(juniorPendingDeliveries) + .set({ leaseExpiresAt: new Date(expiresAtMs) }) + .where( + and( + eq(juniorPendingDeliveries.deliveryId, deliveryId), + eq(juniorPendingDeliveries.leaseOwner, args.lease.owner), + eq(juniorPendingDeliveries.leaseVersion, args.lease.version), + gt(juniorPendingDeliveries.leaseExpiresAt, new Date(nowMs)), + ), + ) + .returning({ version: juniorPendingDeliveries.leaseVersion }); + if (!rows[0]) throw new PendingDeliveryLeaseLostError(); + return { owner: args.lease.owner, version: rows[0].version, expiresAtMs }; +} + +/** Release a claim only when its owner and fencing version still match. */ +export async function releasePendingDeliveryLease( + executor: JuniorSqlDatabase, + args: { deliveryId: string; lease: PendingDeliveryLease }, +): Promise { + const rows = await executor + .db() + .update(juniorPendingDeliveries) + .set({ + leaseOwner: null, + leaseExpiresAt: null, + }) + .where( + and( + eq(juniorPendingDeliveries.deliveryId, args.deliveryId), + eq(juniorPendingDeliveries.leaseOwner, args.lease.owner), + eq(juniorPendingDeliveries.leaseVersion, args.lease.version), + ), + ) + .returning({ deliveryId: juniorPendingDeliveries.deliveryId }); + if (!rows[0]) throw new PendingDeliveryLeaseLostError(); +} + +/** Mutate delivery progress only while the caller still owns its durable fence. */ +async function mutateClaimedProgress( + executor: JuniorSqlDatabase, + args: { + deliveryId: string; + lease: PendingDeliveryLease; + nowMs: number; + mutate: ( + progress: PendingConversationDeliveryProgress, + ) => PendingConversationDeliveryProgress; + nextAttemptAtMs?: number; + }, +): Promise { + return executor.transaction(async () => { + const row = await lockedRow(executor, args.deliveryId); + if (!row) throw new PendingDeliveryLeaseLostError(); + requireFence(row, args.lease, args.nowMs); + const current = parseRow(row); + const progress = pendingConversationDeliveryProgressSchema.parse( + args.mutate(current.progress), + ); + if (progress.acceptedPartCount > current.command.parts.length) { + throw new Error("Pending delivery progress exceeds its command"); + } + const rows = await executor + .db() + .update(juniorPendingDeliveries) + .set({ + progress: sanitizePostgresJson(progress), + ...(args.nextAttemptAtMs !== undefined + ? { nextAttemptAt: new Date(args.nextAttemptAtMs) } + : {}), + }) + .where(eq(juniorPendingDeliveries.deliveryId, args.deliveryId)) + .returning(); + if (!rows[0]) throw new PendingDeliveryLeaseLostError(); + return parseRow(rows[0]); + }); +} + +/** Durably mark the current ordered part as posting before the external call. */ +export async function markPendingDeliveryPosting( + executor: JuniorSqlDatabase, + args: { + deliveryId: string; + lease: PendingDeliveryLease; + nowMs: number; + }, +): Promise { + return mutateClaimedProgress(executor, { + ...args, + mutate: (progress) => { + if (progress.currentState.status !== "pending") { + throw new Error( + `Cannot post delivery in ${progress.currentState.status} state`, + ); + } + return { + ...progress, + currentState: { status: "posting", attemptedAtMs: args.nowMs }, + }; + }, + }); +} + +/** + * Return an uncertain part to pending only after reconciliation explicitly + * confirmed absence and its caller-supplied grace period elapsed. + */ +export async function markPendingDeliveryRepostable( + executor: JuniorSqlDatabase, + args: { + deliveryId: string; + lease: PendingDeliveryLease; + nowMs: number; + graceElapsedAtMs: number; + }, +): Promise { + return mutateClaimedProgress(executor, { + ...args, + mutate: (progress) => { + const state = progress.currentState; + if (state.status !== "uncertain") { + throw new Error(`Cannot make delivery repostable from ${state.status}`); + } + if ( + state.confirmedAbsentAtMs === undefined || + args.graceElapsedAtMs > args.nowMs || + args.graceElapsedAtMs < state.confirmedAbsentAtMs + ) { + throw new Error("Repost reconciliation and grace must be complete"); + } + return { ...progress, currentState: { status: "pending" } }; + }, + }); +} + +/** Record provider acceptance and advance to the next ordered part. */ +export async function recordPendingDeliveryAccepted( + executor: JuniorSqlDatabase, + args: { + deliveryId: string; + lease: PendingDeliveryLease; + messageTs: string; + nowMs: number; + }, +): Promise { + return mutateClaimedProgress(executor, { + ...args, + mutate: (progress) => { + const state = progress.currentState; + if (state.status !== "posting" && state.status !== "uncertain") { + throw new Error(`Cannot accept delivery in ${state.status} state`); + } + return { + acceptedPartCount: progress.acceptedPartCount + 1, + acceptedMessageTs: [...progress.acceptedMessageTs, args.messageTs], + currentState: { status: "pending" }, + }; + }, + }); +} + +/** Preserve an ambiguous external result and its pagination/backoff cursor. */ +export async function recordPendingDeliveryUncertain( + executor: JuniorSqlDatabase, + args: { + deliveryId: string; + lease: PendingDeliveryLease; + nowMs: number; + retryAtMs: number; + reconciliationCursor?: string; + confirmedAbsentAtMs?: number; + }, +): Promise { + return mutateClaimedProgress(executor, { + ...args, + nextAttemptAtMs: args.retryAtMs, + mutate: (progress) => { + const state = progress.currentState; + if (state.status !== "posting" && state.status !== "uncertain") { + throw new Error(`Cannot make delivery uncertain from ${state.status}`); + } + return { + ...progress, + currentState: { + status: "uncertain", + attemptedAtMs: state.attemptedAtMs, + ...(args.reconciliationCursor + ? { reconciliationCursor: args.reconciliationCursor } + : {}), + ...(args.confirmedAbsentAtMs !== undefined + ? { confirmedAbsentAtMs: args.confirmedAbsentAtMs } + : state.status === "uncertain" && + state.confirmedAbsentAtMs !== undefined + ? { confirmedAbsentAtMs: state.confirmedAbsentAtMs } + : {}), + }, + }; + }, + }); +} + +/** Return a definitely absent transient provider rejection to pending. */ +export async function recordPendingDeliveryRetryable( + executor: JuniorSqlDatabase, + args: { + deliveryId: string; + lease: PendingDeliveryLease; + nowMs: number; + retryAtMs: number; + }, +): Promise { + return mutateClaimedProgress(executor, { + ...args, + nextAttemptAtMs: args.retryAtMs, + mutate: (progress) => { + const state = progress.currentState; + if (state.status !== "posting") { + throw new Error(`Cannot retry delivery in ${state.status} state`); + } + return { ...progress, currentState: { status: "pending" } }; + }, + }); +} + +/** Record a privacy-safe definitive failure under the active fence. */ +export async function recordPendingDeliveryFailed( + executor: JuniorSqlDatabase, + args: { + deliveryId: string; + lease: PendingDeliveryLease; + failureCode: z.output; + nowMs: number; + }, +): Promise { + const failureCode = conversationDeliveryFailureCodeSchema.parse( + args.failureCode, + ); + return mutateClaimedProgress(executor, { + ...args, + mutate: (progress) => { + const state = progress.currentState; + if (state.status !== "posting" && state.status !== "uncertain") { + throw new Error(`Cannot fail delivery in ${state.status} state`); + } + return { ...progress, currentState: { status: "failed", failureCode } }; + }, + }); +} + +async function canonicalEventByKey( + executor: JuniorSqlDatabase, + conversationId: string, + idempotencyKey: string, +) { + const rows = await executor + .db() + .select({ + type: juniorConversationEvents.type, + payload: juniorConversationEvents.payload, + }) + .from(juniorConversationEvents) + .where( + and( + eq(juniorConversationEvents.conversationId, conversationId), + eq(juniorConversationEvents.idempotencyKey, idempotencyKey), + ), + ); + const row = rows[0]; + return row + ? conversationEventDataSchema.parse({ ...row.payload, type: row.type }) + : undefined; +} + +async function turnTerminalOutcome( + executor: JuniorSqlDatabase, + conversationId: string, + turnId: string, + acceptanceEvidence: PendingDeliveryAcceptanceEvidence, +): Promise { + const fact = await canonicalEventByKey( + executor, + conversationId, + `turn:${turnId}:terminal`, + ); + if (!fact) return undefined; + if (fact.type !== "turn_completed" && fact.type !== "turn_failed") { + throw new Error( + "Turn terminal idempotency key has an unexpected event type", + ); + } + if (fact.turnId !== turnId) { + throw new Error("Turn terminal idempotency key has conflicting data"); + } + if (fact.type === "turn_failed" && fact.failureCode === "delivery_failed") { + return { deliveryOutcome: "failed", modelSucceeded: false }; + } + if (acceptanceEvidence === "visible_assistant") { + const assistantMessageId = buildDeterministicAssistantMessageId(turnId); + const visibleAssistant = await canonicalEventByKey( + executor, + conversationId, + `visible-message:${assistantMessageId}:recorded`, + ); + if (!visibleAssistant) return undefined; + if ( + visibleAssistant.type !== "visible_message_recorded" || + visibleAssistant.messageId !== assistantMessageId || + visibleAssistant.role !== "assistant" + ) { + throw new Error( + "Finalized assistant idempotency key has conflicting data", + ); + } + } + return { + deliveryOutcome: "accepted", + modelSucceeded: fact.type === "turn_completed", + }; +} + +/** Interpret the authoritative turn terminal after an ambiguous SQL commit. */ +export async function loadDeliveryTerminalOutcome( + executor: JuniorSqlDatabase, + args: { + conversationId: string; + turnId: string; + acceptanceEvidence: PendingDeliveryAcceptanceEvidence; + }, +): Promise { + return turnTerminalOutcome( + executor, + args.conversationId, + args.turnId, + args.acceptanceEvidence, + ); +} + +/** + * Run accepted-delivery persistence and delete control state in one transaction. + * A retry after commit resolves the canonical turn terminal instead. + */ +export async function terminalizeAcceptedPendingDelivery( + executor: JuniorSqlDatabase, + args: { + conversationId: string; + deliveryId: string; + turnId: string; + lease: PendingDeliveryLease; + nowMs: number; + finalizer: (input: { + acceptedMessageTs: string[]; + command: PendingConversationDeliveryCommand; + }) => Promise; + }, +): Promise<"finalized" | "already_finalized" | "not_found"> { + return withConversationEventLock(executor, args.conversationId, async () => + executor.transaction(async () => { + const row = await lockedRow(executor, args.deliveryId); + if (!row) { + const terminal = await turnTerminalOutcome( + executor, + args.conversationId, + args.turnId, + "known_outbox_intent", + ); + return terminal?.deliveryOutcome === "accepted" + ? "already_finalized" + : "not_found"; + } + requireFence(row, args.lease, args.nowMs); + const current = parseRow(row); + if ( + current.conversationId !== args.conversationId || + current.turnId !== args.turnId + ) { + throw new Error("Pending delivery belongs to a different conversation"); + } + if (current.progress.acceptedPartCount !== current.command.parts.length) { + throw new Error("Cannot terminalize before every part is accepted"); + } + if ( + await turnTerminalOutcome( + executor, + args.conversationId, + args.turnId, + "known_outbox_intent", + ) + ) { + throw new Error( + "Pending delivery conflicts with an existing turn terminal", + ); + } + await args.finalizer({ + acceptedMessageTs: current.progress.acceptedMessageTs, + command: current.command, + }); + const terminal = await canonicalEventByKey( + executor, + args.conversationId, + `turn:${args.turnId}:terminal`, + ); + const expected = current.command.completion.terminal; + const matchesExpected = + expected.outcome === "success" + ? terminal?.type === "turn_completed" && + terminal.turnId === args.turnId && + terminal.outcome === "success" + : terminal?.type === "turn_failed" && + terminal.turnId === args.turnId && + terminal.failureCode === expected.failureCode && + terminal.eventId === expected.eventId; + if (!matchesExpected) { + throw new Error( + "Accepted delivery finalizer did not write the expected turn terminal", + ); + } + const deleted = await executor + .db() + .delete(juniorPendingDeliveries) + .where( + and( + eq(juniorPendingDeliveries.deliveryId, args.deliveryId), + eq(juniorPendingDeliveries.leaseOwner, args.lease.owner), + eq(juniorPendingDeliveries.leaseVersion, args.lease.version), + ), + ) + .returning({ deliveryId: juniorPendingDeliveries.deliveryId }); + if (!deleted[0]) throw new PendingDeliveryLeaseLostError(); + return "finalized"; + }), + ); +} + +/** + * Run failure persistence and delete control state in one transaction. Only a + * part already marked definitively failed can close. + */ +export async function terminalizeFailedPendingDelivery( + executor: JuniorSqlDatabase, + args: { + conversationId: string; + deliveryId: string; + turnId: string; + lease: PendingDeliveryLease; + nowMs: number; + finalizer: (input: { + acceptedMessageTs: string[]; + command: PendingConversationDeliveryCommand; + failureCode: z.output; + }) => Promise; + }, +): Promise<"finalized" | "already_finalized" | "not_found"> { + return withConversationEventLock(executor, args.conversationId, async () => + executor.transaction(async () => { + const row = await lockedRow(executor, args.deliveryId); + if (!row) { + const terminal = await turnTerminalOutcome( + executor, + args.conversationId, + args.turnId, + "known_outbox_intent", + ); + return terminal?.deliveryOutcome === "failed" + ? "already_finalized" + : "not_found"; + } + requireFence(row, args.lease, args.nowMs); + const current = parseRow(row); + if ( + current.conversationId !== args.conversationId || + current.turnId !== args.turnId + ) { + throw new Error("Pending delivery belongs to a different conversation"); + } + const currentState = current.progress.currentState; + if (currentState.status !== "failed") { + throw new Error( + "Cannot terminalize failure without a definitive failure", + ); + } + const failureCode = currentState.failureCode; + if ( + await turnTerminalOutcome( + executor, + args.conversationId, + args.turnId, + "known_outbox_intent", + ) + ) { + throw new Error( + "Pending delivery conflicts with an existing turn terminal", + ); + } + await args.finalizer({ + acceptedMessageTs: current.progress.acceptedMessageTs, + command: current.command, + failureCode, + }); + const terminal = await canonicalEventByKey( + executor, + args.conversationId, + `turn:${args.turnId}:terminal`, + ); + if ( + terminal?.type !== "turn_failed" || + terminal.turnId !== args.turnId || + terminal.failureCode !== "delivery_failed" + ) { + throw new Error( + "Failed delivery finalizer did not write the expected turn terminal", + ); + } + const deleted = await executor + .db() + .delete(juniorPendingDeliveries) + .where( + and( + eq(juniorPendingDeliveries.deliveryId, args.deliveryId), + eq(juniorPendingDeliveries.leaseOwner, args.lease.owner), + eq(juniorPendingDeliveries.leaseVersion, args.lease.version), + ), + ) + .returning({ deliveryId: juniorPendingDeliveries.deliveryId }); + if (!deleted[0]) throw new PendingDeliveryLeaseLostError(); + return "finalized"; + }), + ); +} diff --git a/packages/junior/src/chat/slack/outbound.ts b/packages/junior/src/chat/slack/outbound.ts index de8b18216..0404d9b48 100644 --- a/packages/junior/src/chat/slack/outbound.ts +++ b/packages/junior/src/chat/slack/outbound.ts @@ -1,3 +1,4 @@ +import { randomBytes } from "node:crypto"; import { SlackActionError } from "@/chat/slack/client"; import type { SlackMessageBlock } from "@/chat/slack/footer"; import { @@ -13,6 +14,79 @@ import { } from "@/chat/slack/timestamp"; const MAX_SLACK_MESSAGE_TEXT_CHARS = 40_000; +const SLACK_DELIVERY_LOCATOR_PATTERN = /^[A-Za-z0-9_-]{22}$/; +const MAX_SLACK_DELIVERY_PART_INDEX = 9_999; +const SLACK_WEB_API_RATE_LIMITED_ERROR_CODE = "slack_webapi_rate_limited_error"; +/** Longest delay before retrying a recoverable Slack provider operation. */ +export const RECOVERABLE_SLACK_MAX_BACKOFF_MS = 60 * 60 * 1_000; +// Slack applies a 15-item conversations.replies limit to some commercially +// distributed apps. One page per invocation also lets durable orchestration +// honor the provider's rate limit between reconciliation attempts. +const SLACK_RECONCILIATION_PAGE_SIZE = 15; + +/** Public Slack metadata event type used to reconcile ambiguous reply writes. */ +export const SLACK_DELIVERY_METADATA_EVENT_TYPE = "junior_delivery"; + +declare const slackDeliveryLocatorBrand: unique symbol; + +/** Opaque, public-safe 128-bit locator used only to correlate Slack delivery. */ +export type SlackDeliveryLocator = string & { + readonly [slackDeliveryLocatorBrand]: true; +}; + +/** Public-safe marker attached to one part of a recoverable Slack delivery. */ +export interface SlackDeliveryMetadata { + locator: SlackDeliveryLocator; + partIndex: number; +} + +/** Create an opaque random locator suitable for public Slack message metadata. */ +export function createSlackDeliveryLocator(): SlackDeliveryLocator { + return randomBytes(16).toString("base64url") as SlackDeliveryLocator; +} + +/** Parse a persisted delivery locator without accepting arbitrary metadata text. */ +export function parseSlackDeliveryLocator( + value: string, +): SlackDeliveryLocator | undefined { + return SLACK_DELIVERY_LOCATOR_PATTERN.test(value) + ? (value as SlackDeliveryLocator) + : undefined; +} + +function requireSlackDeliveryMetadata( + metadata: SlackDeliveryMetadata, +): SlackDeliveryMetadata { + const locator = parseSlackDeliveryLocator(metadata.locator); + if (!locator) { + throw new Error("Slack delivery metadata requires a valid locator"); + } + if ( + !Number.isInteger(metadata.partIndex) || + metadata.partIndex < 0 || + metadata.partIndex > MAX_SLACK_DELIVERY_PART_INDEX + ) { + throw new Error("Slack delivery metadata requires a valid part index"); + } + return { locator, partIndex: metadata.partIndex }; +} + +function toSlackMessageMetadata(metadata: SlackDeliveryMetadata): { + event_type: typeof SLACK_DELIVERY_METADATA_EVENT_TYPE; + event_payload: { + locator: SlackDeliveryLocator; + part_index: number; + }; +} { + const validated = requireSlackDeliveryMetadata(metadata); + return { + event_type: SLACK_DELIVERY_METADATA_EVENT_TYPE, + event_payload: { + locator: validated.locator, + part_index: validated.partIndex, + }, + }; +} function requireSlackConversationId( channelId: string, @@ -144,6 +218,318 @@ export async function postSlackMessage(input: { }; } +export type RecoverableSlackPostResult = + | { outcome: "accepted"; ts: SlackMessageTs } + | { + outcome: "definitive_failure"; + reason: "api_rejected" | "missing_token"; + } + | { + outcome: "retryable_absence"; + reason: "rate_limited"; + retryAtMs?: number; + } + | { + outcome: "uncertain"; + reason: + | "invalid_response" + | "server_error" + | "transport_error" + | "unknown_error"; + }; + +function getErrorStatusCode(error: unknown): number | undefined { + if (!error || typeof error !== "object") return undefined; + const statusCode = (error as { statusCode?: unknown }).statusCode; + return typeof statusCode === "number" ? statusCode : undefined; +} + +function hasExplicitSlackApiRejection(error: unknown): boolean { + if (!error || typeof error !== "object") return false; + const data = (error as { data?: unknown }).data; + return Boolean( + data && + typeof data === "object" && + typeof (data as { error?: unknown }).error === "string", + ); +} + +function classifyRecoverableSlackPostFailure( + error: unknown, +): Exclude { + const statusCode = getErrorStatusCode(error); + if (statusCode !== undefined && statusCode >= 500) { + return { outcome: "uncertain", reason: "server_error" }; + } + if ( + statusCode === 429 || + (error !== null && + typeof error === "object" && + (error as { code?: unknown }).code === + SLACK_WEB_API_RATE_LIMITED_ERROR_CODE) || + (error instanceof SlackActionError && error.code === "rate_limited") + ) { + const candidate = error as { + retryAfter?: unknown; + retryAfterSeconds?: unknown; + headers?: Record; + }; + const rawRetryAfter = + candidate.retryAfter ?? + candidate.retryAfterSeconds ?? + candidate.headers?.["retry-after"] ?? + candidate.headers?.["Retry-After"]; + const retryAfterSeconds = Number(rawRetryAfter); + const retryAfterMs = Number.isFinite(retryAfterSeconds) + ? Math.min( + Math.max(0, retryAfterSeconds * 1_000), + RECOVERABLE_SLACK_MAX_BACKOFF_MS, + ) + : undefined; + return { + outcome: "retryable_absence", + reason: "rate_limited", + ...(retryAfterMs !== undefined + ? { retryAtMs: Date.now() + retryAfterMs } + : {}), + }; + } + if (error instanceof SlackActionError && error.code === "missing_token") { + return { outcome: "definitive_failure", reason: "missing_token" }; + } + if ( + hasExplicitSlackApiRejection(error) || + (error instanceof SlackActionError && error.apiError) + ) { + return { outcome: "definitive_failure", reason: "api_rejected" }; + } + + const candidate = error as { code?: unknown; message?: unknown } | undefined; + if ( + typeof candidate?.code === "string" || + (typeof candidate?.message === "string" && + candidate.message.toLowerCase().includes("socket hang up")) + ) { + return { outcome: "uncertain", reason: "transport_error" }; + } + return { outcome: "uncertain", reason: "unknown_error" }; +} + +/** + * Attempt one recoverable Slack write. Ambiguous writes are never retried; + * callers must persist the uncertain result and reconcile it separately. + */ +export async function postRecoverableSlackMessage(input: { + blocks?: SlackMessageBlock[]; + channelId: string; + metadata: SlackDeliveryMetadata; + text: string; + threadTs?: string; +}): Promise { + const channelId = requireSlackConversationId( + input.channelId, + "Recoverable Slack message posting", + ); + const text = requireSlackMessageText( + input.text, + "Recoverable Slack message posting", + ); + const threadTs = input.threadTs + ? requireSlackThreadTimestamp( + input.threadTs, + "Recoverable Slack thread message posting", + ) + : undefined; + const metadata = toSlackMessageMetadata(input.metadata); + + try { + const response = await getSlackClient().chat.postMessage({ + channel: channelId, + text, + metadata, + ...(input.blocks?.length + ? { + blocks: input.blocks as unknown as Array>, + } + : {}), + ...(threadTs ? { thread_ts: threadTs } : {}), + }); + const ts = parseSlackMessageTs(response.ts); + return ts + ? { outcome: "accepted", ts } + : { outcome: "uncertain", reason: "invalid_response" }; + } catch (error) { + return classifyRecoverableSlackPostFailure(error); + } +} + +export type RecoverableSlackReconciliationResult = + | { outcome: "accepted"; ts: SlackMessageTs } + | { outcome: "confirmed_absent" } + | { outcome: "continue"; nextCursor: string } + | { outcome: "retryable"; retryAtMs?: number } + | { + outcome: "unresolved"; + reason?: "permanent_provider_error"; + providerErrorCode?: string; + }; + +interface SlackReconciliationIdentity { + appId?: string; + botId?: string; + userId?: string; +} + +function readNonEmptyString(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function readSlackApiErrorCode(error: unknown): string | undefined { + if (!error || typeof error !== "object") return undefined; + const candidate = + error instanceof SlackActionError + ? (error.apiError ?? error.code) + : (error as { data?: { error?: unknown } }).data?.error; + return typeof candidate === "string" && /^[a-z0-9_]{1,80}$/i.test(candidate) + ? candidate + : undefined; +} + +function isMessageFromSlackIdentity( + message: Record, + identity: SlackReconciliationIdentity, +): boolean { + const comparisons = [ + [readNonEmptyString(message.app_id), identity.appId], + [readNonEmptyString(message.bot_id), identity.botId], + [readNonEmptyString(message.user), identity.userId], + ] as const; + return comparisons.some( + ([actual, expected]) => actual !== undefined && actual === expected, + ); +} + +function hasSlackDeliveryMetadata( + message: Record, + expected: ReturnType, +): boolean { + const metadata = message.metadata; + if (!metadata || typeof metadata !== "object") return false; + const candidate = metadata as { + event_payload?: unknown; + event_type?: unknown; + }; + if (candidate.event_type !== expected.event_type) return false; + if (!candidate.event_payload || typeof candidate.event_payload !== "object") { + return false; + } + const payload = candidate.event_payload as Record; + return ( + Object.keys(payload).length === 2 && + payload.locator === expected.event_payload.locator && + payload.part_index === expected.event_payload.part_index + ); +} + +/** + * Resolve an uncertain Slack write by finding its exact public-safe marker. + * The caller persists continuation cursors between rate-limited page reads; + * provider failures remain unresolved and never authorize a duplicate repost. + */ +export async function reconcileRecoverableSlackMessage(input: { + channelId: string; + cursor?: string; + metadata: SlackDeliveryMetadata; + oldestTs: string; + threadTs?: string; +}): Promise { + const channelId = requireSlackConversationId( + input.channelId, + "Recoverable Slack message reconciliation", + ); + const threadTs = input.threadTs + ? requireSlackMessageTimestamp( + input.threadTs, + "Recoverable Slack message reconciliation", + ) + : undefined; + const oldestTs = requireSlackMessageTimestamp( + input.oldestTs, + "Recoverable Slack message reconciliation", + ); + const metadata = toSlackMessageMetadata(input.metadata); + + try { + const client = getSlackClient(); + const auth = await client.auth.test(); + const identity: SlackReconciliationIdentity = { + appId: readNonEmptyString(auth.app_id), + botId: readNonEmptyString(auth.bot_id), + userId: readNonEmptyString(auth.user_id), + }; + if (!identity.appId && !identity.botId && !identity.userId) { + return { outcome: "unresolved" }; + } + + const cursor = readNonEmptyString(input.cursor); + const response = threadTs + ? await client.conversations.replies({ + channel: channelId, + ts: threadTs, + oldest: oldestTs, + inclusive: true, + include_all_metadata: true, + limit: SLACK_RECONCILIATION_PAGE_SIZE, + ...(cursor ? { cursor } : {}), + }) + : await client.conversations.history({ + channel: channelId, + oldest: oldestTs, + inclusive: true, + include_all_metadata: true, + limit: SLACK_RECONCILIATION_PAGE_SIZE, + ...(cursor ? { cursor } : {}), + }); + for (const rawMessage of response.messages ?? []) { + const message = rawMessage as unknown as Record; + if ( + !hasSlackDeliveryMetadata(message, metadata) || + !isMessageFromSlackIdentity(message, identity) + ) { + continue; + } + const ts = parseSlackMessageTs(message.ts); + return ts ? { outcome: "accepted", ts } : { outcome: "unresolved" }; + } + + const nextCursor = readNonEmptyString( + response.response_metadata?.next_cursor, + ); + if (nextCursor) return { outcome: "continue", nextCursor }; + return response.has_more === true + ? { outcome: "unresolved" } + : { outcome: "confirmed_absent" }; + } catch (error) { + const classified = classifyRecoverableSlackPostFailure(error); + if (classified.outcome === "retryable_absence") { + return { + outcome: "retryable", + ...(classified.retryAtMs !== undefined + ? { retryAtMs: classified.retryAtMs } + : {}), + }; + } + if (classified.outcome === "definitive_failure") { + return { + outcome: "unresolved", + reason: "permanent_provider_error", + providerErrorCode: readSlackApiErrorCode(error) ?? "unknown", + }; + } + return { outcome: "unresolved" }; + } +} + /** Delete a previously posted Slack message through the shared outbound boundary. */ export async function deleteSlackMessage(input: { channelId: string; diff --git a/packages/junior/src/chat/slack/recoverable-delivery.ts b/packages/junior/src/chat/slack/recoverable-delivery.ts new file mode 100644 index 000000000..d4a9d3416 --- /dev/null +++ b/packages/junior/src/chat/slack/recoverable-delivery.ts @@ -0,0 +1,725 @@ +/** + * Recover finalized Slack replies after process loss without rerunning Pi. + * + * Ambiguous provider writes remain uncertain until Slack reconciliation proves + * acceptance or explicitly authorizes a repost. + */ +import { randomUUID } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; +import type { JuniorSqlDatabase } from "@/db/db"; +import { + claimPendingConversationDelivery, + createPendingConversationDelivery, + loadDeliveryTerminalOutcome, + listDuePendingDeliveries, + loadPendingDeliveryByConversation, + loadPendingDeliveryByTurn, + markPendingDeliveryPosting, + markPendingDeliveryRepostable, + recordPendingDeliveryAccepted, + recordPendingDeliveryFailed, + recordPendingDeliveryRetryable, + recordPendingDeliveryUncertain, + releasePendingDeliveryLease, + renewPendingDeliveryLease, + terminalizeAcceptedPendingDelivery, + terminalizeFailedPendingDelivery, + type PendingConversationDelivery, + type PendingDeliveryAcceptanceEvidence, + type PendingDeliveryTerminalOutcome, +} from "@/chat/slack/delivery-outbox"; +import { + pendingConversationDeliveryCommandSchema, + type PendingConversationDeliveryCommand, + type PendingConversationDeliveryCommandDraft, +} from "@/chat/slack/delivery-command"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; +import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; +import { commitMessages } from "@/chat/conversations/projection"; +import { piMessageSchema } from "@/chat/pi/messages"; +import type { ConversationModelMessage } from "@/chat/conversations/model-message"; +import { + projectConversationEvents, + type PiConversationEventProjection, +} from "@/chat/pi/conversation-events"; +import { withConversationEventLock } from "@/chat/conversations/sql/event-lock"; +import { toStoredConversationMessage } from "@/chat/conversations/visible-message-serializer"; +import { and, eq, inArray } from "drizzle-orm"; +import { juniorConversationMessages } from "@/db/schema"; +import type { + RecoverableSlackPostResult, + RecoverableSlackReconciliationResult, + SlackDeliveryMetadata, +} from "@/chat/slack/outbound"; +import { logError } from "@/chat/logging"; + +const LEASE_DURATION_MS = 120_000; +const RETRY_DELAY_MS = 5_000; +// Recheck operator-recoverable permissions on the same one-hour cadence as +// capped Slack provider retries without coupling this service to Slack code. +const PERMANENT_RECONCILIATION_RETRY_DELAY_MS = 60 * 60 * 1_000; +const REPOST_GRACE_MS = 30_000; +const RECONCILIATION_CLOCK_SKEW_MS = 60_000; + +/** Narrow provider capability used to post or reconcile one durable part. */ +export interface RecoverableSlackDeliveryPort { + post(input: { + blocks?: PendingConversationDeliveryCommand["parts"][number]["blocks"]; + channelId: string; + metadata: SlackDeliveryMetadata; + teamId: string; + text: string; + threadTs?: string; + }): Promise; + reconcile(input: { + channelId: string; + cursor?: string; + metadata: SlackDeliveryMetadata; + oldestTs: string; + teamId: string; + threadTs?: string; + }): Promise; +} + +export type RecoverableSlackDeliveryOutcome = + | { outcome: "accepted"; messageTs?: string } + | { outcome: "failed" } + | { outcome: "pending"; retryAtMs: number }; + +export interface RecoverableSlackDeliveryTerminalizingInput { + command: PendingConversationDeliveryCommand; + conversationId: string; + deliveryOutcome: "accepted" | "failed"; + deliveryId: string; + turnId: string; +} + +export interface RecoverableSlackDeliveryAdvanceOptions { + /** + * Repair non-SQL projections before the durable intent can be deleted. + * The callback must be idempotent because a later SQL failure retries it. + */ + beforeTerminalize?: ( + input: RecoverableSlackDeliveryTerminalizingInput, + ) => Promise; +} + +/** Slack delivery capabilities consumed by the turn executor. */ +export interface RecoverableSlackDelivery { + createIntent(args: { + conversationId: string; + deliveryId: string; + turnId: string; + command: PendingConversationDeliveryCommandDraft; + modelMessages: ConversationModelMessage[]; + }): Promise; + loadByTurn(args: { + conversationId: string; + turnId: string; + }): Promise; + loadByConversation(args: { + conversationId: string; + }): Promise; + listDue(args: { + limit: number; + nowMs: number; + }): Promise; + loadTerminalOutcome(args: { + conversationId: string; + turnId: string; + acceptanceEvidence: PendingDeliveryAcceptanceEvidence; + }): Promise; + advance( + pending: PendingConversationDelivery, + options?: RecoverableSlackDeliveryAdvanceOptions, + ): Promise; +} + +function oldestSlackTimestamp(attemptedAtMs: number): string { + const seconds = Math.max(0, Math.floor(attemptedAtMs / 1_000)); + const micros = Math.max(0, attemptedAtMs % 1_000) * 1_000; + return `${seconds}.${Math.floor(micros).toString().padStart(6, "0")}`; +} + +/** Persist the Slack-visible assistant prefix and replied-input facts. */ +async function recordVisibleAssistantSql( + sql: JuniorSqlDatabase, + conversationId: string, + command: PendingConversationDeliveryCommand, + text: string, + nowMs: number, + messageTs?: string, +): Promise { + const messages = createSqlConversationMessageStore(sql); + const baselines = await sql + .db() + .select({ messageId: juniorConversationMessages.messageId }) + .from(juniorConversationMessages) + .where( + and( + eq(juniorConversationMessages.conversationId, conversationId), + inArray( + juniorConversationMessages.messageId, + command.completion.inputMessageIds, + ), + ), + ); + if (baselines.length !== command.completion.inputMessageIds.length) { + throw new Error("Delivery finalization requires every input baseline"); + } + const assistant = command.completion.assistantMessage; + await messages.record(conversationId, [ + toStoredConversationMessage({ + id: assistant.messageId, + role: "assistant", + text, + author: assistant.author, + createdAtMs: assistant.createdAtMs, + meta: { replied: true, ...(messageTs ? { slackTs: messageTs } : {}) }, + }), + ]); + for (const inputMessageId of command.completion.inputMessageIds) { + await messages.markReplied(conversationId, inputMessageId, nowMs); + } +} + +/** Commit accepted visible facts and the turn terminal. */ +async function finalizeAcceptedSql( + sql: JuniorSqlDatabase, + conversationId: string, + command: PendingConversationDeliveryCommand, + nowMs: number, + messageTs?: string, +): Promise { + await recordVisibleAssistantSql( + sql, + conversationId, + command, + command.completion.assistantMessage.text, + nowMs, + messageTs, + ); + const lifecycle = new ConversationTurnLifecycleService( + createSqlConversationEventStore(sql), + ); + if (command.completion.terminal.outcome === "success") { + await lifecycle.complete({ + conversationId, + turnId: command.completion.turnId, + createdAtMs: nowMs, + outcome: "success", + }); + } else { + await lifecycle.fail({ + conversationId, + turnId: command.completion.turnId, + createdAtMs: nowMs, + failureCode: command.completion.terminal.failureCode, + ...(command.completion.terminal.eventId + ? { eventId: command.completion.terminal.eventId } + : {}), + }); + } +} + +function deliveryCommandDraft( + command: PendingConversationDeliveryCommand, +): PendingConversationDeliveryCommandDraft { + const { + committedSeq: _committedSeq, + rollbackSeq: _rollbackSeq, + ...model + } = command.completion.model; + return { + ...command, + completion: { ...command.completion, model }, + }; +} + +/** Project the exact canonical Pi boundary identified by an event cursor. */ +async function loadSqlProjectionAt( + sql: JuniorSqlDatabase, + conversationId: string, + committedSeq: number, +): Promise { + const epoch = await createSqlConversationEventStore(sql).loadEpochContaining( + conversationId, + committedSeq, + committedSeq, + ); + return epoch ? projectConversationEvents(epoch) : undefined; +} + +/** Remove an undelivered model generation from the live Pi epoch. */ +async function rollbackRejectedModel( + sql: JuniorSqlDatabase, + conversationId: string, + command: PendingConversationDeliveryCommand, +): Promise { + const rollbackSeq = command.completion.model.rollbackSeq; + const prior = + rollbackSeq < 0 + ? { messages: [], provenance: [], modelId: undefined } + : await loadSqlProjectionAt(sql, conversationId, rollbackSeq); + if (!prior) { + throw new Error("Delivery rollback event boundary no longer exists"); + } + const current = projectConversationEvents( + await createSqlConversationEventStore(sql).loadCurrentEpoch(conversationId), + ); + const tailStart = current.seqs.findIndex( + (seq) => seq > command.completion.model.committedSeq, + ); + const tailMessages = tailStart < 0 ? [] : current.messages.slice(tailStart); + const tailProvenance = + tailStart < 0 ? [] : current.provenance.slice(tailStart); + await commitMessages({ + conversationId, + modelId: prior.modelId ?? command.completion.model.modelId, + messages: [...prior.messages, ...tailMessages], + provenance: [...prior.provenance, ...tailProvenance], + executor: sql, + }); +} + +/** Drives one durable Slack reply without ever rerunning the model. */ +export class RecoverableSlackDeliveryService implements RecoverableSlackDelivery { + constructor( + private readonly sql: JuniorSqlDatabase, + private readonly slack: RecoverableSlackDeliveryPort, + private readonly now: () => number = () => Date.now(), + ) {} + + /** Create or validate the immutable intent for a completed model run. */ + async createIntent(args: { + conversationId: string; + deliveryId: string; + turnId: string; + command: PendingConversationDeliveryCommandDraft; + modelMessages: ConversationModelMessage[]; + }): Promise { + const modelMessages = piMessageSchema.array().parse(args.modelMessages); + return withConversationEventLock(this.sql, args.conversationId, async () => + this.sql.transaction(async () => { + const existing = await loadPendingDeliveryByTurn(this.sql, args); + if (existing) { + const committed = await loadSqlProjectionAt( + this.sql, + args.conversationId, + existing.command.completion.model.committedSeq, + ); + if ( + !isDeepStrictEqual( + deliveryCommandDraft(existing.command), + args.command, + ) || + !isDeepStrictEqual(committed?.messages ?? [], modelMessages) + ) { + throw new Error( + "Pending delivery command does not match its intent", + ); + } + return existing; + } + + const eventStore = createSqlConversationEventStore(this.sql); + const before = projectConversationEvents( + await eventStore.loadCurrentEpoch(args.conversationId), + ); + const commit = await commitMessages({ + conversationId: args.conversationId, + modelId: args.command.completion.model.modelId, + messages: modelMessages, + executor: this.sql, + }); + const command = pendingConversationDeliveryCommandSchema.parse({ + ...args.command, + completion: { + ...args.command.completion, + model: { + ...args.command.completion.model, + committedSeq: commit.committedSeq, + rollbackSeq: before.seqs.at(-1) ?? -1, + }, + }, + }); + return createPendingConversationDelivery(this.sql, { + conversationId: args.conversationId, + deliveryId: args.deliveryId, + turnId: args.turnId, + command, + nowMs: this.now(), + }); + }), + ); + } + + /** Load unresolved control state before deciding whether Pi may run. */ + async loadByTurn(args: { + conversationId: string; + turnId: string; + }): Promise { + return loadPendingDeliveryByTurn(this.sql, args); + } + + /** Load the conversation's only unresolved delivery. */ + async loadByConversation(args: { + conversationId: string; + }): Promise { + return loadPendingDeliveryByConversation(this.sql, args); + } + + /** List due intents without claiming them. */ + async listDue(args: { + limit: number; + nowMs: number; + }): Promise { + return listDuePendingDeliveries(this.sql, args); + } + + /** Resolve the canonical turn terminal after an ambiguous outbox commit. */ + async loadTerminalOutcome(args: { + conversationId: string; + turnId: string; + acceptanceEvidence: PendingDeliveryAcceptanceEvidence; + }): Promise { + return loadDeliveryTerminalOutcome(this.sql, args); + } + + /** Claim and advance one pending delivery until it must defer or terminalizes. */ + async advance( + pending: PendingConversationDelivery, + options: RecoverableSlackDeliveryAdvanceOptions = {}, + ): Promise { + const nowMs = this.now(); + const claimed = await claimPendingConversationDelivery(this.sql, { + deliveryId: pending.deliveryId, + leaseOwner: randomUUID(), + nowMs, + leaseDurationMs: LEASE_DURATION_MS, + }); + if (!claimed?.lease) { + const latest = await loadPendingDeliveryByTurn(this.sql, { + conversationId: pending.conversationId, + turnId: pending.turnId, + }); + if (!latest) { + const terminal = await loadDeliveryTerminalOutcome(this.sql, { + conversationId: pending.conversationId, + turnId: pending.turnId, + acceptanceEvidence: "known_outbox_intent", + }); + if (terminal) return { outcome: terminal.deliveryOutcome }; + return { outcome: "pending", retryAtMs: pending.nextAttemptAtMs }; + } + return { + outcome: "pending", + retryAtMs: Math.max( + latest.nextAttemptAtMs, + latest.lease?.expiresAtMs ?? 0, + ), + }; + } + let lease = claimed.lease; + let current = claimed; + try { + while (current.nextPartIndex < current.command.parts.length) { + const partIndex = current.nextPartIndex; + const part = current.command.parts[partIndex]!; + const state = current.progress.currentState; + const metadata: SlackDeliveryMetadata = { + locator: current.command + .publicLocator as SlackDeliveryMetadata["locator"], + partIndex, + }; + if (state.status === "failed") break; + if (state.status === "uncertain") { + lease = await renewPendingDeliveryLease(this.sql, { + deliveryId: current.deliveryId, + lease, + nowMs: this.now(), + leaseDurationMs: LEASE_DURATION_MS, + }); + const reconciliation = await this.slack.reconcile({ + channelId: current.command.route.channelId, + threadTs: current.command.route.threadTs, + metadata, + oldestTs: oldestSlackTimestamp( + Math.max(0, state.attemptedAtMs - RECONCILIATION_CLOCK_SKEW_MS), + ), + teamId: current.command.session.destination.teamId, + ...(state.reconciliationCursor + ? { cursor: state.reconciliationCursor } + : {}), + }); + const reconciliationNow = this.now(); + if (reconciliation.outcome === "accepted") { + current = await recordPendingDeliveryAccepted(this.sql, { + deliveryId: current.deliveryId, + lease, + messageTs: reconciliation.ts, + nowMs: reconciliationNow, + }); + continue; + } + if (reconciliation.outcome === "retryable") { + const retryAtMs = + reconciliation.retryAtMs ?? reconciliationNow + RETRY_DELAY_MS; + await recordPendingDeliveryUncertain(this.sql, { + deliveryId: current.deliveryId, + lease, + nowMs: reconciliationNow, + retryAtMs, + ...(state.reconciliationCursor + ? { reconciliationCursor: state.reconciliationCursor } + : {}), + ...(state.confirmedAbsentAtMs !== undefined + ? { confirmedAbsentAtMs: state.confirmedAbsentAtMs } + : {}), + }); + return { outcome: "pending", retryAtMs }; + } + if (reconciliation.outcome === "confirmed_absent") { + const confirmedAbsentAtMs = + state.confirmedAbsentAtMs ?? reconciliationNow; + const graceElapsedAtMs = confirmedAbsentAtMs + REPOST_GRACE_MS; + if ( + state.confirmedAbsentAtMs !== undefined && + reconciliationNow >= graceElapsedAtMs + ) { + current = await markPendingDeliveryRepostable(this.sql, { + deliveryId: current.deliveryId, + lease, + nowMs: reconciliationNow, + graceElapsedAtMs, + }); + continue; + } + await recordPendingDeliveryUncertain(this.sql, { + deliveryId: current.deliveryId, + lease, + nowMs: reconciliationNow, + retryAtMs: graceElapsedAtMs, + confirmedAbsentAtMs, + }); + return { outcome: "pending", retryAtMs: graceElapsedAtMs }; + } + const retryDelayMs = + reconciliation.outcome === "unresolved" && + reconciliation.reason === "permanent_provider_error" + ? PERMANENT_RECONCILIATION_RETRY_DELAY_MS + : RETRY_DELAY_MS; + const retryAtMs = reconciliationNow + retryDelayMs; + if ( + reconciliation.outcome === "unresolved" && + reconciliation.reason === "permanent_provider_error" + ) { + logError( + "slack_delivery_reconciliation_blocked", + { + conversationId: current.conversationId, + platform: "slack", + slackChannelId: current.command.route.channelId, + slackThreadId: current.command.route.threadTs, + }, + { + "app.delivery.id": current.deliveryId, + "app.turn.id": current.turnId, + "app.provider.error_code": + reconciliation.providerErrorCode ?? "unknown", + "app.retry.delay_ms": retryDelayMs, + }, + "Slack delivery reconciliation is blocked by a permanent provider error; restore Slack access before the next retry", + ); + } + await recordPendingDeliveryUncertain(this.sql, { + deliveryId: current.deliveryId, + lease, + nowMs: reconciliationNow, + retryAtMs, + ...(reconciliation.outcome === "continue" + ? { reconciliationCursor: reconciliation.nextCursor } + : state.reconciliationCursor + ? { reconciliationCursor: state.reconciliationCursor } + : {}), + ...(state.confirmedAbsentAtMs !== undefined + ? { confirmedAbsentAtMs: state.confirmedAbsentAtMs } + : {}), + }); + return { + outcome: "pending", + retryAtMs, + }; + } + + current = await markPendingDeliveryPosting(this.sql, { + deliveryId: current.deliveryId, + lease, + nowMs: this.now(), + }); + lease = await renewPendingDeliveryLease(this.sql, { + deliveryId: current.deliveryId, + lease, + nowMs: this.now(), + leaseDurationMs: LEASE_DURATION_MS, + }); + const post = await this.slack.post({ + channelId: current.command.route.channelId, + threadTs: current.command.route.threadTs, + text: part.text, + teamId: current.command.session.destination.teamId, + ...(part.blocks ? { blocks: part.blocks } : {}), + metadata, + }); + const postNow = this.now(); + if (post.outcome === "accepted") { + current = await recordPendingDeliveryAccepted(this.sql, { + deliveryId: current.deliveryId, + lease, + messageTs: post.ts, + nowMs: postNow, + }); + continue; + } + if (post.outcome === "definitive_failure") { + current = await recordPendingDeliveryFailed(this.sql, { + deliveryId: current.deliveryId, + lease, + failureCode: "provider_rejected", + nowMs: postNow, + }); + break; + } + if (post.outcome === "retryable_absence") { + current = await recordPendingDeliveryRetryable(this.sql, { + deliveryId: current.deliveryId, + lease, + nowMs: postNow, + retryAtMs: post.retryAtMs ?? postNow + RETRY_DELAY_MS, + }); + return { + outcome: "pending", + retryAtMs: post.retryAtMs ?? postNow + RETRY_DELAY_MS, + }; + } + await recordPendingDeliveryUncertain(this.sql, { + deliveryId: current.deliveryId, + lease, + nowMs: postNow, + retryAtMs: postNow + RETRY_DELAY_MS, + }); + return { outcome: "pending", retryAtMs: postNow + RETRY_DELAY_MS }; + } + + const failed = current.progress.currentState.status === "failed"; + if (failed) { + try { + await options.beforeTerminalize?.({ + command: current.command, + conversationId: current.conversationId, + deliveryOutcome: "failed", + deliveryId: current.deliveryId, + turnId: current.turnId, + }); + await terminalizeFailedPendingDelivery(this.sql, { + conversationId: current.conversationId, + deliveryId: current.deliveryId, + turnId: current.turnId, + lease, + nowMs: this.now(), + finalizer: async ({ acceptedMessageTs, command }) => { + if (current.progress.acceptedPartCount === 0) { + await rollbackRejectedModel( + this.sql, + current.conversationId, + command, + ); + } else { + // Preserve the exact Slack-visible boundary. The full Pi + // transcript remains canonical because its tool calls and + // side effects happened even though the reply tail did not. + await recordVisibleAssistantSql( + this.sql, + current.conversationId, + command, + command.parts + .slice(0, current.progress.acceptedPartCount) + .map((part) => part.text) + .join("\n\n"), + this.now(), + acceptedMessageTs.at(-1), + ); + } + await new ConversationTurnLifecycleService( + createSqlConversationEventStore(this.sql), + ).fail({ + conversationId: current.conversationId, + turnId: command.completion.turnId, + createdAtMs: this.now(), + failureCode: "delivery_failed", + }); + }, + }); + } catch (error) { + const terminal = await loadDeliveryTerminalOutcome(this.sql, { + conversationId: current.conversationId, + turnId: current.turnId, + acceptanceEvidence: "known_outbox_intent", + }); + if (terminal) return { outcome: terminal.deliveryOutcome }; + throw error; + } + return { outcome: "failed" }; + } + try { + await options.beforeTerminalize?.({ + command: current.command, + conversationId: current.conversationId, + deliveryOutcome: "accepted", + deliveryId: current.deliveryId, + turnId: current.turnId, + }); + await terminalizeAcceptedPendingDelivery(this.sql, { + conversationId: current.conversationId, + deliveryId: current.deliveryId, + turnId: current.turnId, + lease, + nowMs: this.now(), + finalizer: async ({ acceptedMessageTs, command }) => { + await finalizeAcceptedSql( + this.sql, + current.conversationId, + command, + this.now(), + acceptedMessageTs.at(-1), + ); + }, + }); + } catch (error) { + const terminal = await loadDeliveryTerminalOutcome(this.sql, { + conversationId: current.conversationId, + turnId: current.turnId, + acceptanceEvidence: "known_outbox_intent", + }); + if (terminal) return { outcome: terminal.deliveryOutcome }; + throw error; + } + return { + outcome: "accepted", + messageTs: current.progress.acceptedMessageTs.at(-1), + }; + } finally { + const stillPending = await loadPendingDeliveryByTurn(this.sql, { + conversationId: current.conversationId, + turnId: current.turnId, + }); + if (stillPending?.lease) { + await releasePendingDeliveryLease(this.sql, { + deliveryId: current.deliveryId, + lease, + }).catch(() => undefined); + } + } + } +} diff --git a/packages/junior/src/chat/state/authorization-recovery-index.ts b/packages/junior/src/chat/state/authorization-recovery-index.ts new file mode 100644 index 000000000..c1b122c7e --- /dev/null +++ b/packages/junior/src/chat/state/authorization-recovery-index.ts @@ -0,0 +1,143 @@ +import { THREAD_STATE_TTL_MS, type StateAdapter } from "chat"; +import { z } from "zod"; + +const AUTHORIZATION_RECOVERY_INDEX_KEY = + "junior:agent_turn_authorization_recovery:index"; +const AUTHORIZATION_RECOVERY_INDEX_LOCK_KEY = + "junior:agent_turn_authorization_recovery:index-lock"; +const AUTHORIZATION_RECOVERY_INDEX_LOCK_TTL_MS = 10_000; +/** Maximum unresolved callbacks heartbeat can inspect in one bounded pass. */ +export const AUTHORIZATION_RECOVERY_INDEX_MAX_ENTRIES = 1_000; +/** Time allowed for registration to finish its authoritative session write. */ +export const AUTHORIZATION_RECOVERY_MISSING_RECORD_GRACE_MS = 5 * 60_000; + +const authorizationRecoveryIndexEntrySchema = z + .object({ + authorizationCompletionId: z.string().min(1), + conversationId: z.string().min(1), + registeredAtMs: z.number().finite().nonnegative(), + sessionId: z.string().min(1), + }) + .strict(); + +/** One unresolved authorization callback recovery discoverable by heartbeat. */ +export type AuthorizationRecoveryIndexEntry = z.output< + typeof authorizationRecoveryIndexEntrySchema +>; + +function parseEntries(value: unknown): AuthorizationRecoveryIndexEntry[] { + if (value === null || value === undefined) return []; + if (!Array.isArray(value)) { + throw new Error("Authorization recovery index is malformed"); + } + if (value.length > AUTHORIZATION_RECOVERY_INDEX_MAX_ENTRIES) { + throw new Error("Authorization recovery index exceeds capacity"); + } + const entries: AuthorizationRecoveryIndexEntry[] = []; + for (let index = 0; index < value.length; index += 1) { + const parsed = authorizationRecoveryIndexEntrySchema.safeParse( + value[index], + ); + if (!parsed.success) { + throw new Error("Authorization recovery index is malformed"); + } + entries.push(parsed.data); + } + return entries; +} + +function entryKey(entry: AuthorizationRecoveryIndexEntry): string { + return `${entry.conversationId}\0${entry.sessionId}\0${entry.authorizationCompletionId}`; +} + +function retainedEntries( + entries: AuthorizationRecoveryIndexEntry[], + nowMs: number, +): AuthorizationRecoveryIndexEntry[] { + const oldestRetainedAtMs = nowMs - THREAD_STATE_TTL_MS; + return entries.filter((entry) => entry.registeredAtMs >= oldestRetainedAtMs); +} + +async function mutateIndex( + state: StateAdapter, + nowMs: number, + mutate: ( + entries: AuthorizationRecoveryIndexEntry[], + ) => AuthorizationRecoveryIndexEntry[], +): Promise { + await state.connect(); + const lock = await state.acquireLock( + AUTHORIZATION_RECOVERY_INDEX_LOCK_KEY, + AUTHORIZATION_RECOVERY_INDEX_LOCK_TTL_MS, + ); + if (!lock) { + throw new Error("Authorization recovery index is busy"); + } + try { + const current = retainedEntries( + parseEntries(await state.get(AUTHORIZATION_RECOVERY_INDEX_KEY)), + nowMs, + ); + const next = mutate(current); + if (next.length === 0) { + await state.delete(AUTHORIZATION_RECOVERY_INDEX_KEY); + return; + } + await state.set( + AUTHORIZATION_RECOVERY_INDEX_KEY, + next, + THREAD_STATE_TTL_MS, + ); + } finally { + await state.releaseLock(lock); + } +} + +/** Register an auth recovery before its one-time provider code is consumed. */ +export async function registerAuthorizationRecovery( + state: StateAdapter, + entry: AuthorizationRecoveryIndexEntry, +): Promise { + const parsed = authorizationRecoveryIndexEntrySchema.parse(entry); + await mutateIndex(state, Date.now(), (entries) => { + const key = entryKey(parsed); + const existingIndex = entries.findIndex( + (candidate) => entryKey(candidate) === key, + ); + if (existingIndex >= 0) { + const next = [...entries]; + next[existingIndex] = parsed; + return next; + } + if (entries.length >= AUTHORIZATION_RECOVERY_INDEX_MAX_ENTRIES) { + throw new Error("Authorization recovery index is at capacity"); + } + return [...entries, parsed]; + }); +} + +/** Remove one exact recovery without deleting a newer callback attempt. */ +export async function removeAuthorizationRecovery( + state: StateAdapter, + entry: Pick< + AuthorizationRecoveryIndexEntry, + "authorizationCompletionId" | "conversationId" | "sessionId" + >, +): Promise { + await mutateIndex(state, Date.now(), (entries) => { + const key = entryKey({ ...entry, registeredAtMs: 0 }); + return entries.filter((candidate) => entryKey(candidate) !== key); + }); +} + +/** List retained recoveries; per-session records remain authoritative. */ +export async function listAuthorizationRecoveries( + state: StateAdapter, + nowMs = Date.now(), +): Promise { + await state.connect(); + return retainedEntries( + parseEntries(await state.get(AUTHORIZATION_RECOVERY_INDEX_KEY)), + nowMs, + ); +} diff --git a/packages/junior/src/chat/state/turn-session.ts b/packages/junior/src/chat/state/turn-session.ts index 08ff78589..abc649708 100644 --- a/packages/junior/src/chat/state/turn-session.ts +++ b/packages/junior/src/chat/state/turn-session.ts @@ -7,6 +7,7 @@ * `junior_conversation_events` so resumes can materialize the exact continuable * boundary without duplicating the event history. */ +import { createHash } from "node:crypto"; import { THREAD_STATE_TTL_MS, type StateAdapter } from "chat"; import { actorSchema, @@ -36,6 +37,10 @@ import type { ConversationExecution, ConversationStore, } from "@/chat/conversations/store"; +import { + registerAuthorizationRecovery, + removeAuthorizationRecovery, +} from "./authorization-recovery-index"; const AGENT_TURN_SESSION_PREFIX = "junior:agent_turn_session"; const AGENT_TURN_SESSION_INDEX_KEY = `${AGENT_TURN_SESSION_PREFIX}:index`; @@ -43,6 +48,17 @@ const AGENT_TURN_SESSION_INDEX_MAX_LENGTH = 5_000; const AGENT_TURN_SESSION_INDEX_READ_CONCURRENCY = 25; const AGENT_TURN_SESSION_TTL_MS = THREAD_STATE_TTL_MS; +/** Derive one opaque receipt ID shared by duplicate callbacks for an auth attempt. */ +export function createAgentTurnAuthorizationCompletionId(args: { + attemptId: string; + authorizationKind: "mcp" | "plugin"; + provider: string; +}): string { + return createHash("sha256") + .update(`${args.authorizationKind}:${args.provider}:${args.attemptId}`) + .digest("hex"); +} + export type AgentTurnSessionStatus = | "running" | "awaiting_resume" @@ -54,12 +70,33 @@ export type AgentTurnSurface = "slack" | "api" | "scheduler" | "internal"; export type AgentTurnResumeReason = "timeout" | "auth" | "yield"; +export interface AgentTurnAuthorizationRecovery { + active: boolean; + authorizationCompletionId: string; + authorizationKind: "mcp" | "plugin"; + preparedAtMs: number; + provider: string; + userId: string; +} + +const agentTurnAuthorizationRecoverySchema = z + .object({ + active: z.boolean(), + authorizationCompletionId: z.string().min(1), + authorizationKind: z.enum(["mcp", "plugin"]), + preparedAtMs: z.number().finite().nonnegative(), + provider: z.string().min(1), + userId: z.string().min(1), + }) + .strict() satisfies z.ZodType; + interface ConversationMessageProjection { messages: PiMessage[]; provenance: ConversationMessageProvenance[]; } export interface AgentTurnSessionRecord { + authorizationRecovery?: AgentTurnAuthorizationRecovery; channelName?: string; version: number; conversationId: string; @@ -147,6 +184,7 @@ const nonNegativeNumberSchema = z.number().finite().nonnegative(); const seqCursorSchema = z.number().int().min(-1); const agentTurnSessionSummarySchema = z .object({ + authorizationRecovery: agentTurnAuthorizationRecoverySchema.optional(), channelName: z.string().min(1).optional(), version: z.number().int().nonnegative(), conversationId: z.string().min(1), @@ -251,6 +289,20 @@ async function appendAgentTurnSessionSummary( ]); } +async function appendStoredAgentTurnSessionSummary( + record: StoredAgentTurnSessionRecord, + ttlMs: number, +): Promise { + const { + actors: _actors, + committedSeq: _committedSeq, + errorMessage: _errorMessage, + turnStartSeq: _turnStartSeq, + ...summary + } = record; + await appendAgentTurnSessionSummary(summary, ttlMs); +} + /** Store run summary metadata in the configured conversation store. */ async function recordConversationActivityMetadata(args: { conversationStore?: ConversationStore; @@ -301,6 +353,9 @@ function materializeAgentTurnSessionRecord( ): AgentTurnSessionRecord { return { version: stored.version, + ...(stored.authorizationRecovery + ? { authorizationRecovery: stored.authorizationRecovery } + : {}), ...(stored.channelName ? { channelName: stored.channelName } : {}), conversationId: stored.conversationId, sessionId: stored.sessionId, @@ -448,6 +503,7 @@ function buildStoredRecord(args: { } async function setStoredRecord(args: { + authorizationRecoveryCompletionIdToRemove?: string; conversationStore?: ConversationStore; /** Source-confirmed destination visibility from the current event's signal. */ destinationVisibility?: ConversationPrivacy; @@ -471,14 +527,25 @@ async function setStoredRecord(args: { args.record, args.ttlMs, ); - const { - actors: _actors, - committedSeq: _committedSeq, - errorMessage: _errorMessage, - turnStartSeq: _turnStartSeq, - ...summary - } = args.record; - await appendAgentTurnSessionSummary(summary, args.ttlMs); + await appendStoredAgentTurnSessionSummary(args.record, args.ttlMs); + if (args.authorizationRecoveryCompletionIdToRemove) { + await removeAuthorizationRecovery(stateAdapter, { + authorizationCompletionId: args.authorizationRecoveryCompletionIdToRemove, + conversationId: args.record.conversationId, + sessionId: args.record.sessionId, + }).catch((error) => { + logWarn( + "agent_turn_authorization_recovery_cleanup_failed", + { conversationId: args.record.conversationId }, + { + "app.ai.session_id": args.record.sessionId, + "exception.message": + error instanceof Error ? error.message : String(error), + }, + "Failed to remove terminal authorization recovery index entry", + ); + }); + } return materializeAgentTurnSessionRecord( args.record, { @@ -507,6 +574,12 @@ async function updateAgentTurnSessionState(args: { } return await setStoredRecord({ + ...(parsed.authorizationRecovery + ? { + authorizationRecoveryCompletionIdToRemove: + parsed.authorizationRecovery.authorizationCompletionId, + } + : {}), piMessages: args.existing.piMessages, piMessageProvenance: args.existing.piMessageProvenance, ttlMs: AGENT_TURN_SESSION_TTL_MS, @@ -625,6 +698,15 @@ export async function upsertAgentTurnSessionRecord(args: { : commit.messageSeqs.filter((seq) => seq <= turnStartSeq).length); return await setStoredRecord({ + ...((args.state === "completed" || + args.state === "failed" || + args.state === "abandoned") && + existingRecord?.authorizationRecovery + ? { + authorizationRecoveryCompletionIdToRemove: + existingRecord.authorizationRecovery.authorizationCompletionId, + } + : {}), conversationStore: args.conversationStore, destinationVisibility: args.destinationVisibility, piMessages: args.piMessages, @@ -909,6 +991,7 @@ export async function listAgentTurnSessionSummariesForConversations( /** Mark an unfinished turn session record as abandoned when a newer turn wins. */ export async function abandonAgentTurnSessionRecord(args: { conversationId: string; + expectedVersion?: number; sessionId: string; errorMessage?: string; }): Promise { @@ -920,7 +1003,9 @@ export async function abandonAgentTurnSessionRecord(args: { !existing || existing.state === "completed" || existing.state === "failed" || - existing.state === "abandoned" + existing.state === "abandoned" || + (args.expectedVersion !== undefined && + existing.version !== args.expectedVersion) ) { return undefined; } @@ -959,3 +1044,148 @@ export async function failAgentTurnSessionRecord(args: { errorMessage: args.errorMessage ?? existing.errorMessage, }); } + +/** Persist an inert recovery intent before exchanging a one-time auth code. */ +export async function prepareAgentTurnAuthorizationRecovery(args: { + authorizationCompletionId: string; + authorizationKind: "mcp" | "plugin"; + conversationId: string; + expectedVersion: number; + provider: string; + sessionId: string; + userId: string; +}): Promise { + const existing = await getAgentTurnSessionRecord( + args.conversationId, + args.sessionId, + ); + if ( + !existing || + existing.state !== "awaiting_resume" || + existing.resumeReason !== "auth" + ) { + return undefined; + } + const current = existing.authorizationRecovery; + if (current) { + if ( + current.authorizationCompletionId !== args.authorizationCompletionId || + current.authorizationKind !== args.authorizationKind || + current.provider !== args.provider || + current.userId !== args.userId + ) { + throw new Error("Authorization recovery intent does not match callback"); + } + const stored = await getStoredAgentTurnSessionRecord( + args.conversationId, + args.sessionId, + ); + if (!stored || stored.version !== existing.version) return undefined; + await registerAuthorizationRecovery(getStateAdapter(), { + authorizationCompletionId: current.authorizationCompletionId, + conversationId: args.conversationId, + registeredAtMs: Date.now(), + sessionId: args.sessionId, + }); + await appendStoredAgentTurnSessionSummary( + stored, + AGENT_TURN_SESSION_TTL_MS, + ); + return existing; + } + if (existing.version !== args.expectedVersion) return undefined; + const stored = await getStoredAgentTurnSessionRecord( + args.conversationId, + args.sessionId, + ); + if (!stored || stored.version !== existing.version) return undefined; + const preparedAtMs = Date.now(); + await registerAuthorizationRecovery(getStateAdapter(), { + authorizationCompletionId: args.authorizationCompletionId, + conversationId: args.conversationId, + registeredAtMs: preparedAtMs, + sessionId: args.sessionId, + }); + return await setStoredRecord({ + piMessages: existing.piMessages, + piMessageProvenance: existing.piMessageProvenance, + ttlMs: AGENT_TURN_SESSION_TTL_MS, + ...(existing.turnStartMessageIndex !== undefined + ? { turnStartMessageIndex: existing.turnStartMessageIndex } + : {}), + record: { + ...stored, + authorizationRecovery: { + active: false, + authorizationCompletionId: args.authorizationCompletionId, + authorizationKind: args.authorizationKind, + preparedAtMs, + provider: args.provider, + userId: args.userId, + }, + updatedAtMs: Date.now(), + version: stored.version + 1, + }, + }); +} + +/** Activate a prepared intent only after its credential receipt is committed. */ +export async function activateAgentTurnAuthorizationRecovery(args: { + authorizationCompletionId: string; + conversationId: string; + expectedVersion: number; + sessionId: string; +}): Promise { + const existing = await getAgentTurnSessionRecord( + args.conversationId, + args.sessionId, + ); + if ( + !existing || + existing.state !== "awaiting_resume" || + existing.resumeReason !== "auth" || + existing.version !== args.expectedVersion || + existing.authorizationRecovery?.authorizationCompletionId !== + args.authorizationCompletionId + ) { + return undefined; + } + const stored = await getStoredAgentTurnSessionRecord( + args.conversationId, + args.sessionId, + ); + if (!stored || stored.version !== existing.version) return undefined; + return await setStoredRecord({ + piMessages: existing.piMessages, + piMessageProvenance: existing.piMessageProvenance, + ttlMs: AGENT_TURN_SESSION_TTL_MS, + ...(existing.turnStartMessageIndex !== undefined + ? { turnStartMessageIndex: existing.turnStartMessageIndex } + : {}), + record: { + ...stored, + authorizationRecovery: { + ...existing.authorizationRecovery, + active: true, + }, + updatedAtMs: Date.now(), + version: stored.version + 1, + }, + }); +} + +/** Check whether an exact auth-paused session version has a completed callback. */ +export async function isAgentTurnAuthorizationRecoveryActive(args: { + conversationId: string; + expectedVersion: number; + sessionId: string; +}): Promise { + const session = await getAgentTurnSessionRecord( + args.conversationId, + args.sessionId, + ); + return ( + session?.version === args.expectedVersion && + session.authorizationRecovery?.active === true + ); +} diff --git a/packages/junior/src/chat/task-execution/slack-work.ts b/packages/junior/src/chat/task-execution/slack-work.ts index 614784c52..d2da6203b 100644 --- a/packages/junior/src/chat/task-execution/slack-work.ts +++ b/packages/junior/src/chat/task-execution/slack-work.ts @@ -467,6 +467,9 @@ export function createSlackConversationWorker( threadJson: latestMetadata.thread, }); const skipped = messages.slice(0, -1); + const initialInboundMessageIds = new Set( + records.map((record) => record.inboundMessageId), + ); const messageContext: MessageContext = { skipped, totalSinceLastHandler: messages.length, @@ -485,6 +488,25 @@ export function createSlackConversationWorker( ); } }; + const ackMessageIds = async ( + slackMessageIds: readonly string[], + ): Promise => { + const ids = new Set(slackMessageIds); + try { + await context.attempt.drain(async (pendingRecords) => + pendingRecords.flatMap((record) => { + const metadata = record.input.metadata; + return isSlackMetadata(metadata) && ids.has(metadata.message.id) + ? [record.inboundMessageId] + : []; + }), + ); + } catch { + throw new TurnInputCommitLostError( + `Conversation work lease lost before partial Slack inbox ack for ${context.conversationId}`, + ); + } + }; // Restore stored mailbox entries as Slack steering candidates; the // runtime returns only the inbound ids it handled durably. const drainSteeringMessages = async ( @@ -493,7 +515,18 @@ export function createSlackConversationWorker( ) => Promise, ): Promise => { await context.attempt.drain(async (pendingRecords) => { - const messages = pendingRecords.map((record) => { + // The initial attempt remains intentionally unacked until delivery + // terminalization. It is already represented by `messageContext` + // and must never be redrained as mid-run steering or accidentally + // consumed by an undefined (ack-all) drain result. + const steeringRecords = pendingRecords.filter( + (record) => + !initialInboundMessageIds.has(record.inboundMessageId), + ); + if (steeringRecords.length === 0) { + return []; + } + const messages = steeringRecords.map((record) => { const metadata = record.input.metadata; if (!isSlackMetadata(metadata)) { throw new Error( @@ -509,7 +542,7 @@ export function createSlackConversationWorker( message, }; }); - return await accept(messages); + return (await accept(messages)) ?? []; }); }; @@ -520,6 +553,7 @@ export function createSlackConversationWorker( messageContext, drainSteeringMessages, ack, + ackMessageIds, isFinalAttempt: context.attempt.isFinalAttempt, shouldYield: context.shouldYield, }); @@ -531,12 +565,19 @@ export function createSlackConversationWorker( messageContext, drainSteeringMessages, ack, + ackMessageIds, isFinalAttempt: context.attempt.isFinalAttempt, shouldYield: context.shouldYield, }); } catch (error) { if (isTurnInputDeferredError(error)) { - return { status: "deferred" } satisfies ConversationWorkerResult; + return { + status: "deferred", + ...(error.retryAfterMs !== undefined + ? { delayMs: error.retryAfterMs } + : {}), + ...(error.immediate ? { immediate: true } : {}), + } satisfies ConversationWorkerResult; } if (isCooperativeTurnYieldError(error)) { return { status: "yielded" } satisfies ConversationWorkerResult; diff --git a/packages/junior/src/chat/task-execution/worker.ts b/packages/junior/src/chat/task-execution/worker.ts index fa95f24ff..29e2d58bd 100644 --- a/packages/junior/src/chat/task-execution/worker.ts +++ b/packages/junior/src/chat/task-execution/worker.ts @@ -55,6 +55,8 @@ export interface InboxAttempt { export interface ConversationWorkerResult { status: "completed" | "deferred" | "lost_lease" | "yielded"; + delayMs?: number; + immediate?: boolean; } export interface ConversationWorkProcessResult { @@ -184,28 +186,34 @@ function startLeaseCheckIn(args: { leaseToken: string; onLostLease: () => void; options: ProcessConversationWorkOptions; -}): ReturnType { - const timer = setInterval(() => { +}): { stop(): Promise } { + let inFlight: Promise | undefined; + let stopped = false; + const checkIn = (): void => { + if (stopped || inFlight) { + return; + } const nowMs = now(args.options); - void checkInConversationWork({ + const renewal = checkInConversationWork({ conversationId: args.conversationId, leaseToken: args.leaseToken, conversationStore: args.options.conversationStore, nowMs, state: args.options.state, - }).then( - (checkedIn) => { - if (!checkedIn) { - args.onLostLease(); - logWarn( - "conversation_work_check_in_failed", - { conversationId: args.conversationId }, - {}, - "Conversation work check-in lost its lease", - ); + }) + .then((checkedIn) => { + if (checkedIn) { + return; } - }, - (error) => { + args.onLostLease(); + logWarn( + "conversation_work_check_in_failed", + { conversationId: args.conversationId }, + {}, + "Conversation work check-in lost its lease", + ); + }) + .catch((error) => { logException( error, "conversation_work_check_in_failed", @@ -213,11 +221,25 @@ function startLeaseCheckIn(args: { {}, "Conversation work check-in failed", ); - }, - ); + }) + .finally(() => { + if (inFlight === renewal) { + inFlight = undefined; + } + }); + inFlight = renewal; + }; + const timer = setInterval(() => { + checkIn(); }, args.options.checkInIntervalMs ?? CONVERSATION_WORK_CHECK_IN_INTERVAL_MS); (timer as { unref?: () => void }).unref?.(); - return timer; + return { + async stop(): Promise { + stopped = true; + clearInterval(timer); + await inFlight; + }, + }; } /** Process one queue wake-up for a conversation. */ @@ -326,7 +348,7 @@ export async function processConversationWork( const markLeaseLost = (): void => { leaseLost = true; }; - const timer = startLeaseCheckIn({ + const leaseCheckIn = startLeaseCheckIn({ conversationId, leaseToken: lease.leaseToken, onLostLease: markLeaseLost, @@ -402,6 +424,7 @@ export async function processConversationWork( try { const result = await options.run(workerContext); + await leaseCheckIn.stop(); if (result.status === "lost_lease") { await requestLostLeaseRecovery({ conversationId, @@ -487,6 +510,9 @@ export async function processConversationWork( deferredNowMs, ), nowMs: deferredNowMs, + delayMs: result.immediate + ? 0 + : Math.max(CONVERSATION_WORK_DEFER_DELAY_MS, result.delayMs ?? 0), queue: options.queue, state: options.state, }); @@ -557,6 +583,7 @@ export async function processConversationWork( ); return { status: "completed" }; } catch (error) { + await leaseCheckIn.stop(); const errorNowMs = now(options); // A failed run must not both NACK the queue delivery and schedule a // recovery nudge. Once durable recovery state is recorded and one nudge is @@ -639,6 +666,6 @@ export async function processConversationWork( } return { status: "failed" }; } finally { - clearInterval(timer); + await leaseCheckIn.stop(); } } diff --git a/packages/junior/src/chat/tools/index.ts b/packages/junior/src/chat/tools/index.ts index d4427c0b9..0d3c89a42 100644 --- a/packages/junior/src/chat/tools/index.ts +++ b/packages/junior/src/chat/tools/index.ts @@ -97,7 +97,9 @@ export function createTools( ? resolveChannelCapabilities(slackContext.sourceChannelId) : undefined; const canSendFilesToActiveConversation = Boolean( - slackContext && slackSourceCapabilities?.canSendMessage, + slackContext && + slackSourceCapabilities?.canSendMessage && + context.conversationId?.startsWith("slack:"), ); const tools: ToolRegistry = { loadSkill: createLoadSkillTool(availableSkills, { @@ -187,7 +189,7 @@ export function createTools( ); } - if (rawChannelCapabilities.canSendMessage) { + if (canSendFilesToActiveConversation) { tools.sendMessage = createSendMessageTool(slackContext, state, (input) => readSandboxFileUpload(context.sandbox, input), ); diff --git a/packages/junior/src/db/schema.ts b/packages/junior/src/db/schema.ts index e284178fb..f6b3d1a2e 100644 --- a/packages/junior/src/db/schema.ts +++ b/packages/junior/src/db/schema.ts @@ -4,6 +4,7 @@ import { juniorConversations } from "./schema/conversations"; import { juniorDestinations } from "./schema/destinations"; import { juniorIdentities } from "./schema/identities"; import { juniorUsers } from "./schema/users"; +import { juniorPendingDeliveries } from "./schema/pending-deliveries"; export { juniorConversationEvents, @@ -12,6 +13,7 @@ export { juniorDestinations, juniorIdentities, juniorUsers, + juniorPendingDeliveries, }; export const juniorSqlSchema = { @@ -21,4 +23,5 @@ export const juniorSqlSchema = { juniorDestinations, juniorIdentities, juniorUsers, + juniorPendingDeliveries, }; diff --git a/packages/junior/src/db/schema/pending-deliveries.ts b/packages/junior/src/db/schema/pending-deliveries.ts new file mode 100644 index 000000000..b8e57689a --- /dev/null +++ b/packages/junior/src/db/schema/pending-deliveries.ts @@ -0,0 +1,52 @@ +import { sql } from "drizzle-orm"; +import { + check, + foreignKey, + index, + integer, + jsonb, + pgTable, + text, + uniqueIndex, +} from "drizzle-orm/pg-core"; +import type { + PendingConversationDeliveryCommand, + PendingConversationDeliveryProgress, +} from "@/chat/slack/delivery-command"; +import { juniorConversations } from "./conversations"; +import { timestamptz } from "./timestamps"; + +/** Mutable, deletable control state for unresolved external deliveries. */ +export const juniorPendingDeliveries = pgTable( + "junior_pending_deliveries", + { + deliveryId: text("delivery_id").primaryKey(), + conversationId: text("conversation_id").notNull(), + turnId: text("turn_id").notNull(), + command: jsonb("command_json") + .$type() + .notNull(), + progress: jsonb("progress_json") + .$type() + .notNull(), + nextAttemptAt: timestamptz("next_attempt_at").notNull(), + leaseOwner: text("lease_owner"), + leaseVersion: integer("lease_version").default(0).notNull(), + leaseExpiresAt: timestamptz("lease_expires_at"), + }, + (table) => [ + foreignKey({ + name: "junior_pending_deliveries_conversation_id_fk", + columns: [table.conversationId], + foreignColumns: [juniorConversations.conversationId], + }).onDelete("cascade"), + uniqueIndex("junior_pending_deliveries_conversation_idx").on( + table.conversationId, + ), + index("junior_pending_deliveries_due_idx").on(table.nextAttemptAt), + check( + "junior_pending_deliveries_lease_version_check", + sql`${table.leaseVersion} >= 0`, + ), + ], +); diff --git a/packages/junior/src/deployment.ts b/packages/junior/src/deployment.ts index 3660f259a..df3149ca5 100644 --- a/packages/junior/src/deployment.ts +++ b/packages/junior/src/deployment.ts @@ -6,6 +6,8 @@ export const JUNIOR_RETENTION_ROUTE = "/api/internal/retention"; export const JUNIOR_RETENTION_CRON_SCHEDULE = "0 4 * * *"; export const JUNIOR_CONVERSATION_WORK_CALLBACK_ROUTE = "/api/internal/agent/continue"; +export const JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE = + "/api/internal/agent-dispatch"; export const JUNIOR_PLUGIN_TASK_CALLBACK_ROUTE = "/api/internal/plugin/tasks"; export const LEGACY_JUNIOR_CONVERSATION_WORK_FUNCTION = "api/internal/agent/continue.ts"; diff --git a/packages/junior/src/function-duration.ts b/packages/junior/src/function-duration.ts new file mode 100644 index 000000000..0b2f71348 --- /dev/null +++ b/packages/junior/src/function-duration.ts @@ -0,0 +1,30 @@ +export const DEFAULT_FUNCTION_MAX_DURATION_SECONDS = 300; + +/** Resolve the numeric host duration shared by build and runtime defaults. */ +export function resolveConfiguredFunctionMaxDurationSeconds( + env: NodeJS.ProcessEnv = process.env, +): number { + const raw = + env.FUNCTION_MAX_DURATION_SECONDS ?? + env.QUEUE_CALLBACK_MAX_DURATION_SECONDS; + const value = Number.parseInt(raw ?? "", 10); + return Number.isNaN(value) || value <= 0 + ? DEFAULT_FUNCTION_MAX_DURATION_SECONDS + : value; +} + +/** Keep dispatch ownership beyond the host window used to execute its slice. */ +export function dispatchSliceLeaseMs( + functionMaxDurationSeconds: number, + bufferSeconds: number, +): number { + return (functionMaxDurationSeconds + bufferSeconds) * 1000; +} + +/** Reserve host time after agent execution for durable turn finalization. */ +export function agentExecutionBudgetMs( + functionMaxDurationSeconds: number, + bufferSeconds: number, +): number { + return Math.max(1, (functionMaxDurationSeconds - bufferSeconds) * 1000); +} diff --git a/packages/junior/src/handlers/agent-dispatch.ts b/packages/junior/src/handlers/agent-dispatch.ts index 407bfbcc8..d3c6d2312 100644 --- a/packages/junior/src/handlers/agent-dispatch.ts +++ b/packages/junior/src/handlers/agent-dispatch.ts @@ -1,11 +1,16 @@ import { logException } from "@/chat/logging"; -import { runAgentDispatchSlice } from "@/chat/agent-dispatch/runner"; +import { processAgentDispatchCallback } from "@/chat/agent-dispatch/runner"; import { verifyDispatchCallbackRequest } from "@/chat/agent-dispatch/signing"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; import type { WaitUntilFn } from "@/handlers/types"; +import type { ConversationTurnLifecycle } from "@/chat/conversations/turn-lifecycle"; +import type { RecoverableSlackDelivery } from "@/chat/slack/recoverable-delivery"; interface AgentDispatchHandlerOptions { agentRunner: AgentRunner; + functionMaxDurationSeconds: number; + recoverableSlackDelivery: RecoverableSlackDelivery; + turnLifecycle: ConversationTurnLifecycle; } /** Handle the authenticated internal agent-dispatch callback. */ @@ -20,8 +25,11 @@ export async function POST( } waitUntil(() => - runAgentDispatchSlice(payload, { + processAgentDispatchCallback(payload, { agentRunner: options.agentRunner, + functionMaxDurationSeconds: options.functionMaxDurationSeconds, + recoverableSlackDelivery: options.recoverableSlackDelivery, + turnLifecycle: options.turnLifecycle, }).catch((error) => { logException( error, diff --git a/packages/junior/src/handlers/heartbeat.ts b/packages/junior/src/handlers/heartbeat.ts index 84850a015..97c7392a6 100644 --- a/packages/junior/src/handlers/heartbeat.ts +++ b/packages/junior/src/handlers/heartbeat.ts @@ -3,9 +3,11 @@ import { runHeartbeat } from "@/chat/agent-dispatch/heartbeat"; import type { ConversationWorkQueue } from "@/chat/task-execution/queue"; import { logException } from "@/chat/logging"; import type { WaitUntilFn } from "@/handlers/types"; +import type { RecoverableSlackDelivery } from "@/chat/slack/recoverable-delivery"; export interface HeartbeatHandlerOptions { conversationWorkQueue?: ConversationWorkQueue; + recoverableSlackDelivery?: RecoverableSlackDelivery; } function getHeartbeatSecret(): string | undefined { @@ -45,6 +47,7 @@ export async function GET( runHeartbeat({ conversationWorkQueue: options.conversationWorkQueue, nowMs, + recoverableSlackDelivery: options.recoverableSlackDelivery, }).catch((error) => { logException( error, diff --git a/packages/junior/src/handlers/mcp-oauth-callback.ts b/packages/junior/src/handlers/mcp-oauth-callback.ts index ae5869120..b7c32ef67 100644 --- a/packages/junior/src/handlers/mcp-oauth-callback.ts +++ b/packages/junior/src/handlers/mcp-oauth-callback.ts @@ -13,6 +13,7 @@ import { hydrateConversationMessages } from "@/chat/conversations/visible-messag import { deleteMcpAuthSession, getMcpAuthSession, + getMcpStoredOAuthCredentials, type McpAuthSessionState, } from "@/chat/mcp/auth-store"; import { finalizeMcpAuthorization } from "@/chat/mcp/oauth"; @@ -25,7 +26,11 @@ import { getPersistedThreadState, persistThreadStateById, } from "@/chat/runtime/thread-state"; -import { buildDeliveredTurnStatePatch } from "@/chat/runtime/delivered-turn-state"; +import { + buildDeliveredTurnStatePatch, + buildRecoveredDeliveredTurnStatePatch, +} from "@/chat/runtime/delivered-turn-state"; +import { recoverSlackDeliveryForTurn } from "@/chat/runtime/slack-delivery-recovery"; import { getTurnUserMessage, getTurnUserReplyAttachmentContext, @@ -38,7 +43,10 @@ import { updateConversationStats, } from "@/chat/services/conversation-memory"; import { coerceThreadArtifactsState } from "@/chat/state/artifacts"; -import { resumeAuthorizedRequest } from "@/chat/runtime/slack-resume"; +import { + ResumeTurnBusyError, + resumeAuthorizedRequest, +} from "@/chat/runtime/slack-resume"; import { persistAuthPauseTurnState } from "@/chat/runtime/auth-pause-state"; import { clearPendingAuth, @@ -46,6 +54,8 @@ import { isPendingAuthLatestRequest, } from "@/chat/services/pending-auth"; import { + activateAgentTurnAuthorizationRecovery, + createAgentTurnAuthorizationCompletionId, failAgentTurnSessionRecord, abandonAgentTurnSessionRecord, getAgentTurnSessionRecord, @@ -55,11 +65,19 @@ import { recordAuthorizationCompleted, } from "@/chat/conversations/projection"; import { markTurnFailed } from "@/chat/runtime/turn"; -import { scheduleAgentContinue } from "@/chat/services/agent-continue"; +import { + activateAndScheduleAgentTurnAuthorizationRecovery, + prepareAgentTurnAuthorizationRecoveryUnderActiveLock, + scheduleAgentContinue, + wakeAuthorizationCompletedAgentTurn, + type ScheduleAgentContinueOptions, +} from "@/chat/services/agent-continue"; import { htmlCallbackResponse } from "@/handlers/oauth-html"; import type { WaitUntilFn } from "@/handlers/types"; import { createSlackResumeActor, isUserActor, type Actor } from "@/chat/actor"; import { requireSlackDestination } from "@/chat/destination"; +import type { ConversationTurnLifecycle } from "@/chat/conversations/turn-lifecycle"; +import type { RecoverableSlackDelivery } from "@/chat/slack/recoverable-delivery"; const CALLBACK_PAGES = { missing_state: { @@ -99,6 +117,9 @@ const CALLBACK_PAGES = { interface McpOAuthCallbackOptions { agentRunner: AgentRunner; + agentContinueOptions?: ScheduleAgentContinueOptions; + recoverableSlackDelivery?: RecoverableSlackDelivery; + turnLifecycle?: ConversationTurnLifecycle; } class McpOAuthAttemptExpiredError extends Error { @@ -124,7 +145,7 @@ async function persistCompletedReplyState( channelId: string, threadTs: string, sessionId: string, - reply: AgentRunResult, + reply?: AgentRunResult, ): Promise { const threadId = `slack:${channelId}:${threadTs}`; const currentState = await getPersistedThreadState(threadId); @@ -132,13 +153,19 @@ async function persistCompletedReplyState( await hydrateConversationMessages({ conversation, conversationId: threadId }); const artifacts = coerceThreadArtifactsState(currentState); const userMessage = getTurnUserMessage(conversation, sessionId); - const statePatch = buildDeliveredTurnStatePatch({ - artifacts, - conversation, - reply, - sessionId, - userMessageId: userMessage?.id, - }); + const statePatch = reply + ? buildDeliveredTurnStatePatch({ + artifacts, + conversation, + reply, + sessionId, + userMessageId: userMessage?.id, + }) + : buildRecoveredDeliveredTurnStatePatch({ + conversation, + sessionId, + inputMessageIds: userMessage ? [userMessage.id] : [], + }); await persistThreadStateById(threadId, { ...statePatch, @@ -208,8 +235,16 @@ async function resumeAuthorizedMcpTurn(args: { authSession: McpAuthSessionState; agentRunner: AgentRunner; provider: string; + turnLifecycle?: ConversationTurnLifecycle; + recoverableSlackDelivery?: RecoverableSlackDelivery; }): Promise { - const { authSession, agentRunner, provider } = args; + const { + authSession, + agentRunner, + provider, + recoverableSlackDelivery, + turnLifecycle, + } = args; if ( !authSession.channelId || !authSession.destination || @@ -226,6 +261,48 @@ async function resumeAuthorizedMcpTurn(args: { const currentState = await getPersistedThreadState(threadId); const conversation = coerceThreadConversationState(currentState); await hydrateConversationMessages({ conversation, conversationId: threadId }); + const recoveredDelivery = await recoverSlackDeliveryForTurn({ + conversationId: authSession.conversationId, + delivery: recoverableSlackDelivery, + turnId: authSession.sessionId, + }); + if (recoveredDelivery) { + if (recoveredDelivery.outcome === "accepted") { + await persistCompletedReplyState( + authSession.channelId, + authSession.threadTs, + authSession.sessionId, + ); + } + return; + } + const sessionRecord = await getAgentTurnSessionRecord( + authSession.conversationId, + authSession.sessionId, + ); + if (sessionRecord?.state === "completed") { + const userMessage = getTurnUserMessage(conversation, authSession.sessionId); + await persistThreadStateById( + threadId, + buildRecoveredDeliveredTurnStatePatch({ + conversation, + sessionId: authSession.sessionId, + inputMessageIds: userMessage ? [userMessage.id] : [], + }), + ); + return; + } + if ( + sessionRecord?.state === "failed" || + sessionRecord?.state === "abandoned" + ) { + clearPendingAuth(conversation, authSession.sessionId); + if (conversation.processing.activeTurnId === authSession.sessionId) { + conversation.processing.activeTurnId = undefined; + } + await persistThreadStateById(threadId, { conversation }); + return; + } const pendingAuth = getConversationPendingAuth({ conversation, kind: "mcp", @@ -258,12 +335,14 @@ async function resumeAuthorizedMcpTurn(args: { threadTs: authSession.threadTs, messageTs: getTurnUserSlackMessageTs(userMessage), lockKey: threadId, + connectedText: "", + agentRunner, + recoverableSlackDelivery, lifecycleCorrelation: { conversationId: authSession.conversationId, turnId: resolvedSessionId, }, - connectedText: "", - agentRunner, + turnLifecycle, beforeStart: async () => { const lockedState = await getPersistedThreadState(threadId); const lockedConversation = coerceThreadConversationState(lockedState); @@ -287,6 +366,9 @@ async function resumeAuthorizedMcpTurn(args: { } if (!isPendingAuthLatestRequest(lockedConversation, lockedPendingAuth)) { clearPendingAuth(lockedConversation, lockedPendingAuth.sessionId); + if (lockedConversation.processing.activeTurnId === lockedSessionId) { + lockedConversation.processing.activeTurnId = undefined; + } await persistThreadStateById(threadId, { conversation: lockedConversation, }); @@ -343,6 +425,13 @@ async function resumeAuthorizedMcpTurn(args: { sessionId: lockedSessionId, errorMessage: "Stored Slack actor identity did not match OAuth actor", }); + clearPendingAuth(lockedConversation, lockedSessionId); + if (lockedConversation.processing.activeTurnId === lockedSessionId) { + lockedConversation.processing.activeTurnId = undefined; + } + await persistThreadStateById(threadId, { + conversation: lockedConversation, + }); return false; } if (!lockedSessionRecord.source) { @@ -352,6 +441,13 @@ async function resumeAuthorizedMcpTurn(args: { sessionId: lockedSessionId, errorMessage: "Stored Slack source missing for MCP OAuth resume", }); + clearPendingAuth(lockedConversation, lockedSessionId); + if (lockedConversation.processing.activeTurnId === lockedSessionId) { + lockedConversation.processing.activeTurnId = undefined; + } + await persistThreadStateById(threadId, { + conversation: lockedConversation, + }); return false; } @@ -371,6 +467,7 @@ async function resumeAuthorizedMcpTurn(args: { messageText: lockedUserMessage.text, messageTs: lockedMessageTs, inputMessageIds: [lockedUserMessage.id], + sliceId: lockedSessionRecord.sliceId, replyContext: { input: { conversationContext: lockedConversationContext, @@ -424,6 +521,13 @@ async function resumeAuthorizedMcpTurn(args: { reply, ); }, + onRecoveredSuccess: async () => { + await persistCompletedReplyState( + authSession.channelId!, + authSession.threadTs!, + lockedSessionId, + ); + }, onPostDeliveryCommitFailure: async () => { await failAgentTurnSessionRecord({ conversationId: authSession.conversationId, @@ -500,30 +604,93 @@ async function isCurrentMcpAuthorizationAttempt( ); } -/** Commit one shared credential mutation only while this attempt owns the thread. */ -async function runCurrentMcpCredentialMutation( - authSession: McpAuthSessionState, - provider: string, - mutation: () => Promise, -): Promise { - if (!authSession.channelId || !authSession.threadTs) { - throw new McpOAuthAttemptExpiredError(); - } - - const threadId = `slack:${authSession.channelId}:${authSession.threadTs}`; +/** Exchange and persist one MCP callback while owning its exact parked turn. */ +async function finalizeOwnedMcpAuthorization(args: { + authorizationCode: string; + pendingSession: McpAuthSessionState; + provider: string; + options: McpOAuthCallbackOptions; + recovery?: { + authorizationCompletionId: string; + expectedVersion: number; + }; +}): Promise { const stateAdapter = getStateAdapter(); await stateAdapter.connect(); - const lock = await acquireActiveLock(stateAdapter, threadId); + const lock = await acquireActiveLock( + stateAdapter, + args.pendingSession.conversationId, + ); if (!lock) { throw new Error( - `Could not acquire MCP OAuth callback lock for ${threadId}`, + `Could not acquire MCP OAuth callback lock for ${args.pendingSession.conversationId}`, ); } try { - if (!(await isCurrentMcpAuthorizationAttempt(authSession, provider))) { + const current = args.recovery + ? await getAgentTurnSessionRecord( + args.pendingSession.conversationId, + args.pendingSession.sessionId, + ) + : undefined; + if ( + !(await isCurrentMcpAuthorizationAttempt( + args.pendingSession, + args.provider, + )) + ) { throw new McpOAuthAttemptExpiredError(); } - return await mutation(); + if ( + args.recovery && + (!current || + current.state !== "awaiting_resume" || + current.resumeReason !== "auth" || + current.version !== args.recovery.expectedVersion || + current.authorizationRecovery?.authorizationCompletionId !== + args.recovery.authorizationCompletionId) + ) { + throw new McpOAuthAttemptExpiredError(); + } + + const authSession = await finalizeMcpAuthorization( + args.provider, + args.pendingSession.authSessionId, + args.authorizationCode, + async (mutation) => { + if ( + !(await isCurrentMcpAuthorizationAttempt( + args.pendingSession, + args.provider, + )) + ) { + throw new McpOAuthAttemptExpiredError(); + } + return await mutation(); + }, + args.recovery?.authorizationCompletionId, + ); + if (args.recovery) { + const activated = await activateAgentTurnAuthorizationRecovery({ + authorizationCompletionId: args.recovery.authorizationCompletionId, + conversationId: args.pendingSession.conversationId, + expectedVersion: args.recovery.expectedVersion, + sessionId: args.pendingSession.sessionId, + }); + if (!activated?.destination) { + throw new Error("MCP turn changed while activating callback recovery"); + } + await scheduleAgentContinue( + { + conversationId: activated.conversationId, + destination: activated.destination, + expectedVersion: activated.version, + sessionId: activated.sessionId, + }, + args.options.agentContinueOptions, + ); + } + return authSession; } finally { await stateAdapter.releaseLock(lock); } @@ -563,17 +730,72 @@ export async function GET( return htmlResponse("expired"); } - const authSession = await finalizeMcpAuthorization( - provider, - state, - code, - async (mutation) => - await runCurrentMcpCredentialMutation( - pendingSession, - provider, - mutation, - ), + const sessionRecord = await getAgentTurnSessionRecord( + pendingSession.conversationId, + pendingSession.sessionId, ); + const recoverableSession = + sessionRecord?.state === "awaiting_resume" && + sessionRecord.resumeReason === "auth" + ? sessionRecord + : undefined; + const prepared = recoverableSession + ? await prepareAgentTurnAuthorizationRecoveryUnderActiveLock({ + authorizationCompletionId: createAgentTurnAuthorizationCompletionId({ + attemptId: state, + authorizationKind: "mcp", + provider, + }), + authorizationKind: "mcp", + conversationId: pendingSession.conversationId, + expectedVersion: recoverableSession.version, + provider, + sessionId: pendingSession.sessionId, + userId: pendingSession.userId, + }) + : undefined; + if (recoverableSession && !prepared) { + throw new Error("MCP turn changed while preparing callback recovery"); + } + const recovery = prepared?.authorizationRecovery; + const receiptCommitted = recovery + ? recovery.active || + (await getMcpStoredOAuthCredentials(recovery.userId, recovery.provider)) + ?.authorizationCompletionId === recovery.authorizationCompletionId + : false; + let authSession: McpAuthSessionState; + if (!receiptCommitted) { + authSession = await finalizeOwnedMcpAuthorization({ + authorizationCode: code, + pendingSession, + provider, + options, + ...(prepared && recovery + ? { + recovery: { + authorizationCompletionId: recovery.authorizationCompletionId, + expectedVersion: prepared.version, + }, + } + : {}), + }); + } else { + authSession = pendingSession; + } + if (prepared && recovery && receiptCommitted) { + const completed = await activateAndScheduleAgentTurnAuthorizationRecovery( + { + authorizationCompletionId: recovery.authorizationCompletionId, + conversationId: prepared.conversationId, + expectedVersion: prepared.version, + sessionId: prepared.sessionId, + }, + options.agentContinueOptions, + ); + if (!completed) { + throw new Error("MCP turn changed while activating callback recovery"); + } + } try { await deleteMcpAuthSession(authSession.authSessionId); } catch (cleanupError) { @@ -586,13 +808,39 @@ export async function GET( ); } - waitUntil(() => - resumeAuthorizedMcpTurn({ - authSession, - agentRunner: options.agentRunner, - provider, - }), - ); + waitUntil(async () => { + try { + await resumeAuthorizedMcpTurn({ + authSession, + agentRunner: options.agentRunner, + provider, + recoverableSlackDelivery: options.recoverableSlackDelivery, + turnLifecycle: options.turnLifecycle, + }); + } catch (resumeError) { + if (resumeError instanceof ResumeTurnBusyError) { + await wakeAuthorizationCompletedAgentTurn( + { + conversationId: authSession.conversationId, + provider, + sessionId: authSession.sessionId, + }, + options.agentContinueOptions, + ); + return; + } + logException( + resumeError, + "mcp_oauth_callback_resume_failed", + { conversationId: authSession.conversationId }, + { + "app.ai.session_id": authSession.sessionId, + "app.credential.provider": provider, + }, + "Failed to resume MCP OAuth-authorized Slack turn", + ); + } + }); return htmlResponse("success"); } catch (callbackError) { diff --git a/packages/junior/src/handlers/oauth-callback.ts b/packages/junior/src/handlers/oauth-callback.ts index d8ef21cc9..d24aaccf8 100644 --- a/packages/junior/src/handlers/oauth-callback.ts +++ b/packages/junior/src/handlers/oauth-callback.ts @@ -28,7 +28,11 @@ import { getPersistedThreadState, persistThreadStateById, } from "@/chat/runtime/thread-state"; -import { buildDeliveredTurnStatePatch } from "@/chat/runtime/delivered-turn-state"; +import { + buildDeliveredTurnStatePatch, + buildRecoveredDeliveredTurnStatePatch, +} from "@/chat/runtime/delivered-turn-state"; +import { recoverSlackDeliveryForTurn } from "@/chat/runtime/slack-delivery-recovery"; import { pluginCatalogRuntime } from "@/chat/plugins/catalog-runtime"; import { buildOAuthTokenRequest, @@ -48,6 +52,7 @@ import { lookupSlackActor } from "@/chat/slack/user"; import { getStateAdapter } from "@/chat/state/adapter"; import { coerceThreadArtifactsState } from "@/chat/state/artifacts"; import { + createAgentTurnAuthorizationCompletionId, failAgentTurnSessionRecord, getAgentTurnSessionRecord, abandonAgentTurnSessionRecord, @@ -63,13 +68,24 @@ import { } from "@/chat/services/pending-auth"; import { escapeXml } from "@/chat/xml"; import type { WaitUntilFn } from "@/handlers/types"; -import { scheduleAgentContinue } from "@/chat/services/agent-continue"; +import { + activateAndScheduleAgentTurnAuthorizationRecovery, + prepareAgentTurnAuthorizationRecoveryUnderActiveLock, + scheduleAgentContinue, + wakeAuthorizationCompletedAgentTurn, + type ScheduleAgentContinueOptions, +} from "@/chat/services/agent-continue"; import type { AgentRunResult } from "@/chat/services/turn-result"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; import { requireSlackDestination } from "@/chat/destination"; +import type { ConversationTurnLifecycle } from "@/chat/conversations/turn-lifecycle"; +import type { RecoverableSlackDelivery } from "@/chat/slack/recoverable-delivery"; interface OAuthCallbackOptions { agentRunner: AgentRunner; + agentContinueOptions?: ScheduleAgentContinueOptions; + recoverableSlackDelivery?: RecoverableSlackDelivery; + turnLifecycle?: ConversationTurnLifecycle; } /** @@ -90,7 +106,7 @@ function htmlErrorResponse( async function persistCompletedOAuthReplyState(args: { conversationId: string; sessionId: string; - reply: AgentRunResult; + reply?: AgentRunResult; }): Promise { const currentState = await getPersistedThreadState(args.conversationId); const conversation = coerceThreadConversationState(currentState); @@ -100,13 +116,19 @@ async function persistCompletedOAuthReplyState(args: { }); const artifacts = coerceThreadArtifactsState(currentState); const userMessage = getTurnUserMessage(conversation, args.sessionId); - const statePatch = buildDeliveredTurnStatePatch({ - artifacts, - conversation, - reply: args.reply, - sessionId: args.sessionId, - userMessageId: userMessage?.id, - }); + const statePatch = args.reply + ? buildDeliveredTurnStatePatch({ + artifacts, + conversation, + reply: args.reply, + sessionId: args.sessionId, + userMessageId: userMessage?.id, + }) + : buildRecoveredDeliveredTurnStatePatch({ + conversation, + sessionId: args.sessionId, + inputMessageIds: userMessage ? [userMessage.id] : [], + }); await persistThreadStateById(args.conversationId, { ...statePatch, @@ -242,13 +264,39 @@ async function resumeOAuthSessionRecordTurn( if (!sessionRecord) { return false; } + const recoveredDelivery = await recoverSlackDeliveryForTurn({ + conversationId: stored.resumeConversationId, + delivery: options.recoverableSlackDelivery, + turnId: resolvedSessionId, + }); + if (recoveredDelivery) { + if (recoveredDelivery.outcome === "accepted") { + await persistCompletedOAuthReplyState({ + conversationId: stored.resumeConversationId, + sessionId: resolvedSessionId, + }); + } + return true; + } + if (sessionRecord.state === "completed") { + await persistThreadStateById( + stored.resumeConversationId, + buildRecoveredDeliveredTurnStatePatch({ + conversation, + sessionId: resolvedSessionId, + inputMessageIds: userMessage ? [userMessage.id] : [], + }), + ); + return true; + } // Terminal session record states are already handled; do not fall through to // the pending-message resume which would re-post the original request. - if ( - sessionRecord.state === "completed" || - sessionRecord.state === "failed" || - sessionRecord.state === "abandoned" - ) { + if (sessionRecord.state === "failed" || sessionRecord.state === "abandoned") { + clearPendingAuth(conversation, resolvedSessionId); + if (conversation.processing.activeTurnId === resolvedSessionId) { + conversation.processing.activeTurnId = undefined; + } + await persistThreadStateById(stored.resumeConversationId, { conversation }); return true; } if ( @@ -275,12 +323,14 @@ async function resumeOAuthSessionRecordTurn( threadTs: stored.threadTs, messageTs: getTurnUserSlackMessageTs(userMessage), lockKey: stored.resumeConversationId, + initialText: "", + agentRunner: options.agentRunner, + recoverableSlackDelivery: options.recoverableSlackDelivery, lifecycleCorrelation: { conversationId: stored.resumeConversationId, turnId: resolvedSessionId, }, - initialText: "", - agentRunner: options.agentRunner, + turnLifecycle: options.turnLifecycle, beforeStart: async () => { const lockedState = await getPersistedThreadState( stored.resumeConversationId!, @@ -319,6 +369,9 @@ async function resumeOAuthSessionRecordTurn( !isPendingAuthLatestRequest(lockedConversation, lockedPendingAuth) ) { clearPendingAuth(lockedConversation, lockedPendingAuth.sessionId); + if (lockedConversation.processing.activeTurnId === lockedSessionId) { + lockedConversation.processing.activeTurnId = undefined; + } await persistThreadStateById(stored.resumeConversationId!, { conversation: lockedConversation, }); @@ -369,6 +422,13 @@ async function resumeOAuthSessionRecordTurn( sessionId: lockedSessionId, errorMessage: "Stored Slack actor identity did not match OAuth actor", }); + clearPendingAuth(lockedConversation, lockedSessionId); + if (lockedConversation.processing.activeTurnId === lockedSessionId) { + lockedConversation.processing.activeTurnId = undefined; + } + await persistThreadStateById(stored.resumeConversationId!, { + conversation: lockedConversation, + }); return false; } if (!lockedSessionRecord.source) { @@ -378,6 +438,13 @@ async function resumeOAuthSessionRecordTurn( sessionId: lockedSessionId, errorMessage: "Stored Slack source missing for OAuth resume", }); + clearPendingAuth(lockedConversation, lockedSessionId); + if (lockedConversation.processing.activeTurnId === lockedSessionId) { + lockedConversation.processing.activeTurnId = undefined; + } + await persistThreadStateById(stored.resumeConversationId!, { + conversation: lockedConversation, + }); return false; } @@ -399,6 +466,7 @@ async function resumeOAuthSessionRecordTurn( : (stored.pendingMessage ?? lockedUserMessage.text), messageTs: lockedMessageTs, inputMessageIds: [lockedUserMessage.id], + sliceId: lockedSessionRecord.sliceId, replyContext: { input: { conversationContext: lockedConversationContext, @@ -463,6 +531,12 @@ async function resumeOAuthSessionRecordTurn( reply, }); }, + onRecoveredSuccess: async () => { + await persistCompletedOAuthReplyState({ + conversationId: stored.resumeConversationId!, + sessionId: lockedSessionId, + }); + }, onPostDeliveryCommitFailure: async () => { await failAgentTurnSessionRecord({ conversationId: stored.resumeConversationId!, @@ -539,6 +613,8 @@ async function resumePendingOAuthMessage( messageTs, connectedText: "", agentRunner: options.agentRunner, + recoverableSlackDelivery: options.recoverableSlackDelivery, + turnLifecycle: options.turnLifecycle, replyContext: { input: { conversationContext, @@ -646,8 +722,6 @@ export async function GET( ); } - await stateAdapter.delete(stateKey); - const clientId = process.env[providerConfig.clientIdEnv]?.trim(); const clientSecret = process.env[providerConfig.clientSecretEnv]?.trim(); if (!clientId || !clientSecret) { @@ -669,87 +743,161 @@ export async function GET( const redirectUri = `${baseUrl}${providerConfig.callbackPath}`; const requestedScope = stored.scope ?? providerConfig.scope; - - let tokenResponse: Response; - try { - const tokenRequest = buildOAuthTokenRequest({ - clientId, - clientSecret, - payload: { - grant_type: "authorization_code", - code, - redirect_uri: redirectUri, - }, - tokenAuthMethod: providerConfig.tokenAuthMethod, - tokenExtraHeaders: providerConfig.tokenExtraHeaders, - }); - tokenResponse = await fetch(providerConfig.tokenEndpoint, { - method: "POST", - headers: tokenRequest.headers, - body: tokenRequest.body, - }); - } catch { - return htmlErrorResponse( - "Connection failed", - "Failed to exchange the authorization code. Please try again.", - 500, - ); + const userTokenStore = createUserTokenStore(); + const sessionRecord = + stored.resumeConversationId && stored.resumeSessionId + ? await getAgentTurnSessionRecord( + stored.resumeConversationId, + stored.resumeSessionId, + ) + : undefined; + const recoverableSession = + sessionRecord?.state === "awaiting_resume" && + sessionRecord.resumeReason === "auth" + ? sessionRecord + : undefined; + const prepared = recoverableSession + ? await prepareAgentTurnAuthorizationRecoveryUnderActiveLock({ + authorizationCompletionId: createAgentTurnAuthorizationCompletionId({ + attemptId: state, + authorizationKind: "plugin", + provider, + }), + authorizationKind: "plugin", + conversationId: stored.resumeConversationId!, + expectedVersion: recoverableSession.version, + provider, + sessionId: stored.resumeSessionId!, + userId: stored.userId, + }) + : undefined; + if (recoverableSession && !prepared) { + throw new Error("OAuth turn changed while preparing callback recovery"); } + const recovery = prepared?.authorizationRecovery; + const receiptCommitted = recovery + ? recovery.active || + (await userTokenStore.hasAuthorizationCompletion( + recovery.userId, + recovery.provider, + recovery.authorizationCompletionId, + )) + : false; + if (!receiptCommitted) { + let tokenResponse: Response; + try { + const tokenRequest = buildOAuthTokenRequest({ + clientId, + clientSecret, + payload: { + grant_type: "authorization_code", + code, + redirect_uri: redirectUri, + }, + tokenAuthMethod: providerConfig.tokenAuthMethod, + tokenExtraHeaders: providerConfig.tokenExtraHeaders, + }); + tokenResponse = await fetch(providerConfig.tokenEndpoint, { + method: "POST", + headers: tokenRequest.headers, + body: tokenRequest.body, + }); + } catch { + return htmlErrorResponse( + "Connection failed", + "Failed to exchange the authorization code. Please try again.", + 500, + ); + } - if (!tokenResponse.ok) { - return htmlErrorResponse( - "Connection failed", - "The token exchange with the provider failed. Please try again.", - 500, - ); - } + if (!tokenResponse.ok) { + return htmlErrorResponse( + "Connection failed", + "The token exchange with the provider failed. Please try again.", + 500, + ); + } - let parsedTokenResponse; - try { - const tokenData = (await tokenResponse.json()) as Record; - parsedTokenResponse = parseOAuthTokenResponse(tokenData, requestedScope, { - treatEmptyScopeAsUnreported: providerConfig.treatEmptyScopeAsUnreported, - }); - } catch { - return htmlErrorResponse( - "Connection failed", - "The provider returned an incomplete token response. Please try again.", - 500, - ); - } + let parsedTokenResponse; + try { + const tokenData = (await tokenResponse.json()) as Record; + parsedTokenResponse = parseOAuthTokenResponse(tokenData, requestedScope, { + treatEmptyScopeAsUnreported: providerConfig.treatEmptyScopeAsUnreported, + }); + } catch { + return htmlErrorResponse( + "Connection failed", + "The provider returned an incomplete token response. Please try again.", + 500, + ); + } - if (!hasRequiredOAuthScope(parsedTokenResponse.scope, requestedScope)) { - return htmlErrorResponse( - "Connection failed", - `The ${providerLabel} authorization did not grant the access Junior requires. Return to Slack and ask Junior to connect your ${providerLabel} account again.`, - 400, - ); + if (!hasRequiredOAuthScope(parsedTokenResponse.scope, requestedScope)) { + return htmlErrorResponse( + "Connection failed", + `The ${providerLabel} authorization did not grant the access Junior requires. Return to Slack and ask Junior to connect your ${providerLabel} account again.`, + 400, + ); + } + + let account: Awaited>; + try { + account = await resolvePluginOAuthAccount({ + provider, + tokens: parsedTokenResponse, + }); + } catch { + return htmlErrorResponse( + "Connection failed", + `Junior could not verify the connected ${providerLabel} account. Please try again.`, + 500, + ); + } + const tokens = { + ...parsedTokenResponse, + ...(account ? { account } : {}), + }; + if (recovery) { + // Persist the receipt immediately after consuming the one-time code. + // Activation is separately fenced and heartbeat can recover it later. + await userTokenStore.setForAuthorizationCompletion( + stored.userId, + provider, + tokens, + recovery.authorizationCompletionId, + ); + } else { + await userTokenStore.set(stored.userId, provider, tokens); + } } - const userTokenStore = createUserTokenStore(); - let account: Awaited>; - try { - account = await resolvePluginOAuthAccount({ - provider, - tokens: parsedTokenResponse, - }); - } catch { - return htmlErrorResponse( - "Connection failed", - `Junior could not verify the connected ${providerLabel} account. Please try again.`, - 500, + if (prepared && recovery) { + const activated = await activateAndScheduleAgentTurnAuthorizationRecovery( + { + authorizationCompletionId: recovery.authorizationCompletionId, + conversationId: prepared.conversationId, + expectedVersion: prepared.version, + sessionId: prepared.sessionId, + }, + options.agentContinueOptions, ); + if (!activated) { + throw new Error("OAuth turn changed while activating callback recovery"); + } } - await userTokenStore.set(stored.userId, provider, { - ...parsedTokenResponse, - ...(account ? { account } : {}), - }); + await stateAdapter.delete(stateKey); waitUntil(async () => { try { await publishAppHomeView(getSlackClient(), stored.userId, userTokenStore); - } catch { - // best effort + } catch (error) { + logException( + error, + "oauth_callback_app_home_publish_failed", + {}, + { "app.credential.provider": stored.provider }, + "Failed to refresh Slack app home after OAuth callback", + ); } }); @@ -762,37 +910,70 @@ export async function GET( } } catch (error) { if (error instanceof ResumeTurnBusyError) { - logWarn( - "oauth_callback_resume_busy", - { - ...(stored.resumeConversationId && { + if (stored.resumeConversationId && stored.resumeSessionId) { + await wakeAuthorizationCompletedAgentTurn( + { conversationId: stored.resumeConversationId, - }), - ...(stored.userId && { actorId: stored.userId }), - ...(stored.channelId && { channelId: stored.channelId }), - }, - { - "app.credential.provider": stored.provider, - ...(stored.resumeSessionId && { - "app.ai.session_id": stored.resumeSessionId, - }), - }, - "OAuth callback resume was busy; user must send another message to continue", - ); + provider: stored.provider, + sessionId: stored.resumeSessionId, + }, + options.agentContinueOptions, + ); + } else { + logWarn( + "oauth_callback_resume_busy_without_session", + {}, + { + "app.credential.provider": stored.provider, + ...(stored.channelId + ? { "messaging.destination.name": stored.channelId } + : {}), + }, + "OAuth callback resume was busy without a durable session to wake", + ); + } return; } - throw error; + logException( + error, + "oauth_callback_resume_failed", + { + ...(stored.resumeConversationId && { + conversationId: stored.resumeConversationId, + }), + }, + { + "app.credential.provider": stored.provider, + ...(stored.resumeSessionId && { + "app.ai.session_id": stored.resumeSessionId, + }), + }, + "Failed to resume OAuth-authorized Slack turn", + ); } }); } else if (stored.channelId && stored.threadTs) { const { channelId, threadTs } = stored; - waitUntil(() => - postSlackMessage({ - channelId, - threadTs, - text: `Your ${providerLabel} account is now connected. You can start using ${providerLabel} commands.`, - }), - ); + waitUntil(async () => { + try { + await postSlackMessage({ + channelId, + threadTs, + text: `Your ${providerLabel} account is now connected. You can start using ${providerLabel} commands.`, + }); + } catch (error) { + logException( + error, + "oauth_callback_connected_message_failed", + {}, + { + "app.credential.provider": stored.provider, + "app.messaging.channel.id": channelId, + }, + "Failed to post OAuth connection confirmation to Slack", + ); + } + }); } const statusMessage = stored.pendingMessage diff --git a/packages/junior/src/nitro.ts b/packages/junior/src/nitro.ts index b154bf3bc..a07268f81 100644 --- a/packages/junior/src/nitro.ts +++ b/packages/junior/src/nitro.ts @@ -13,6 +13,7 @@ import { import { PLUGIN_TASK_QUEUE_TOPIC } from "@/chat/plugins/task-queue"; import { resolveConversationWorkQueueTopic } from "@/chat/task-execution/vercel-queue"; import { + JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE, JUNIOR_CONVERSATION_WORK_CALLBACK_ROUTE, JUNIOR_HEARTBEAT_CRON_SCHEDULE, JUNIOR_HEARTBEAT_ROUTE, @@ -27,6 +28,7 @@ import { } from "./plugins"; import { loadPluginSetFromModule, resolvePluginModule } from "./plugin-module"; import type { JuniorDashboardOptions } from "./app"; +import { resolveConfiguredFunctionMaxDurationSeconds } from "./function-duration"; export type JuniorNitroDashboardOptions = JuniorDashboardOptions; @@ -64,7 +66,6 @@ type RollupLikeConfig = { plugins?: unknown[]; }; -const DEFAULT_FUNCTION_MAX_DURATION_SECONDS = 300; const VERCEL_QUEUE_TRIGGER_TYPE = "queue/v2beta"; function isPluginModuleReference( @@ -143,9 +144,50 @@ function bundleOpenTelemetryLoaderHooks(nitro: Nitro): void { } } -function configureVercelDeployment(nitro: Nitro, options: JuniorNitroOptions) { - const defaultMaxDuration = - options.maxDuration ?? DEFAULT_FUNCTION_MAX_DURATION_SECONDS; +function resolveFunctionMaxDuration( + nitro: Nitro, + options: JuniorNitroOptions, +): number { + const existing = nitro.options.vercel?.functions?.maxDuration; + if (existing === "max") { + throw new Error( + 'Junior requires a numeric Vercel maxDuration so runtime leases match the host window. Configure juniorNitro({ maxDuration: }) instead of maxDuration: "max".', + ); + } + if ( + options.maxDuration !== undefined && + existing !== undefined && + existing !== options.maxDuration + ) { + throw new Error( + `juniorNitro({ maxDuration: ${options.maxDuration} }) conflicts with Nitro Vercel functions.maxDuration ${existing}. Configure the duration once.`, + ); + } + const resolved = + existing ?? + options.maxDuration ?? + resolveConfiguredFunctionMaxDurationSeconds(); + for (const route of [ + JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE, + JUNIOR_CONVERSATION_WORK_CALLBACK_ROUTE, + JUNIOR_PLUGIN_TASK_CALLBACK_ROUTE, + ]) { + const routeDuration = + nitro.options.vercel?.functionRules?.[route]?.maxDuration; + if (routeDuration !== undefined && routeDuration !== resolved) { + throw new Error( + `Vercel function rule ${route} maxDuration ${routeDuration} conflicts with Junior's configured duration ${resolved}. Configure the duration once with juniorNitro({ maxDuration }).`, + ); + } + } + return resolved; +} + +function configureVercelDeployment( + nitro: Nitro, + options: JuniorNitroOptions, +): number { + const functionMaxDurationSeconds = resolveFunctionMaxDuration(nitro, options); const queueTopic = resolveConversationWorkQueueTopic({ topic: options.conversationWorkQueueTopic, }); @@ -175,11 +217,17 @@ function configureVercelDeployment(nitro: Nitro, options: JuniorNitroOptions) { } nitro.options.vercel.functions ??= {}; - nitro.options.vercel.functions.maxDuration ??= defaultMaxDuration; - const callbackMaxDuration = - nitro.options.vercel.functions.maxDuration ?? defaultMaxDuration; + nitro.options.vercel.functions.maxDuration = functionMaxDurationSeconds; nitro.options.vercel.functionRules ??= {}; + const existingAgentDispatchRule = + nitro.options.vercel.functionRules[JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE] ?? + {}; + nitro.options.vercel.functionRules[JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE] = { + ...existingAgentDispatchRule, + maxDuration: functionMaxDurationSeconds, + }; + const existingRule = nitro.options.vercel.functionRules[ JUNIOR_CONVERSATION_WORK_CALLBACK_ROUTE @@ -193,8 +241,8 @@ function configureVercelDeployment(nitro: Nitro, options: JuniorNitroOptions) { nitro.options.vercel.functionRules[JUNIOR_CONVERSATION_WORK_CALLBACK_ROUTE] = { - maxDuration: callbackMaxDuration, ...existingRule, + maxDuration: functionMaxDurationSeconds, experimentalTriggers: [ ...otherTriggers, { @@ -216,8 +264,8 @@ function configureVercelDeployment(nitro: Nitro, options: JuniorNitroOptions) { ); nitro.options.vercel.functionRules[JUNIOR_PLUGIN_TASK_CALLBACK_ROUTE] = { - maxDuration: callbackMaxDuration, ...existingPluginTaskRule, + maxDuration: functionMaxDurationSeconds, experimentalTriggers: [ ...otherPluginTaskTriggers, { @@ -226,6 +274,7 @@ function configureVercelDeployment(nitro: Nitro, options: JuniorNitroOptions) { }, ], }; + return functionMaxDurationSeconds; } /** Nitro module that configures deployment wiring and copies app/plugin content into the Vercel build output. */ @@ -239,7 +288,10 @@ export function juniorNitro(options: JuniorNitroOptions = {}): { options.cwd ?? nitro.options.rootDir ?? process.cwd(), ); - configureVercelDeployment(nitro, options); + const functionMaxDurationSeconds = configureVercelDeployment( + nitro, + options, + ); bundleOpenTelemetryLoaderHooks(nitro); applyRolldownTreeshakeWorkaround(nitro); @@ -277,6 +329,7 @@ export function juniorNitro(options: JuniorNitroOptions = {}): { plugins: pluginCatalogConfig, pluginRuntimeRegistrations, dashboard: options.dashboard, + functionMaxDurationSeconds, }); const copyBuildContent = async () => { diff --git a/packages/junior/src/virtual-modules.d.ts b/packages/junior/src/virtual-modules.d.ts index ed439eb84..954279a5c 100644 --- a/packages/junior/src/virtual-modules.d.ts +++ b/packages/junior/src/virtual-modules.d.ts @@ -20,6 +20,7 @@ declare module "#junior/config" { }) | undefined; export const dashboard: VirtualDashboardConfig | undefined; + export const functionMaxDurationSeconds: number | undefined; export const pluginSet: JuniorPluginSet | undefined; export const plugins: PluginCatalogConfig; export const pluginRuntimeRegistrations: string[]; diff --git a/packages/junior/tests/component/conversations/delivery-outbox.test.ts b/packages/junior/tests/component/conversations/delivery-outbox.test.ts new file mode 100644 index 000000000..cd259e42f --- /dev/null +++ b/packages/junior/tests/component/conversations/delivery-outbox.test.ts @@ -0,0 +1,568 @@ +import { asc, eq } from "drizzle-orm"; +import { describe, expect, it, vi } from "vitest"; +import { + createPendingConversationDelivery, + claimPendingConversationDelivery, + loadPendingDeliveryByTurn, + markPendingDeliveryPosting, + recordPendingDeliveryAccepted, + recordPendingDeliveryFailed, + terminalizeAcceptedPendingDelivery, + terminalizeFailedPendingDelivery, + PendingDeliveryLeaseLostError, +} from "@/chat/slack/delivery-outbox"; +import { + conversationDeliveryFailureCodeSchema, + pendingConversationDeliveryCommandSchema, + type PendingConversationDeliveryCommand, +} from "@/chat/slack/delivery-command"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; +import { purgeConversation } from "@/chat/conversations/retention"; +import { migrateSchema } from "@/chat/conversations/sql/migrations"; +import { + juniorConversationEvents, + juniorConversations, + juniorPendingDeliveries, +} from "@/db/schema"; +import { createLocalJuniorSqlFixture } from "../../fixtures/sql"; + +const CONVERSATION_ID = "slack:C123:1718123456.000000"; +const DELIVERY_ID = "delivery:turn-1"; + +function command( + overrides: Partial = {}, +): PendingConversationDeliveryCommand { + return pendingConversationDeliveryCommandSchema.parse({ + publicLocator: "0123456789Abcdefgh_-XY", + session: { + surface: "slack", + source: { + platform: "slack", + teamId: "T123", + channelId: "C123", + type: "pub", + messageTs: "1718123456.000000", + threadTs: "1718123456.000000", + }, + destination: { platform: "slack", teamId: "T123", channelId: "C123" }, + destinationVisibility: "public", + actor: { platform: "slack", teamId: "T123", userId: "U123" }, + channelName: "eng-runtime", + startedAtMs: 900, + }, + route: { channelId: "C123", threadTs: "1718123456.000000" }, + parts: [{ text: "First" }, { text: "Second" }], + completion: { + turnId: "turn-1", + inputMessageIds: ["user-1"], + assistantMessage: { + messageId: "assistant:turn-1", + text: "First\nSecond", + createdAtMs: 950, + author: { userName: "junior", isBot: true }, + }, + model: { + modelId: "openai/gpt-5.4", + committedSeq: 0, + rollbackSeq: -1, + }, + sliceId: 1, + terminal: { outcome: "success" }, + }, + ...overrides, + }); +} + +async function createDelivery( + fixture: Awaited>, + overrides: { + conversationId?: string; + deliveryId?: string; + turnId?: string; + command?: PendingConversationDeliveryCommand; + } = {}, +) { + return createPendingConversationDelivery(fixture.sql, { + conversationId: overrides.conversationId ?? CONVERSATION_ID, + deliveryId: overrides.deliveryId ?? DELIVERY_ID, + turnId: overrides.turnId ?? "turn-1", + command: overrides.command ?? command(), + nowMs: 1_000, + }); +} + +describe("pending conversation delivery outbox", { timeout: 10_000 }, () => { + it("validates the delivery destination independently from source provenance", () => { + const base = command(); + const localSource = { + ...base, + route: { channelId: "C123" }, + session: { + ...base.session, + source: { + platform: "local" as const, + type: "priv" as const, + conversationId: "local:cli:dispatch", + }, + }, + }; + expect( + pendingConversationDeliveryCommandSchema.safeParse(localSource).success, + ).toBe(true); + + const crossChannelSource = { + ...base, + route: { channelId: "C123", threadTs: "1718999999.000001" }, + session: { + ...base.session, + source: { + platform: "slack" as const, + teamId: "T999", + channelId: "C999", + type: "priv" as const, + messageTs: "1718000000.000001", + threadTs: "1718000000.000000", + }, + }, + }; + expect( + pendingConversationDeliveryCommandSchema.safeParse(crossChannelSource) + .success, + ).toBe(true); + expect( + pendingConversationDeliveryCommandSchema.safeParse({ + ...crossChannelSource, + route: { ...crossChannelSource.route, channelId: "C998" }, + }).success, + ).toBe(false); + }); + + it("validates a narrow immutable command and creates control state idempotently", async () => { + expect( + conversationDeliveryFailureCodeSchema.safeParse("retry_exhausted") + .success, + ).toBe(false); + expect( + pendingConversationDeliveryCommandSchema.safeParse({ + ...command(), + authorizationUrl: "https://secret.example/oauth", + }).success, + ).toBe(false); + const validCommand = command(); + for (const invalidCommand of [ + { + ...validCommand, + route: { ...validCommand.route, channelId: "C999" }, + }, + { + ...validCommand, + completion: { + ...validCommand.completion, + assistantMessage: { + ...validCommand.completion.assistantMessage, + messageId: "assistant:wrong-turn", + }, + }, + }, + { + ...validCommand, + completion: { + ...validCommand.completion, + model: { + ...validCommand.completion.model, + rollbackSeq: validCommand.completion.model.committedSeq + 1, + }, + }, + }, + ]) { + expect( + pendingConversationDeliveryCommandSchema.safeParse(invalidCommand) + .success, + ).toBe(false); + } + + const fixture = await createLocalJuniorSqlFixture(); + try { + await migrateSchema(fixture.sql); + const first = await createDelivery(fixture); + const second = await createDelivery(fixture); + + expect(second).toEqual(first); + expect( + await loadPendingDeliveryByTurn(fixture.sql, { + conversationId: CONVERSATION_ID, + turnId: "turn-1", + }), + ).toEqual(first); + await expect( + createSqlConversationEventStore(fixture.sql).loadHistory( + CONVERSATION_ID, + ), + ).resolves.toEqual([]); + + await expect( + createPendingConversationDelivery(fixture.sql, { + conversationId: CONVERSATION_ID, + deliveryId: "delivery:different", + turnId: "turn-1", + command: command(), + nowMs: 2_000, + }), + ).rejects.toThrow("different pending delivery"); + const nextTurnCommand = command({ + completion: { + ...command().completion, + turnId: "turn-2", + assistantMessage: { + ...command().completion.assistantMessage, + messageId: "assistant:turn-2", + }, + }, + }); + await expect( + createDelivery(fixture, { + deliveryId: "delivery:turn-2", + turnId: "turn-2", + command: nextTurnCommand, + }), + ).rejects.toThrow( + "Conversation already has a different pending delivery", + ); + } finally { + await fixture.close(); + } + }); + + it("fences concurrent claims and recovers stale posting as uncertain", async () => { + const fixture = await createLocalJuniorSqlFixture(); + try { + await migrateSchema(fixture.sql); + await createDelivery(fixture); + const claims = await Promise.all([ + claimPendingConversationDelivery(fixture.sql, { + deliveryId: DELIVERY_ID, + leaseOwner: "worker-a", + nowMs: 1_001, + leaseDurationMs: 100, + }), + claimPendingConversationDelivery(fixture.sql, { + deliveryId: DELIVERY_ID, + leaseOwner: "worker-b", + nowMs: 1_001, + leaseDurationMs: 100, + }), + ]); + const claimed = claims.find((claim) => claim !== undefined)!; + expect(claims.filter(Boolean)).toHaveLength(1); + await markPendingDeliveryPosting(fixture.sql, { + deliveryId: DELIVERY_ID, + lease: claimed.lease!, + nowMs: 1_002, + }); + + const recovered = await claimPendingConversationDelivery(fixture.sql, { + deliveryId: DELIVERY_ID, + leaseOwner: "worker-c", + nowMs: 1_102, + leaseDurationMs: 100, + }); + expect(recovered?.progress.currentState).toEqual({ + status: "uncertain", + attemptedAtMs: 1_002, + }); + await expect( + recordPendingDeliveryAccepted(fixture.sql, { + deliveryId: DELIVERY_ID, + lease: claimed.lease!, + messageTs: "1700000000.000001", + nowMs: 1_103, + }), + ).rejects.toBeInstanceOf(PendingDeliveryLeaseLostError); + } finally { + await fixture.close(); + } + }); + + it("persists ordered multipart receipts and a definitive failure", async () => { + const fixture = await createLocalJuniorSqlFixture(); + try { + await migrateSchema(fixture.sql); + await createDelivery(fixture); + const claimed = await claimPendingConversationDelivery(fixture.sql, { + deliveryId: DELIVERY_ID, + leaseOwner: "worker-a", + nowMs: 1_001, + leaseDurationMs: 1_000, + }); + const lease = claimed!.lease!; + await markPendingDeliveryPosting(fixture.sql, { + deliveryId: DELIVERY_ID, + lease, + nowMs: 1_002, + }); + const firstAccepted = await recordPendingDeliveryAccepted(fixture.sql, { + deliveryId: DELIVERY_ID, + lease, + messageTs: "1700000000.000001", + nowMs: 1_003, + }); + expect(firstAccepted.nextPartIndex).toBe(1); + expect(firstAccepted.progress.acceptedPartCount).toBe(1); + await markPendingDeliveryPosting(fixture.sql, { + deliveryId: DELIVERY_ID, + lease, + nowMs: 1_004, + }); + const failed = await recordPendingDeliveryFailed(fixture.sql, { + deliveryId: DELIVERY_ID, + lease, + failureCode: "provider_rejected", + nowMs: 1_005, + }); + expect(failed.progress.currentState).toEqual({ + status: "failed", + failureCode: "provider_rejected", + }); + await expect( + terminalizeAcceptedPendingDelivery(fixture.sql, { + conversationId: CONVERSATION_ID, + deliveryId: DELIVERY_ID, + turnId: "turn-1", + lease, + nowMs: 1_006, + finalizer: vi.fn(), + }), + ).rejects.toThrow("every part is accepted"); + + const failureFinalizer = vi.fn(async () => { + await new ConversationTurnLifecycleService( + createSqlConversationEventStore(fixture.sql), + ).fail({ + conversationId: CONVERSATION_ID, + turnId: "turn-1", + createdAtMs: 1_007, + failureCode: "delivery_failed", + }); + }); + await expect( + terminalizeFailedPendingDelivery(fixture.sql, { + conversationId: CONVERSATION_ID, + deliveryId: DELIVERY_ID, + turnId: "turn-1", + lease, + nowMs: 1_007, + finalizer: failureFinalizer, + }), + ).resolves.toBe("finalized"); + await expect( + terminalizeFailedPendingDelivery(fixture.sql, { + conversationId: CONVERSATION_ID, + deliveryId: DELIVERY_ID, + turnId: "turn-1", + lease, + nowMs: 1_008, + finalizer: failureFinalizer, + }), + ).resolves.toBe("already_finalized"); + expect(failureFinalizer).toHaveBeenCalledTimes(1); + const history = await createSqlConversationEventStore( + fixture.sql, + ).loadHistory(CONVERSATION_ID); + expect( + history.filter( + (event) => + event.data.type === "turn_failed" && + event.data.failureCode === "delivery_failed", + ), + ).toHaveLength(1); + } finally { + await fixture.close(); + } + }); + + it("rolls back finalization and retries without duplicate canonical facts", async () => { + const fixture = await createLocalJuniorSqlFixture(); + try { + await migrateSchema(fixture.sql); + await createDelivery(fixture); + const claimed = await claimPendingConversationDelivery(fixture.sql, { + deliveryId: DELIVERY_ID, + leaseOwner: "worker-a", + nowMs: 1_001, + leaseDurationMs: 5_000, + }); + const lease = claimed!.lease!; + for (const index of [0, 1]) { + await markPendingDeliveryPosting(fixture.sql, { + deliveryId: DELIVERY_ID, + lease, + nowMs: 1_010 + index * 2, + }); + await recordPendingDeliveryAccepted(fixture.sql, { + deliveryId: DELIVERY_ID, + lease, + messageTs: `1700000000.00000${index + 1}`, + nowMs: 1_011 + index * 2, + }); + } + + await expect( + terminalizeAcceptedPendingDelivery(fixture.sql, { + conversationId: CONVERSATION_ID, + deliveryId: DELIVERY_ID, + turnId: "turn-1", + lease, + nowMs: 1_020, + finalizer: async () => { + await fixture.sql + .db() + .update(juniorConversations) + .set({ title: "must roll back" }) + .where(eq(juniorConversations.conversationId, CONVERSATION_ID)); + throw new Error("finalizer failed"); + }, + }), + ).rejects.toThrow("finalizer failed"); + expect( + await loadPendingDeliveryByTurn(fixture.sql, { + conversationId: CONVERSATION_ID, + turnId: "turn-1", + }), + ).toBeDefined(); + const [conversationAfterRollback] = await fixture.sql + .db() + .select({ title: juniorConversations.title }) + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, CONVERSATION_ID)); + expect(conversationAfterRollback?.title).toBeNull(); + + await expect( + terminalizeAcceptedPendingDelivery(fixture.sql, { + conversationId: CONVERSATION_ID, + deliveryId: DELIVERY_ID, + turnId: "turn-1", + lease, + nowMs: 1_020, + finalizer: async () => { + await fixture.sql + .db() + .update(juniorConversations) + .set({ title: "wrong terminal" }) + .where(eq(juniorConversations.conversationId, CONVERSATION_ID)); + await new ConversationTurnLifecycleService( + createSqlConversationEventStore(fixture.sql), + ).complete({ + conversationId: CONVERSATION_ID, + turnId: "turn-1", + createdAtMs: 1_020, + outcome: "no_reply", + }); + }, + }), + ).rejects.toThrow("did not write the expected turn terminal"); + expect( + await loadPendingDeliveryByTurn(fixture.sql, { + conversationId: CONVERSATION_ID, + turnId: "turn-1", + }), + ).toBeDefined(); + + const finalizer = vi.fn(async () => { + await fixture.sql + .db() + .update(juniorConversations) + .set({ title: "finalized" }) + .where(eq(juniorConversations.conversationId, CONVERSATION_ID)); + await new ConversationTurnLifecycleService( + createSqlConversationEventStore(fixture.sql), + ).complete({ + conversationId: CONVERSATION_ID, + turnId: "turn-1", + createdAtMs: 1_021, + outcome: "success", + }); + }); + await expect( + terminalizeAcceptedPendingDelivery(fixture.sql, { + conversationId: CONVERSATION_ID, + deliveryId: DELIVERY_ID, + turnId: "turn-1", + lease, + nowMs: 1_021, + finalizer, + }), + ).resolves.toBe("finalized"); + await expect( + terminalizeAcceptedPendingDelivery(fixture.sql, { + conversationId: CONVERSATION_ID, + deliveryId: DELIVERY_ID, + turnId: "turn-1", + lease, + nowMs: 1_022, + finalizer, + }), + ).resolves.toBe("already_finalized"); + expect(finalizer).toHaveBeenCalledTimes(1); + await expect(createDelivery(fixture)).rejects.toThrow( + "already terminalized", + ); + const events = await createSqlConversationEventStore( + fixture.sql, + ).loadHistory(CONVERSATION_ID); + expect( + events.filter((event) => event.data.type === "turn_completed"), + ).toHaveLength(1); + } finally { + await fixture.close(); + } + }); + + it("cascades unresolved control state when its conversation row is deleted", async () => { + const fixture = await createLocalJuniorSqlFixture(); + try { + await migrateSchema(fixture.sql); + await createDelivery(fixture); + await fixture.sql + .db() + .delete(juniorConversationEvents) + .where(eq(juniorConversationEvents.conversationId, CONVERSATION_ID)); + await fixture.sql + .db() + .delete(juniorConversations) + .where(eq(juniorConversations.conversationId, CONVERSATION_ID)); + const rows = await fixture.sql + .db() + .select({ id: juniorPendingDeliveries.deliveryId }) + .from(juniorPendingDeliveries) + .orderBy(asc(juniorPendingDeliveries.deliveryId)); + expect(rows).toEqual([]); + } finally { + await fixture.close(); + } + }); + + it("deletes raw pending commands when conversation content is retained only as metadata", async () => { + const fixture = await createLocalJuniorSqlFixture(); + try { + await migrateSchema(fixture.sql); + await createDelivery(fixture); + await purgeConversation(fixture.sql, CONVERSATION_ID, { nowMs: 5_000 }); + + expect( + await loadPendingDeliveryByTurn(fixture.sql, { + conversationId: CONVERSATION_ID, + turnId: "turn-1", + }), + ).toBeUndefined(); + const [conversation] = await fixture.sql + .db() + .select({ transcriptPurgedAt: juniorConversations.transcriptPurgedAt }) + .from(juniorConversations) + .where(eq(juniorConversations.conversationId, CONVERSATION_ID)); + expect(conversation?.transcriptPurgedAt).toBeInstanceOf(Date); + } finally { + await fixture.close(); + } + }); +}); diff --git a/packages/junior/tests/component/conversations/recoverable-slack-delivery.test.ts b/packages/junior/tests/component/conversations/recoverable-slack-delivery.test.ts new file mode 100644 index 000000000..13bfc2e05 --- /dev/null +++ b/packages/junior/tests/component/conversations/recoverable-slack-delivery.test.ts @@ -0,0 +1,729 @@ +import { describe, expect, it, vi } from "vitest"; +import { migrateSchema } from "@/chat/conversations/sql/migrations"; +import { createSqlConversationMessageStore } from "@/chat/conversations/sql/messages"; +import { createSqlConversationEventStore } from "@/chat/conversations/sql/history"; +import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; +import { + claimPendingConversationDelivery, + loadPendingDeliveryByTurn, +} from "@/chat/slack/delivery-outbox"; +import { + RecoverableSlackDeliveryService, + type RecoverableSlackDeliveryPort, +} from "@/chat/slack/recoverable-delivery"; +import type { PendingConversationDeliveryCommandDraft } from "@/chat/slack/delivery-command"; +import { createLocalJuniorSqlFixture } from "../../fixtures/sql"; +import type { JuniorSqlDatabase } from "@/db/db"; +import type { ConversationModelMessage } from "@/chat/conversations/model-message"; +import type { PiMessage } from "@/chat/pi/messages"; +import { commitMessages } from "@/chat/conversations/projection"; +import { projectConversationEvents } from "@/chat/pi/conversation-events"; + +const conversationId = "slack:C123:1718123456.000000"; + +function command( + parts = ["First", "Second"], +): PendingConversationDeliveryCommandDraft { + return { + publicLocator: "0123456789Abcdefgh_-XY", + route: { channelId: "C123", threadTs: "1718123456.000000" }, + session: { + surface: "slack", + source: { + platform: "slack", + teamId: "T123", + channelId: "C123", + type: "pub", + messageTs: "1718123456.000000", + threadTs: "1718123456.000000", + }, + destination: { platform: "slack", teamId: "T123", channelId: "C123" }, + destinationVisibility: "public", + actor: { platform: "slack", teamId: "T123", userId: "U123" }, + startedAtMs: 900, + }, + parts: parts.map((text) => ({ + text, + blocks: [{ type: "markdown", text }], + })), + completion: { + turnId: "turn-1", + inputMessageIds: ["user-1"], + assistantMessage: { + messageId: "assistant:turn-1", + text: parts.join("\n"), + createdAtMs: 950, + author: { userName: "junior-test", isBot: true }, + }, + model: { + modelId: "openai/gpt-5.4", + }, + sliceId: 1, + terminal: { outcome: "success" }, + }, + }; +} + +async function setup(port: RecoverableSlackDeliveryPort) { + const fixture = await createLocalJuniorSqlFixture(); + await migrateSchema(fixture.sql); + await createSqlConversationMessageStore(fixture.sql).record(conversationId, [ + { + messageId: "user-1", + role: "user", + text: "Question", + createdAtMs: 900, + }, + ]); + const priorModelMessages = [ + { role: "user", content: [{ type: "text", text: "Question" }] }, + ] as unknown as PiMessage[]; + await commitMessages({ + conversationId, + modelId: "openai/gpt-5.4", + messages: priorModelMessages, + executor: fixture.sql, + }); + let nowMs = 1_000; + const service = new RecoverableSlackDeliveryService( + fixture.sql, + port, + () => nowMs, + ); + const pending = await service.createIntent({ + conversationId, + deliveryId: "slack:turn-1", + turnId: "turn-1", + command: command(), + modelMessages: [ + ...priorModelMessages, + { + role: "assistant", + content: [{ type: "text", text: "First\nSecond" }], + }, + ] as unknown as ConversationModelMessage[], + }); + return { + fixture, + pending, + service, + setNow(value: number) { + nowMs = value; + }, + }; +} + +describe("recoverable Slack delivery", { timeout: 10_000 }, () => { + it("rejects an empty transcript when retrying a nonempty intent", async () => { + const test = await setup({ post: vi.fn(), reconcile: vi.fn() }); + try { + await expect( + test.service.createIntent({ + conversationId, + deliveryId: "slack:turn-1", + turnId: "turn-1", + command: command(), + modelMessages: [], + }), + ).rejects.toThrow("Pending delivery command does not match its intent"); + } finally { + await test.fixture.close(); + } + }); + + it("posts multipart once and atomically finalizes the accepted turn", async () => { + let posted = 0; + const port: RecoverableSlackDeliveryPort = { + post: vi.fn(async () => ({ + outcome: "accepted" as const, + ts: `1718123457.00000${++posted}` as never, + })), + reconcile: vi.fn(), + }; + const test = await setup(port); + try { + expect(test.pending.command.completion.model).toEqual({ + modelId: "openai/gpt-5.4", + committedSeq: expect.any(Number), + rollbackSeq: expect.any(Number), + }); + expect(test.pending.command.completion.model).not.toHaveProperty( + "messages", + ); + expect( + ( + await createSqlConversationEventStore(test.fixture.sql).loadHistory( + conversationId, + ) + ).filter((event) => event.data.type === "message"), + ).toHaveLength(2); + await expect(test.service.advance(test.pending)).resolves.toEqual({ + outcome: "accepted", + messageTs: "1718123457.000002", + }); + expect(port.post).toHaveBeenCalledTimes(2); + expect(port.reconcile).not.toHaveBeenCalled(); + expect( + await loadPendingDeliveryByTurn(test.fixture.sql, { + conversationId, + turnId: "turn-1", + }), + ).toBeUndefined(); + const history = await createSqlConversationEventStore( + test.fixture.sql, + ).loadHistory(conversationId); + expect( + history.filter( + (event) => + event.data.type === "visible_message_recorded" && + event.data.messageId === "assistant:turn-1", + ), + ).toHaveLength(1); + expect( + history.filter((event) => event.data.type === "turn_completed"), + ).toHaveLength(1); + expect( + history.filter((event) => event.data.type === "message"), + ).toHaveLength(2); + await expect( + test.service.loadTerminalOutcome({ + conversationId, + turnId: "turn-1", + acceptanceEvidence: "visible_assistant", + }), + ).resolves.toEqual({ + deliveryOutcome: "accepted", + modelSucceeded: true, + }); + } finally { + await test.fixture.close(); + } + }); + + it("retains accepted intent when pre-terminal repair crashes", async () => { + let posted = 0; + const port: RecoverableSlackDeliveryPort = { + post: vi.fn(async () => ({ + outcome: "accepted" as const, + ts: `1718123457.00000${++posted}` as never, + })), + reconcile: vi.fn(), + }; + const test = await setup(port); + const repair = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error("repair crashed")) + .mockResolvedValue(undefined); + try { + await expect( + test.service.advance(test.pending, { + beforeTerminalize: repair, + }), + ).rejects.toThrow("repair crashed"); + const retained = await test.service.loadByTurn({ + conversationId, + turnId: "turn-1", + }); + expect(retained?.progress.acceptedPartCount).toBe(2); + expect( + await test.service.loadTerminalOutcome({ + conversationId, + turnId: "turn-1", + acceptanceEvidence: "known_outbox_intent", + }), + ).toBeUndefined(); + + await expect( + test.service.advance(retained!, { beforeTerminalize: repair }), + ).resolves.toEqual({ + outcome: "accepted", + messageTs: "1718123457.000002", + }); + expect(port.post).toHaveBeenCalledTimes(2); + expect(repair).toHaveBeenCalledTimes(2); + await expect( + test.service.loadByTurn({ conversationId, turnId: "turn-1" }), + ).resolves.toBeUndefined(); + } finally { + await test.fixture.close(); + } + }); + + it("reconciles an ambiguous accepted write without reposting part one", async () => { + const post = vi + .fn() + .mockResolvedValueOnce({ + outcome: "uncertain", + reason: "transport_error", + }) + .mockResolvedValueOnce({ + outcome: "accepted", + ts: "1718123457.000002" as never, + }); + const reconcile = vi + .fn() + .mockResolvedValue({ + outcome: "accepted", + ts: "1718123457.000001" as never, + }); + const test = await setup({ post, reconcile }); + try { + await expect(test.service.advance(test.pending)).resolves.toEqual({ + outcome: "pending", + retryAtMs: 6_000, + }); + test.setNow(6_000); + const pending = await test.service.loadByTurn({ + conversationId, + turnId: "turn-1", + }); + await expect(test.service.advance(pending!)).resolves.toEqual({ + outcome: "accepted", + messageTs: "1718123457.000002", + }); + expect(post).toHaveBeenCalledTimes(2); + expect(reconcile).toHaveBeenCalledTimes(1); + expect(post.mock.calls.map(([input]) => input.teamId)).toEqual([ + "T123", + "T123", + ]); + expect(reconcile.mock.calls[0]?.[0].teamId).toBe("T123"); + } finally { + await test.fixture.close(); + } + }); + + it("waits a grace after first absence and performs a fresh scan before repost", async () => { + const post = vi + .fn() + .mockResolvedValueOnce({ + outcome: "uncertain", + reason: "transport_error", + }) + .mockResolvedValueOnce({ + outcome: "accepted", + ts: "1718123457.000001" as never, + }) + .mockResolvedValueOnce({ + outcome: "accepted", + ts: "1718123457.000002" as never, + }); + const reconcile = vi + .fn() + .mockResolvedValue({ outcome: "confirmed_absent" }); + const test = await setup({ post, reconcile }); + try { + await test.service.advance(test.pending); + test.setNow(6_000); + await test.service.advance( + (await test.service.loadByTurn({ conversationId, turnId: "turn-1" }))!, + ); + expect(post).toHaveBeenCalledTimes(1); + test.setNow(36_000); + await expect( + test.service.advance( + (await test.service.loadByTurn({ + conversationId, + turnId: "turn-1", + }))!, + ), + ).resolves.toEqual({ + outcome: "accepted", + messageTs: "1718123457.000002", + }); + expect(reconcile).toHaveBeenCalledTimes(2); + expect(reconcile.mock.calls[1]?.[0]).not.toHaveProperty("cursor"); + expect(reconcile.mock.calls[0]?.[0].oldestTs).toBe("0.000000"); + expect(post).toHaveBeenCalledTimes(3); + } finally { + await test.fixture.close(); + } + }); + + it("keeps rate-limited writes pending without reconciliation", async () => { + const post = vi + .fn() + .mockResolvedValue({ + outcome: "retryable_absence", + reason: "rate_limited", + retryAtMs: 61_000, + }); + const reconcile = vi.fn(); + const test = await setup({ post, reconcile }); + try { + await expect(test.service.advance(test.pending)).resolves.toEqual({ + outcome: "pending", + retryAtMs: 61_000, + }); + const pending = await test.service.loadByTurn({ + conversationId, + turnId: "turn-1", + }); + expect(pending?.progress.currentState).toEqual({ status: "pending" }); + expect(reconcile).not.toHaveBeenCalled(); + } finally { + await test.fixture.close(); + } + }); + + it("honors reconciliation Retry-After without a fixed-delay loop", async () => { + const post = vi + .fn() + .mockResolvedValue({ outcome: "uncertain", reason: "transport_error" }); + const reconcile = vi + .fn() + .mockResolvedValue({ outcome: "retryable", retryAtMs: 90_000 }); + const test = await setup({ post, reconcile }); + try { + await test.service.advance(test.pending); + test.setNow(6_000); + await expect( + test.service.advance( + (await test.service.loadByTurn({ + conversationId, + turnId: "turn-1", + }))!, + ), + ).resolves.toEqual({ outcome: "pending", retryAtMs: 90_000 }); + expect(reconcile).toHaveBeenCalledOnce(); + expect(post).toHaveBeenCalledOnce(); + } finally { + await test.fixture.close(); + } + }); + + it("backs off permanent reconciliation failures without authorizing a repost", async () => { + const post = vi + .fn() + .mockResolvedValue({ outcome: "uncertain", reason: "transport_error" }); + const reconcile = vi + .fn() + .mockResolvedValue({ + outcome: "unresolved", + reason: "permanent_provider_error", + providerErrorCode: "missing_scope", + }); + const test = await setup({ post, reconcile }); + try { + await test.service.advance(test.pending); + test.setNow(6_000); + await expect( + test.service.advance( + (await test.service.loadByTurn({ + conversationId, + turnId: "turn-1", + }))!, + ), + ).resolves.toEqual({ + outcome: "pending", + retryAtMs: 3_606_000, + }); + const pending = await test.service.loadByTurn({ + conversationId, + turnId: "turn-1", + }); + expect(pending?.progress.currentState.status).toBe("uncertain"); + expect(pending?.nextAttemptAtMs).toBe(3_606_000); + test.setNow(11_000); + await expect(test.service.advance(pending!)).resolves.toEqual({ + outcome: "pending", + retryAtMs: 3_606_000, + }); + expect(post).toHaveBeenCalledOnce(); + expect(reconcile).toHaveBeenCalledOnce(); + } finally { + await test.fixture.close(); + } + }); + + it("defers behind an active lease until the lease expires", async () => { + const port: RecoverableSlackDeliveryPort = { + post: vi.fn(), + reconcile: vi.fn(), + }; + const test = await setup(port); + try { + const claimed = await claimPendingConversationDelivery(test.fixture.sql, { + deliveryId: test.pending.deliveryId, + leaseOwner: "other-worker", + nowMs: 1_001, + leaseDurationMs: 59_999, + }); + expect(claimed?.lease?.expiresAtMs).toBe(61_000); + await expect(test.service.advance(test.pending)).resolves.toEqual({ + outcome: "pending", + retryAtMs: 61_000, + }); + expect(port.post).not.toHaveBeenCalled(); + } finally { + await test.fixture.close(); + } + }); + + it("terminally fails a permanent provider rejection without persisting the assistant", async () => { + const port: RecoverableSlackDeliveryPort = { + post: vi.fn(async () => ({ + outcome: "definitive_failure" as const, + reason: "api_rejected" as const, + })), + reconcile: vi.fn(), + }; + const test = await setup(port); + try { + await createSqlConversationEventStore(test.fixture.sql).append( + conversationId, + [ + { + data: { + type: "authorization_completed", + kind: "mcp", + provider: "linear", + actorId: "U123", + authorizationId: "auth-1", + }, + createdAtMs: 1_001, + }, + ], + ); + await expect(test.service.advance(test.pending)).resolves.toEqual({ + outcome: "failed", + }); + const history = await createSqlConversationEventStore( + test.fixture.sql, + ).loadHistory(conversationId); + expect( + history.filter( + (event) => + event.data.type === "turn_failed" && + event.data.failureCode === "delivery_failed", + ), + ).toHaveLength(1); + expect( + history.some( + (event) => + event.data.type === "visible_message_recorded" && + event.data.messageId === "assistant:turn-1", + ), + ).toBe(false); + expect( + history.filter((event) => event.data.type === "message"), + ).toHaveLength(4); + expect( + projectConversationEvents( + await createSqlConversationEventStore( + test.fixture.sql, + ).loadCurrentEpoch(conversationId), + ).messages, + ).toEqual([ + { role: "user", content: [{ type: "text", text: "Question" }] }, + { + role: "user", + content: [ + { + type: "text", + text: 'MCP authorization completed for provider "linear". Continue the blocked request and retry the provider operation if needed.', + }, + ], + timestamp: 1_001, + }, + ]); + await expect( + test.service.loadTerminalOutcome({ + conversationId, + turnId: "turn-1", + acceptanceEvidence: "visible_assistant", + }), + ).resolves.toEqual({ + deliveryOutcome: "failed", + modelSucceeded: false, + }); + expect( + history.some( + (event) => + event.data.type === "visible_message_replied" && + event.data.messageId === "user-1", + ), + ).toBe(false); + } finally { + await test.fixture.close(); + } + }); + + it("retains model continuity when Slack accepted part of a failed multipart reply", async () => { + let attempt = 0; + const port: RecoverableSlackDeliveryPort = { + post: vi.fn(async () => { + attempt += 1; + return attempt === 1 + ? ({ + outcome: "accepted", + ts: "1718123457.000001" as never, + } as const) + : ({ + outcome: "definitive_failure", + reason: "api_rejected", + } as const); + }), + reconcile: vi.fn(), + }; + const test = await setup(port); + try { + await expect(test.service.advance(test.pending)).resolves.toEqual({ + outcome: "failed", + }); + expect( + projectConversationEvents( + await createSqlConversationEventStore( + test.fixture.sql, + ).loadCurrentEpoch(conversationId), + ).messages, + ).toEqual([ + { role: "user", content: [{ type: "text", text: "Question" }] }, + { + role: "assistant", + content: [{ type: "text", text: "First\nSecond" }], + }, + ]); + expect( + await createSqlConversationEventStore(test.fixture.sql).loadHistory( + conversationId, + ), + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + data: expect.objectContaining({ + type: "visible_message_recorded", + role: "assistant", + text: "First", + }), + }), + expect.objectContaining({ + data: expect.objectContaining({ + type: "turn_failed", + failureCode: "delivery_failed", + }), + }), + ]), + ); + } finally { + await test.fixture.close(); + } + }); + + it("maps a lost terminal commit acknowledgement from the authoritative fact", async () => { + const port: RecoverableSlackDeliveryPort = { + post: vi.fn(async () => ({ + outcome: "accepted" as const, + ts: "1718123457.000001" as never, + })), + reconcile: vi.fn(), + }; + const test = await setup(port); + let lockDepth = 0; + let injected = false; + const sql: JuniorSqlDatabase = { + db: () => test.fixture.sql.db(), + transaction: (callback) => test.fixture.sql.transaction(callback), + withLock: async (name, callback) => { + lockDepth += 1; + try { + const result = await test.fixture.sql.withLock(name, callback); + if (lockDepth === 1 && !injected) { + const terminal = ( + await createSqlConversationEventStore( + test.fixture.sql, + ).loadHistory(conversationId) + ).some((event) => event.data.type === "turn_completed"); + if (terminal) { + injected = true; + throw new Error("commit acknowledgement lost"); + } + } + return result; + } finally { + lockDepth -= 1; + } + }, + }; + const service = new RecoverableSlackDeliveryService(sql, port, () => 1_001); + try { + await expect(service.advance(test.pending)).resolves.toEqual({ + outcome: "accepted", + }); + expect(injected).toBe(true); + await expect( + service.loadByTurn({ conversationId, turnId: test.pending.turnId }), + ).resolves.toBeUndefined(); + expect( + await service.loadTerminalOutcome({ + conversationId, + turnId: test.pending.turnId, + acceptanceEvidence: "visible_assistant", + }), + ).toEqual({ deliveryOutcome: "accepted", modelSucceeded: true }); + } finally { + await test.fixture.close(); + } + }); + + it("requires assistant evidence at startup while known-intent recovery trusts the terminal", async () => { + const fixture = await createLocalJuniorSqlFixture(); + await migrateSchema(fixture.sql); + const events = createSqlConversationEventStore(fixture.sql); + const service = new RecoverableSlackDeliveryService( + fixture.sql, + { post: vi.fn(), reconcile: vi.fn() }, + () => 1_000, + ); + try { + await new ConversationTurnLifecycleService(events).fail({ + conversationId, + turnId: "turn-1", + createdAtMs: 1_000, + failureCode: "agent_run_failed", + }); + + await expect( + service.loadTerminalOutcome({ + conversationId, + turnId: "turn-1", + acceptanceEvidence: "visible_assistant", + }), + ).resolves.toBeUndefined(); + await expect( + service.loadTerminalOutcome({ + conversationId, + turnId: "turn-1", + acceptanceEvidence: "known_outbox_intent", + }), + ).resolves.toEqual({ + deliveryOutcome: "accepted", + modelSucceeded: false, + }); + + await createSqlConversationMessageStore(fixture.sql).record( + conversationId, + [ + { + messageId: "assistant:turn-1", + role: "assistant", + text: "Fallback response", + createdAtMs: 1_001, + }, + ], + ); + await expect( + service.loadTerminalOutcome({ + conversationId, + turnId: "turn-1", + acceptanceEvidence: "visible_assistant", + }), + ).resolves.toEqual({ + deliveryOutcome: "accepted", + modelSucceeded: false, + }); + } finally { + await fixture.close(); + } + }); +}); diff --git a/packages/junior/tests/component/runtime/agent-continue-runner.test.ts b/packages/junior/tests/component/runtime/agent-continue-runner.test.ts index b5ae92fc9..32cbaab07 100644 --- a/packages/junior/tests/component/runtime/agent-continue-runner.test.ts +++ b/packages/junior/tests/component/runtime/agent-continue-runner.test.ts @@ -8,6 +8,7 @@ import { } from "@/chat/state/turn-session"; import { neverRunAgentRunner } from "../../fixtures/agent-runner"; import { SLACK_DESTINATION } from "../../fixtures/conversation-work"; +import type { ConversationTurnLifecycle } from "@/chat/conversations/turn-lifecycle"; const SLACK_SOURCE = createSlackSource({ teamId: SLACK_DESTINATION.teamId, @@ -33,6 +34,11 @@ function restoreEnv(name: string, value: string | undefined): void { } const agentRunnerShouldNotRun = neverRunAgentRunner(); +const turnLifecycle: ConversationTurnLifecycle = { + start: async () => undefined, + complete: async () => undefined, + fail: async () => undefined, +}; describe("agent continuation runner callbacks", () => { beforeEach(async () => { @@ -121,6 +127,7 @@ describe("agent continuation runner callbacks", () => { }, { agentRunner: agentRunnerShouldNotRun, + turnLifecycle, resumeTurn: async (args) => { const prepared = await args.beforeStart?.(); if (!prepared) { @@ -224,6 +231,7 @@ describe("agent continuation runner callbacks", () => { }, { agentRunner: agentRunnerShouldNotRun, + turnLifecycle, resumeTurn: async (args) => { const prepared = await args.beforeStart?.(); if (prepared !== false) { @@ -241,94 +249,4 @@ describe("agent continuation runner callbacks", () => { errorMessage: "Stored Slack source missing for continuation", }); }); - - it("fails before continuing when stored actor and message author differ", async () => { - const conversationId = "slack:C123:1712345.0006"; - const sessionId = "turn_msg_6"; - const sessionRecord = await upsertAgentTurnSessionRecord({ - modelId: "test/model", - conversationId, - sessionId, - sliceId: 2, - state: "awaiting_resume", - destination: SLACK_DESTINATION, - resumeReason: "timeout", - actor: { - platform: "slack", - teamId: SLACK_DESTINATION.teamId, - userId: "U999", - userName: "wrong-user", - }, - piMessages: [ - { - role: "user", - content: [{ type: "text", text: "hello" }], - timestamp: 1, - }, - ], - }); - await persistThreadStateById(conversationId, { - conversation: { - schemaVersion: 1, - backfill: {}, - compactions: [], - messages: [ - { - id: "msg.6", - role: "user", - text: "resume this request", - createdAtMs: 1, - author: { - userId: "U123", - }, - }, - ], - processing: { - activeTurnId: sessionId, - }, - stats: { - compactedMessageCount: 0, - estimatedContextTokens: 0, - totalMessageCount: 1, - updatedAtMs: 1, - }, - vision: { - byFileId: {}, - }, - }, - }); - - const { continueSlackAgentRun } = - await import("@/chat/runtime/agent-continue-runner"); - - // A mismatched stored actor must never throw out of the continue - // callback (issue #727: a throw NACKs the queue delivery and wedges the - // conversation); it terminally fails the session instead. - await expect( - continueSlackAgentRun( - { - conversationId, - destination: SLACK_DESTINATION, - sessionId, - expectedVersion: sessionRecord.version, - }, - { - agentRunner: agentRunnerShouldNotRun, - resumeTurn: async (args) => { - const prepared = await args.beforeStart?.(); - if (prepared !== false) { - throw new Error("Expected continuation preparation to fail"); - } - return true; - }, - }, - ), - ).resolves.toBe(true); - await expect( - getAgentTurnSessionRecord(conversationId, sessionId), - ).resolves.toMatchObject({ - state: "failed", - errorMessage: "Stored Slack actor missing for continuation", - }); - }); }); diff --git a/packages/junior/tests/component/runtime/agent-continue.test.ts b/packages/junior/tests/component/runtime/agent-continue.test.ts index 55d4b6fab..a773fcc0b 100644 --- a/packages/junior/tests/component/runtime/agent-continue.test.ts +++ b/packages/junior/tests/component/runtime/agent-continue.test.ts @@ -12,6 +12,7 @@ import { createConversationWorkQueueTestAdapter, } from "../../fixtures/conversation-work"; import { neverRunAgentRunner } from "../../fixtures/agent-runner"; +import type { ConversationTurnLifecycle } from "@/chat/conversations/turn-lifecycle"; const ORIGINAL_ENV = vi.hoisted(() => { const original = { @@ -30,6 +31,11 @@ function restoreEnv(name: string, value: string | undefined): void { } const agentRunnerShouldNotRun = neverRunAgentRunner(); +const turnLifecycle: ConversationTurnLifecycle = { + start: async () => undefined, + complete: async () => undefined, + fail: async () => undefined, +}; describe("agent continuation scheduling", () => { beforeEach(async () => { @@ -128,7 +134,11 @@ describe("agent continuation scheduling", () => { sessionId: "turn_msg_2", expectedVersion: 1, }, - { agentRunner: agentRunnerShouldNotRun, scheduleAgentContinue }, + { + agentRunner: agentRunnerShouldNotRun, + scheduleAgentContinue, + turnLifecycle, + }, ); await vi.advanceTimersByTimeAsync(4_000); @@ -163,6 +173,7 @@ describe("agent continuation scheduling", () => { await expect( resumeAwaitingSlackContinuation(conversationId, { agentRunner: agentRunnerShouldNotRun, + turnLifecycle, }), ).resolves.toBe(false); await expect( @@ -196,6 +207,7 @@ describe("agent continuation scheduling", () => { resumeAwaitingSlackContinuation(conversationId, { agentRunner: { run: generateReply }, resumeTurn, + turnLifecycle, }), ).resolves.toBe(true); @@ -263,6 +275,7 @@ describe("agent continuation scheduling", () => { await expect( resumeAwaitingSlackContinuation(conversationId, { agentRunner: agentRunnerShouldNotRun, + turnLifecycle, }), ).resolves.toBe(false); await expect( diff --git a/packages/junior/tests/component/services/turn-session-record.test.ts b/packages/junior/tests/component/services/turn-session-record.test.ts index 4385a21d8..16f80a72e 100644 --- a/packages/junior/tests/component/services/turn-session-record.test.ts +++ b/packages/junior/tests/component/services/turn-session-record.test.ts @@ -253,6 +253,182 @@ describe("persistAuthPauseSessionRecord", () => { ); }); + it.each(["plugin", "mcp"] as const)( + "rejects a different %s authorization attempt for one parked turn", + async (authorizationKind) => { + const { + prepareAgentTurnAuthorizationRecovery, + upsertAgentTurnSessionRecord, + } = await import("@/chat/state/turn-session"); + const conversationId = `slack:C123:auth-attempt-${authorizationKind}`; + const sessionId = `turn-auth-attempt-${authorizationKind}`; + const parked = await upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + destination: SLACK_DESTINATION, + piMessages: [userMessage("connect")], + resumeReason: "auth", + sessionId, + sliceId: 2, + source: SLACK_SOURCE, + state: "awaiting_resume", + surface: "slack", + }); + const prepared = await prepareAgentTurnAuthorizationRecovery({ + authorizationCompletionId: "first-attempt", + authorizationKind, + conversationId, + expectedVersion: parked.version, + provider: "provider", + sessionId, + userId: "U123", + }); + if (!prepared) throw new Error("Expected prepared recovery"); + + await expect( + prepareAgentTurnAuthorizationRecovery({ + authorizationCompletionId: "newer-attempt", + authorizationKind, + conversationId, + expectedVersion: prepared.version, + provider: "provider", + sessionId, + userId: "U123", + }), + ).rejects.toThrow( + "Authorization recovery intent does not match callback", + ); + }, + ); + + it("prunes a missing auth recovery record only after the grace period", async () => { + const { getStateAdapter } = await import("@/chat/state/adapter"); + const { + AUTHORIZATION_RECOVERY_MISSING_RECORD_GRACE_MS, + listAuthorizationRecoveries, + registerAuthorizationRecovery, + } = await import("@/chat/state/authorization-recovery-index"); + const { recoverAuthorizationCompletedAgentTurns } = + await import("@/chat/services/agent-continue"); + const state = getStateAdapter(); + const registeredAtMs = Date.now(); + const entry = { + authorizationCompletionId: "incomplete-registration", + conversationId: "slack:C123:incomplete-auth-registration", + registeredAtMs, + sessionId: "turn-incomplete-auth-registration", + }; + await registerAuthorizationRecovery(state, entry); + + const beforeGraceMs = + registeredAtMs + AUTHORIZATION_RECOVERY_MISSING_RECORD_GRACE_MS - 1; + await expect( + recoverAuthorizationCompletedAgentTurns({ + nowMs: beforeGraceMs, + state, + }), + ).resolves.toBe(0); + await expect( + listAuthorizationRecoveries(state, beforeGraceMs), + ).resolves.toEqual([entry]); + + const afterGraceMs = beforeGraceMs + 1; + await expect( + recoverAuthorizationCompletedAgentTurns({ + nowMs: afterGraceMs, + state, + }), + ).resolves.toBe(0); + await expect( + listAuthorizationRecoveries(state, afterGraceMs), + ).resolves.toEqual([]); + }); + + it("does not resume after mailbox failure wins the auth callback race", async () => { + const { + failAgentTurnSessionRecord, + prepareAgentTurnAuthorizationRecovery, + upsertAgentTurnSessionRecord, + } = await import("@/chat/state/turn-session"); + const { activateAndScheduleAgentTurnAuthorizationRecovery } = + await import("@/chat/services/agent-continue"); + const { acquireActiveLock } = await import("@/chat/state/locks"); + const { getStateAdapter } = await import("@/chat/state/adapter"); + const { getConversationEventStore } = await import("@/chat/db"); + const { ConversationTurnLifecycleService } = + await import("@/chat/conversations/turn-lifecycle"); + const { createConversationWorkQueueTestAdapter } = + await import("../../fixtures/conversation-work"); + const conversationId = "slack:C123:callback-mailbox-race"; + const sessionId = "turn-callback-mailbox-race"; + const parked = await upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + destination: SLACK_DESTINATION, + piMessages: [userMessage("connect")], + resumeReason: "auth", + sessionId, + sliceId: 2, + source: SLACK_SOURCE, + state: "awaiting_resume", + surface: "slack", + }); + const prepared = await prepareAgentTurnAuthorizationRecovery({ + authorizationCompletionId: "callback-receipt", + authorizationKind: "plugin", + conversationId, + expectedVersion: parked.version, + provider: "provider", + sessionId, + userId: "U123", + }); + if (!prepared) throw new Error("Expected prepared recovery"); + const lifecycle = new ConversationTurnLifecycleService( + getConversationEventStore(), + ); + await lifecycle.start({ + conversationId, + createdAtMs: 1, + inputMessageIds: ["message-callback-mailbox-race"], + surface: "slack", + turnId: sessionId, + }); + + const state = getStateAdapter(); + await state.connect(); + const mailboxLock = await acquireActiveLock(state, conversationId); + if (!mailboxLock) throw new Error("Expected mailbox lock"); + const queue = createConversationWorkQueueTestAdapter(); + const activation = activateAndScheduleAgentTurnAuthorizationRecovery( + { + authorizationCompletionId: "callback-receipt", + conversationId, + expectedVersion: prepared.version, + sessionId, + }, + { queue, state }, + ); + await lifecycle.fail({ + conversationId, + createdAtMs: 2, + failureCode: "agent_run_failed", + turnId: sessionId, + }); + await failAgentTurnSessionRecord({ + conversationId, + errorMessage: "Mailbox terminalized the parked turn", + expectedVersion: prepared.version, + sessionId, + }); + const { listAuthorizationRecoveries } = + await import("@/chat/state/authorization-recovery-index"); + await expect(listAuthorizationRecoveries(state)).resolves.toEqual([]); + await state.releaseLock(mailboxLock); + + await expect(activation).resolves.toBeUndefined(); + expect(queue.sentRecords()).toHaveLength(0); + }); + it("reads legacy requester summaries and skips invalid index entries", async () => { const { getStateAdapter } = await import("@/chat/state/adapter"); const { listBoundedAgentTurnSessionSummariesForConversation } = diff --git a/packages/junior/tests/component/task-execution/conversation-work.test.ts b/packages/junior/tests/component/task-execution/conversation-work.test.ts index 6c1abb8a6..e720a6840 100644 --- a/packages/junior/tests/component/task-execution/conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/conversation-work.test.ts @@ -84,6 +84,52 @@ function metadataEventsStore(events: string[]): ConversationStore { }; } +function controlConversationLeaseRenewal(baseState: StateAdapter): { + arm(): void; + attempts(): number; + release(): void; + started: Promise; + state: StateAdapter; +} { + const started = deferred(); + const release = deferred(); + let armed = false; + let attempts = 0; + const state = new Proxy(baseState, { + get(target, property, receiver) { + if (property === "acquireLock") { + return async ( + threadId: Parameters[0], + ttlMs: Parameters[1], + ) => { + if (armed && String(threadId).includes(":mutation")) { + attempts += 1; + if (attempts === 1) { + started.resolve(); + await release.promise; + } + } + return await target.acquireLock(threadId, ttlMs); + }; + } + const value = Reflect.get(target, property, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as StateAdapter; + return { + arm() { + armed = true; + }, + attempts: () => attempts, + release() { + armed = false; + release.resolve(); + }, + started: started.promise, + state, + }; +} + describe("conversation work execution", () => { const originalJuniorSecret = process.env.JUNIOR_SECRET; @@ -290,7 +336,6 @@ describe("conversation work execution", () => { const work = await getConversationWorkState({ conversationId: CONVERSATION_ID, - state, }); expect(work?.messages).toHaveLength(1); expect(queue.sentRecords()).toHaveLength(1); @@ -583,6 +628,30 @@ describe("conversation work execution", () => { await expect(first).resolves.toEqual({ status: "completed" }); }); + it("immediately requeues partial completion without incrementing attempts", async () => { + const queue = createConversationWorkQueueTestAdapter(); + await appendInboundMessage({ message: inboundMessage("m1"), nowMs: 1_000 }); + + await expect( + processConversationWork(conversationQueueMessage(), { + queue, + run: async () => ({ + status: "deferred", + immediate: true, + delayMs: 0, + }), + }), + ).resolves.toEqual({ status: "pending_requeued" }); + + expect(queue.sentRecords()).toMatchObject([ + { conversationId: CONVERSATION_ID, delayMs: 0 }, + ]); + const work = await getConversationWorkState({ + conversationId: CONVERSATION_ID, + }); + expect(work?.messages[0]?.attemptCount).toBeUndefined(); + }); + it("wakes fresh inbound work after a consumed deferred nudge left a recent marker", async () => { const queue = createConversationWorkQueueTestAdapter(); let currentNowMs = 1_000; @@ -1259,6 +1328,102 @@ describe("conversation work execution", () => { await expect(running).resolves.toEqual({ status: "completed" }); }); + it("keeps slow periodic lease renewal single-flight", async () => { + vi.useFakeTimers({ now: 1_000 }); + const queue = createConversationWorkQueueTestAdapter(); + const baseState = getStateAdapter(); + await baseState.connect(); + const renewal = controlConversationLeaseRenewal(baseState); + await appendInboundMessage({ + message: inboundMessage("m1"), + nowMs: 1_000, + state: renewal.state, + }); + const entered = deferred(); + const finish = deferred(); + + const running = processConversationWork(conversationQueueMessage(), { + checkInIntervalMs: 15_000, + queue, + run: async (context) => { + await context.attempt.drain(async () => {}); + renewal.arm(); + entered.resolve(); + await finish.promise; + return { status: "completed" }; + }, + state: renewal.state, + }); + + await entered.promise; + await vi.advanceTimersByTimeAsync(15_000); + await renewal.started; + await vi.advanceTimersByTimeAsync(30_000); + expect(renewal.attempts()).toBe(1); + + renewal.release(); + await vi.waitFor(async () => { + await expect( + getConversationWorkState({ + conversationId: CONVERSATION_ID, + state: renewal.state, + }), + ).resolves.toMatchObject({ + lease: { + lastCheckInAtMs: 16_000, + leaseExpiresAtMs: 16_000 + CONVERSATION_WORK_LEASE_TTL_MS, + }, + }); + }); + + finish.resolve(); + await expect(running).resolves.toEqual({ status: "completed" }); + }); + + it("joins an in-flight lease renewal before worker teardown", async () => { + vi.useFakeTimers({ now: 1_000 }); + const queue = createConversationWorkQueueTestAdapter(); + const baseState = getStateAdapter(); + await baseState.connect(); + const renewal = controlConversationLeaseRenewal(baseState); + await appendInboundMessage({ + message: inboundMessage("m1"), + nowMs: 1_000, + state: renewal.state, + }); + const entered = deferred(); + const finish = deferred(); + let settled = false; + + const running = processConversationWork(conversationQueueMessage(), { + checkInIntervalMs: 15_000, + queue, + run: async (context) => { + await context.attempt.drain(async () => {}); + renewal.arm(); + entered.resolve(); + await finish.promise; + return { status: "completed" }; + }, + state: renewal.state, + }); + void running.then(() => { + settled = true; + }); + + await entered.promise; + await vi.advanceTimersByTimeAsync(15_000); + await renewal.started; + finish.resolve(); + await Promise.resolve(); + expect(settled).toBe(false); + + renewal.release(); + await expect(running).resolves.toEqual({ status: "completed" }); + await vi.advanceTimersByTimeAsync(30_000); + expect(renewal.attempts()).toBe(1); + }); + it("reports lost lease after periodic check-in loses ownership", async () => { vi.useFakeTimers({ now: 1_000 }); const queue = createConversationWorkQueueTestAdapter(); diff --git a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts index 00689ed1a..1608b2cbb 100644 --- a/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts +++ b/packages/junior/tests/component/task-execution/slack-conversation-work.test.ts @@ -1674,7 +1674,11 @@ describe("Slack conversation work execution", () => { state, }); expect(work?.needsRun).toBe(true); - expect(work?.messages).toEqual([]); + // The executor keeps the original inbox record as its recovery trigger; + // Pi's input-commit callback no longer acknowledges queue ownership. + expect(work?.messages.map((entry) => entry.inboundMessageId)).toEqual([ + "slack:T123:slack:C123:1712345.0001:1712345.0001", + ]); const persistedState = await getPersistedThreadState(CONVERSATION_ID); const conversation = coerceThreadConversationState(persistedState); expect(conversation.processing.activeTurnId).toBe(yieldedSessionId); diff --git a/packages/junior/tests/fixtures/mcp-oauth-callback-harness.ts b/packages/junior/tests/fixtures/mcp-oauth-callback-harness.ts index bef31ff92..8f3d023cb 100644 --- a/packages/junior/tests/fixtures/mcp-oauth-callback-harness.ts +++ b/packages/junior/tests/fixtures/mcp-oauth-callback-harness.ts @@ -1,15 +1,23 @@ import type { AgentRunner } from "@/chat/runtime/agent-runner"; +import type { ScheduleAgentContinueOptions } from "@/chat/services/agent-continue"; +import { getConversationEventStore } from "@/chat/db"; +import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; import { waitUntilCallbacks, testWaitUntil, } from "./oauth-callback-after-harness"; import { realAgentRunner } from "./agent-runner"; +import { createConversationWorkQueueTestAdapter } from "./conversation-work"; +import type { RecoverableSlackDelivery } from "@/chat/slack/recoverable-delivery"; export async function runMcpOauthCallbackRoute(args: { provider: string; state: string; code: string; agentRunner?: AgentRunner; + beforeWaitUntilFlush?: () => Promise; + agentContinueOptions?: ScheduleAgentContinueOptions; + recoverableSlackDelivery?: RecoverableSlackDelivery; }) { waitUntilCallbacks.length = 0; const { GET } = await import("@/handlers/mcp-oauth-callback"); @@ -20,8 +28,22 @@ export async function runMcpOauthCallbackRoute(args: { ), args.provider, testWaitUntil, - { agentRunner: args.agentRunner ?? realAgentRunner }, + { + agentRunner: args.agentRunner ?? realAgentRunner, + ...(args.agentContinueOptions + ? { agentContinueOptions: args.agentContinueOptions } + : { + agentContinueOptions: { + queue: createConversationWorkQueueTestAdapter(), + }, + }), + turnLifecycle: new ConversationTurnLifecycleService( + getConversationEventStore(), + ), + recoverableSlackDelivery: args.recoverableSlackDelivery, + }, ); + await args.beforeWaitUntilFlush?.(); const callbacks = waitUntilCallbacks.splice(0, waitUntilCallbacks.length); for (const callback of callbacks) { await callback(); diff --git a/packages/junior/tests/fixtures/oauth-callback-harness.ts b/packages/junior/tests/fixtures/oauth-callback-harness.ts index 5ffcdd484..6ca53559d 100644 --- a/packages/junior/tests/fixtures/oauth-callback-harness.ts +++ b/packages/junior/tests/fixtures/oauth-callback-harness.ts @@ -1,15 +1,22 @@ import type { AgentRunner } from "@/chat/runtime/agent-runner"; +import type { ScheduleAgentContinueOptions } from "@/chat/services/agent-continue"; +import { getConversationEventStore } from "@/chat/db"; +import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; import { waitUntilCallbacks, testWaitUntil, } from "./oauth-callback-after-harness"; import { realAgentRunner } from "./agent-runner"; +import { createConversationWorkQueueTestAdapter } from "./conversation-work"; +import type { RecoverableSlackDelivery } from "@/chat/slack/recoverable-delivery"; export async function runOauthCallbackRoute(args: { provider: string; state: string; code: string; agentRunner?: AgentRunner; + agentContinueOptions?: ScheduleAgentContinueOptions; + recoverableSlackDelivery?: RecoverableSlackDelivery; }) { waitUntilCallbacks.length = 0; const { GET } = await import("@/handlers/oauth-callback"); @@ -20,7 +27,20 @@ export async function runOauthCallbackRoute(args: { ), args.provider, testWaitUntil, - { agentRunner: args.agentRunner ?? realAgentRunner }, + { + agentRunner: args.agentRunner ?? realAgentRunner, + ...(args.agentContinueOptions + ? { agentContinueOptions: args.agentContinueOptions } + : { + agentContinueOptions: { + queue: createConversationWorkQueueTestAdapter(), + }, + }), + turnLifecycle: new ConversationTurnLifecycleService( + getConversationEventStore(), + ), + recoverableSlackDelivery: args.recoverableSlackDelivery, + }, ); const callbacks = waitUntilCallbacks.splice(0, waitUntilCallbacks.length); for (const callback of callbacks) { diff --git a/packages/junior/tests/integration/agent-continue-slack.test.ts b/packages/junior/tests/integration/agent-continue-slack.test.ts index a17336506..31c025177 100644 --- a/packages/junior/tests/integration/agent-continue-slack.test.ts +++ b/packages/junior/tests/integration/agent-continue-slack.test.ts @@ -14,6 +14,17 @@ import type { AgentRunRequest } from "@/chat/agent/request"; import type { SandboxWorkspace } from "@/chat/sandbox/workspace"; import { createTools } from "@/chat/tools"; import type { ToolRuntimeContext } from "@/chat/tools/types"; +import type { ConversationTurnLifecycle } from "@/chat/conversations/turn-lifecycle"; +import type { ConversationEventStore } from "@/chat/conversations/history"; +import { getConversationMessageStore, getSqlExecutor } from "@/chat/db"; +import { + RecoverableSlackDeliveryService, + type RecoverableSlackDelivery, +} from "@/chat/slack/recoverable-delivery"; +import { + postRecoverableSlackMessage, + reconcileRecoverableSlackMessage, +} from "@/chat/slack/outbound"; const executeAgentRunMock = vi.fn(); @@ -113,9 +124,24 @@ let turnSessionStoreModule: TurnSessionStoreModule; let agentContinueServiceModule: AgentContinueServiceModule; let taskExecutionStoreModule: TaskExecutionStoreModule; let queue: ConversationWorkQueueTestAdapter; +let turnLifecycle: ConversationTurnLifecycle; +let conversationEventStore: ConversationEventStore; +let recoverableSlackDelivery: RecoverableSlackDeliveryService; + +async function loadTurnLifecycle(conversationId: string, turnId: string) { + return (await conversationEventStore.loadHistory(conversationId)) + .filter( + (event) => + event.data.type.startsWith("turn_") && + "turnId" in event.data && + event.data.turnId === turnId, + ) + .map((event) => event.data); +} function continueAgentRun(args: { conversationId: string; + delivery?: RecoverableSlackDelivery; sessionId: string; expectedVersion: number; }): Promise { @@ -129,6 +155,8 @@ function continueAgentRun(args: { }, { agentRunner: { run: executeAgentRunMock }, + recoverableSlackDelivery: args.delivery ?? recoverableSlackDelivery, + turnLifecycle, scheduleAgentContinue: (request) => agentContinueServiceModule.scheduleAgentContinue(request, { queue, @@ -166,6 +194,20 @@ describe("agent continuation Slack integration", () => { turnSessionStoreModule = await import("@/chat/state/turn-session"); agentContinueServiceModule = await import("@/chat/services/agent-continue"); taskExecutionStoreModule = await import("@/chat/task-execution/store"); + const { ConversationTurnLifecycleService } = + await import("@/chat/conversations/turn-lifecycle"); + const { getConversationEventStore } = await import("@/chat/db"); + conversationEventStore = getConversationEventStore(); + turnLifecycle = new ConversationTurnLifecycleService( + conversationEventStore, + ); + recoverableSlackDelivery = new RecoverableSlackDeliveryService( + getSqlExecutor(), + { + post: postRecoverableSlackMessage, + reconcile: reconcileRecoverableSlackMessage, + }, + ); await stateAdapterModule.disconnectStateAdapter(); await stateAdapterModule.getStateAdapter().connect(); @@ -354,22 +396,17 @@ describe("agent continuation Slack integration", () => { role: "assistant", text: "Final resumed answer", }); - const { getConversationEventStore } = await import("@/chat/db"); const lifecycle = ( - await getConversationEventStore().loadHistory(conversationId) + await conversationEventStore.loadHistory(conversationId) ).filter((event) => event.data.type.startsWith("turn_")); expect(lifecycle.map((event) => event.data)).toEqual([ - expect.objectContaining({ + { type: "turn_started", turnId: sessionId, inputMessageIds: ["msg.1"], surface: "slack", - }), - expect.objectContaining({ - type: "turn_completed", - turnId: sessionId, - outcome: "success", - }), + }, + { type: "turn_completed", turnId: sessionId, outcome: "success" }, ]); }); @@ -435,7 +472,6 @@ describe("agent continuation Slack integration", () => { }, }, }); - const continued = await continueAgentRun({ conversationId, sessionId, @@ -797,7 +833,13 @@ describe("agent continuation Slack integration", () => { }, }, }); - + await turnLifecycle.start({ + conversationId, + turnId: sessionId, + inputMessageIds: ["msg.7"], + createdAtMs: 1, + surface: "slack", + }); const continued = await continueAgentRun({ conversationId, sessionId, @@ -815,15 +857,124 @@ describe("agent continuation Slack integration", () => { state: "failed", errorMessage: "Paused agent run failed while continuing", }); - const { getConversationEventStore } = await import("@/chat/db"); - const lifecycle = - await getConversationEventStore().loadHistory(conversationId); - expect(lifecycle.at(-1)?.data).toMatchObject({ - type: "turn_failed", + const lifecycle = ( + await conversationEventStore.loadHistory(conversationId) + ).filter((event) => event.data.type.startsWith("turn_")); + expect(lifecycle.map((event) => event.data.type)).toEqual([ + "turn_started", + "turn_failed", + ]); + }); + + it("fails an unmaterializable continuation once in canonical history", async () => { + const conversationId = "slack:C123:1712345.00071"; + const sessionId = "turn_msg_71"; + await turnSessionStoreModule.upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + sessionId, + sliceId: 1, + state: "awaiting_resume", + destination: SLACK_DESTINATION, + resumeReason: "timeout", + piMessages: [], + }); + await turnLifecycle.start({ + conversationId, turnId: sessionId, - failureCode: "agent_run_failed", - eventId: expect.stringMatching(/^[a-f0-9]{32}$/i), + inputMessageIds: ["msg.71"], + createdAtMs: 1, + surface: "slack", }); + + const resume = () => + agentContinueRunnerModule.resumeAwaitingSlackContinuation( + conversationId, + { + agentRunner: { run: executeAgentRunMock }, + turnLifecycle, + }, + ); + await expect(resume()).resolves.toBe(false); + await expect(resume()).resolves.toBe(false); + + expect(await loadTurnLifecycle(conversationId, sessionId)).toEqual([ + expect.objectContaining({ type: "turn_started" }), + expect.objectContaining({ + type: "turn_failed", + failureCode: "persistence_failed", + }), + ]); + }); + + it("does not fail a continuation whose session completed concurrently", async () => { + const conversationId = "slack:C123:1712345.00072"; + const sessionId = "turn_msg_72"; + await turnSessionStoreModule.upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + sessionId, + sliceId: 2, + state: "awaiting_resume", + destination: SLACK_DESTINATION, + source: slackSource("1712345.00072"), + resumeReason: "timeout", + piMessages: [ + { + role: "user", + content: [{ type: "text", text: "hello" }], + timestamp: 1, + }, + ], + }); + await turnLifecycle.start({ + conversationId, + turnId: sessionId, + inputMessageIds: ["msg.72"], + createdAtMs: 1, + surface: "slack", + }); + + await expect( + agentContinueRunnerModule.resumeAwaitingSlackContinuation( + conversationId, + { + agentRunner: { run: executeAgentRunMock }, + turnLifecycle, + resumeTurn: async () => { + await turnSessionStoreModule.upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + sessionId, + sliceId: 2, + state: "completed", + destination: SLACK_DESTINATION, + source: slackSource("1712345.00072"), + piMessages: [ + { + role: "user", + content: [{ type: "text", text: "hello" }], + timestamp: 1, + }, + ], + }); + return false; + }, + }, + ), + ).resolves.toBe(false); + await turnLifecycle.complete({ + conversationId, + turnId: sessionId, + createdAtMs: 2, + outcome: "success", + }); + + expect( + (await loadTurnLifecycle(conversationId, sessionId)).map( + (event) => event.type, + ), + ).toEqual(["turn_started", "turn_completed"]); }); it("resumes resource-event turns with the stored system actor", async () => { @@ -887,7 +1038,6 @@ describe("agent continuation Slack integration", () => { }, }, }); - const continued = await continueAgentRun({ conversationId, sessionId, @@ -974,15 +1124,65 @@ describe("agent continuation Slack integration", () => { }, }, }); + await getConversationMessageStore().record(conversationId, [ + { + messageId: "msg.10", + role: "user", + text: "resume this request", + createdAtMs: 1, + }, + ]); + + await turnLifecycle.start({ + conversationId, + turnId: sessionId, + inputMessageIds: ["msg.10"], + createdAtMs: 1, + surface: "slack", + }); + let nowMs = 1_000; + const post = vi + .fn() + .mockResolvedValueOnce({ + outcome: "retryable_absence" as const, + retryAtMs: 1_001, + }) + .mockResolvedValueOnce({ + outcome: "accepted" as const, + ts: "1712345.001001" as never, + }); + const delivery = new RecoverableSlackDeliveryService( + getSqlExecutor(), + { post, reconcile: vi.fn() }, + () => nowMs, + ); const continued = await continueAgentRun({ conversationId, + delivery, sessionId, expectedVersion: sessionRecord.version, }); expect(continued).toBe(false); expect(executeAgentRunMock).not.toHaveBeenCalled(); + await expect( + turnSessionStoreModule.getAgentTurnSessionRecord( + conversationId, + sessionId, + ), + ).resolves.toMatchObject({ state: "awaiting_resume" }); + expect(await loadTurnLifecycle(conversationId, sessionId)).toEqual([ + expect.objectContaining({ type: "turn_started" }), + ]); + + nowMs = 1_002; + const { recoverDueSlackDeliveries } = + await import("@/chat/runtime/slack-delivery-recovery"); + await expect(recoverDueSlackDeliveries({ delivery, nowMs })).resolves.toBe( + 1, + ); + expect(post).toHaveBeenCalledTimes(2); await expect( turnSessionStoreModule.getAgentTurnSessionRecord( conversationId, @@ -990,17 +1190,13 @@ describe("agent continuation Slack integration", () => { ), ).resolves.toMatchObject({ state: "failed", - errorMessage: "Stored Slack actor missing for continuation", + errorMessage: "Delivered terminal failure reply", }); - expect(slackApiOutbox.messages()).toEqual([ + expect(await loadTurnLifecycle(conversationId, sessionId)).toEqual([ + expect.objectContaining({ type: "turn_started" }), expect.objectContaining({ - params: expect.objectContaining({ - channel: "C123", - thread_ts: "1712345.0010", - text: expect.stringContaining( - "I ran into an internal error while processing that.", - ), - }), + type: "turn_failed", + failureCode: "persistence_failed", }), ]); }); @@ -1077,7 +1273,6 @@ describe("agent continuation Slack integration", () => { }, }, }); - const continued = await continueAgentRun({ conversationId, sessionId, @@ -1283,12 +1478,12 @@ describe("agent continuation Slack integration", () => { }, }, }); - const resumed = await requestDeadlineModule.runWithTurnRequestDeadline(() => agentContinueRunnerModule.resumeAwaitingSlackContinuation( conversationId, { agentRunner: { run: executeAgentRunMock }, + turnLifecycle, scheduleAgentContinue: (request) => agentContinueServiceModule.scheduleAgentContinue(request, { queue, @@ -1454,6 +1649,7 @@ describe("agent continuation Slack integration", () => { conversationId, { agentRunner: { run: executeAgentRunMock }, + turnLifecycle, }, ), ); @@ -1578,12 +1774,22 @@ describe("agent continuation Slack integration", () => { }, }, }); + await getConversationMessageStore().record(conversationId, [ + { + messageId: "msg.9", + role: "user", + text: "resume this request", + createdAtMs: 1, + }, + ]); const resumed = await requestDeadlineModule.runWithTurnRequestDeadline(() => agentContinueRunnerModule.resumeAwaitingSlackContinuation( conversationId, { agentRunner: { run: executeAgentRunMock }, + recoverableSlackDelivery, + turnLifecycle, }, ), ); @@ -1597,7 +1803,7 @@ describe("agent continuation Slack integration", () => { ), ).resolves.toMatchObject({ state: "failed", - errorMessage: expect.stringContaining("no resumable boundary"), + errorMessage: "Delivered terminal failure reply", }); expect(slackApiOutbox.messages()).toEqual([ expect.objectContaining({ diff --git a/packages/junior/tests/integration/agent-dispatch-runner.test.ts b/packages/junior/tests/integration/agent-dispatch-runner.test.ts index bc979ee08..bee792aa3 100644 --- a/packages/junior/tests/integration/agent-dispatch-runner.test.ts +++ b/packages/junior/tests/integration/agent-dispatch-runner.test.ts @@ -10,8 +10,15 @@ import { updateDispatchRecord, withDispatchLock, } from "@/chat/agent-dispatch/store"; -import { runAgentDispatchSlice } from "@/chat/agent-dispatch/runner"; -import { getConversationEventStore, getConversationStore } from "@/chat/db"; +import { + processAgentDispatchCallback as processAgentDispatchCallbackImpl, + type AgentDispatchRunnerDeps, +} from "@/chat/agent-dispatch/runner"; +import { + getConversationEventStore, + getConversationStore, + getSqlExecutor, +} from "@/chat/db"; import { getPersistedThreadState } from "@/chat/runtime/thread-state"; import { coerceThreadConversationState } from "@/chat/state/conversation"; import { @@ -34,8 +41,39 @@ import { chatPostMessageOk } from "../fixtures/slack/factories/api"; import { getCapturedSlackApiCalls, queueSlackApiResponse, + queueSlackRateLimit, } from "../msw/handlers/slack-api"; import { flattenAgentRunRequestForTest } from "../fixtures/agent-runner"; +import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; +import { RecoverableSlackDeliveryService } from "@/chat/slack/recoverable-delivery"; +import { + postRecoverableSlackMessage, + reconcileRecoverableSlackMessage, +} from "@/chat/slack/outbound"; +import { recoverStaleDispatches } from "@/chat/agent-dispatch/heartbeat"; +import { deferred } from "../fixtures/conversation-work"; + +async function processAgentDispatchCallback( + callback: Parameters[0], + deps: Omit< + AgentDispatchRunnerDeps, + "recoverableSlackDelivery" | "turnLifecycle" + >, +): Promise { + await processAgentDispatchCallbackImpl(callback, { + ...deps, + recoverableSlackDelivery: new RecoverableSlackDeliveryService( + getSqlExecutor(), + { + post: postRecoverableSlackMessage, + reconcile: reconcileRecoverableSlackMessage, + }, + ), + turnLifecycle: new ConversationTurnLifecycleService( + getConversationEventStore(), + ), + }); +} vi.hoisted(() => { process.env.JUNIOR_STATE_ADAPTER = "memory"; @@ -171,12 +209,14 @@ function slackSource(channelId = "C123") { describe("agent dispatch runner", () => { beforeEach(async () => { + process.env.JUNIOR_BASE_URL = "https://junior.example.com"; process.env.JUNIOR_SECRET = "dispatch-runner-secret"; await disconnectStateAdapter(); }); afterEach(async () => { await disconnectStateAdapter(); + delete process.env.JUNIOR_BASE_URL; delete process.env.JUNIOR_SECRET; }); @@ -229,7 +269,7 @@ describe("agent dispatch runner", () => { }); const scheduleSessionCompletedPluginTasks = vi.fn(async () => undefined); - await runAgentDispatchSlice( + await processAgentDispatchCallback( { id: created.record.id, expectedVersion: created.record.version, @@ -271,10 +311,9 @@ describe("agent dispatch runner", () => { }), }), expect.objectContaining({ - id: `dispatch:${created.record.id}:assistant`, + id: `assistant:dispatch:${created.record.id}`, meta: expect.objectContaining({ slackTs: "1700000000.000001", - replied: true, }), }), ]), @@ -308,18 +347,154 @@ describe("agent dispatch runner", () => { await getConversationEventStore().loadHistory(dispatchConversationId) ).filter((event) => event.data.type.startsWith("turn_")); expect(lifecycle.map((event) => event.data)).toEqual([ - expect.objectContaining({ + { type: "turn_started", turnId: `dispatch:${created.record.id}`, inputMessageIds: [`dispatch:${created.record.id}:user`], surface: "api", - }), - expect.objectContaining({ + }, + { type: "turn_completed", turnId: `dispatch:${created.record.id}`, outcome: "success", + }, + ]); + }); + + it.each([120, 500])( + "uses the configured %ds host window for the agent deadline and lease", + async (functionMaxDurationSeconds) => { + const created = await createOrGetDispatch({ + plugin: "scheduler", + nowMs: Date.now(), + options: { + idempotencyKey: `configured-function-lease-${functionMaxDurationSeconds}`, + destination: slackAddress(), + destinationVisibility: "private", + input: "Run the scheduled task.", + source: slackSource(), + }, + }); + const entered = deferred(); + const finish = deferred(); + let turnDeadlineAtMs: number | undefined; + let turnTimeoutMs: number | undefined; + const running = processAgentDispatchCallback( + { + id: created.record.id, + expectedVersion: created.record.version, + }, + { + agentRunner: { + run: async (request) => { + turnDeadlineAtMs = request.policy?.turnDeadlineAtMs; + turnTimeoutMs = request.policy?.turnTimeoutMs; + entered.resolve(); + await finish.promise; + return completedAgentRun(createReply()); + }, + }, + functionMaxDurationSeconds, + }, + ); + await entered.promise; + + const active = await getDispatchRecord(created.record.id); + expect(active).toMatchObject({ + lastCallbackAtMs: expect.any(Number), + leaseExpiresAtMs: expect.any(Number), + status: "running", + }); + const startedAtMs = active!.lastCallbackAtMs!; + expect(turnTimeoutMs).toBe((functionMaxDurationSeconds - 20) * 1000); + expect(turnDeadlineAtMs).toBe( + startedAtMs + (functionMaxDurationSeconds - 20) * 1000, + ); + expect(active!.leaseExpiresAtMs).toBe( + startedAtMs + (functionMaxDurationSeconds + 20) * 1000, + ); + + const originalFetch = global.fetch; + const fetchMock = vi.fn( + async (..._args: Parameters) => + new Response("Accepted", { status: 202 }), + ); + global.fetch = fetchMock as typeof fetch; + try { + await expect( + recoverStaleDispatches({ nowMs: active!.leaseExpiresAtMs! - 1 }), + ).resolves.toBe(0); + await expect( + recoverStaleDispatches({ nowMs: active!.leaseExpiresAtMs! }), + ).resolves.toBe(1); + expect(fetchMock).toHaveBeenCalledTimes(1); + } finally { + global.fetch = originalFetch; + finish.resolve(); + await running; + } + }, + ); + + it("retries pending delivery without consuming another model attempt", async () => { + queueSlackRateLimit("chat.postMessage", 0); + queueSlackApiResponse("chat.postMessage", { + body: chatPostMessageOk({ + channel: "C123", + ts: "1700000000.000009", }), + }); + const created = await createOrGetDispatch({ + plugin: "scheduler", + nowMs: Date.parse("2026-05-26T12:00:00.000Z"), + options: { + idempotencyKey: "run-delivery-retry", + destination: slackAddress(), + destinationVisibility: "private", + input: "Run the scheduled task.", + source: slackSource(), + }, + }); + const executeAgentRun = vi.fn(async () => completedAgentRun(createReply())); + const scheduledCallbacks: Array<{ + expectedVersion: number; + id: string; + kind?: "delivery"; + }> = []; + const scheduleCallback = vi.fn(async (callback) => { + scheduledCallbacks.push(callback); + }); + + await processAgentDispatchCallback( + { id: created.record.id, expectedVersion: created.record.version }, + { agentRunner: { run: executeAgentRun }, scheduleCallback }, + ); + + await expect(getDispatchRecord(created.record.id)).resolves.toMatchObject({ + attempt: 1, + status: "awaiting_resume", + nextCallbackKind: "delivery", + }); + expect(scheduledCallbacks).toEqual([ + { + id: created.record.id, + expectedVersion: expect.any(Number), + kind: "delivery", + }, ]); + + await processAgentDispatchCallback(scheduledCallbacks[0]!, { + agentRunner: { run: executeAgentRun }, + scheduleCallback, + }); + + await expect(getDispatchRecord(created.record.id)).resolves.toMatchObject({ + attempt: 1, + status: "completed", + resultMessageTs: "1700000000.000009", + }); + expect(executeAgentRun).toHaveBeenCalledTimes(1); + expect(getCapturedSlackApiCalls("chat.postMessage")).toHaveLength(2); }); it("starts dispatches without inherited destination conversation memory", async () => { @@ -361,7 +536,7 @@ describe("agent dispatch runner", () => { return completedAgentRun(createReply()); }); - await runAgentDispatchSlice( + await processAgentDispatchCallback( { id: created.record.id, expectedVersion: created.record.version, @@ -392,7 +567,7 @@ describe("agent dispatch runner", () => { id: `dispatch:${created.record.id}:user`, }), expect.objectContaining({ - id: `dispatch:${created.record.id}:assistant`, + id: `assistant:dispatch:${created.record.id}`, }), ]), ); @@ -413,7 +588,7 @@ describe("agent dispatch runner", () => { const dispatchConversationId = getDispatchConversationId(created.record); const sideEffectReply = createReply(); - await runAgentDispatchSlice( + await processAgentDispatchCallback( { id: created.record.id, expectedVersion: created.record.version, @@ -497,7 +672,7 @@ describe("agent dispatch runner", () => { return completedAgentRun(createReply()); }); - await runAgentDispatchSlice( + await processAgentDispatchCallback( { id: created.record.id, expectedVersion: created.record.version, @@ -519,7 +694,7 @@ describe("agent dispatch runner", () => { ts: "1700000000.000001", }), }); - await runAgentDispatchSlice( + await processAgentDispatchCallback( { id: created.record.id, expectedVersion: awaitingResume!.version, @@ -571,7 +746,7 @@ describe("agent dispatch runner", () => { return completedAgentRun(createReply()); }); - await runAgentDispatchSlice( + await processAgentDispatchCallback( { id: created.record.id, expectedVersion: created.record.version, @@ -625,7 +800,7 @@ describe("agent dispatch runner", () => { return completedAgentRun(createReply()); }); - await runAgentDispatchSlice( + await processAgentDispatchCallback( { id: created.record.id, expectedVersion: created.record.version, @@ -657,6 +832,7 @@ describe("agent dispatch runner", () => { source: slackSource(), }, }); + const dispatchConversationId = getDispatchConversationId(created.record); const state = getStateAdapter(); await state.connect(); const originalSet = state.set.bind(state); @@ -668,43 +844,180 @@ describe("agent dispatch runner", () => { } return originalSet(key, value, ttlMs); }); + const scheduleSessionCompletedPluginTasks = vi.fn(async () => undefined); try { - await runAgentDispatchSlice( + await processAgentDispatchCallback( { id: created.record.id, expectedVersion: created.record.version, }, { agentRunner: { run: async () => completedAgentRun(createReply()) }, + scheduleSessionCompletedPluginTasks, }, ); } finally { setSpy.mockRestore(); } - // Delivery already happened: the dispatch is terminal so a retry cannot - // re-post, and the persistence failure is logged instead of failing it. - await expect(getDispatchRecord(created.record.id)).resolves.toMatchObject({ - status: "completed", - resultMessageTs: "1700000000.000004", + // Slack accepted the write, but the durable intent remains until all + // derived state is repaired. A delivery callback retries no model work. + const awaitingRepair = await getDispatchRecord(created.record.id); + expect(awaitingRepair).toMatchObject({ + attempt: 1, + status: "awaiting_resume", + nextCallbackKind: "delivery", }); + expect( + ( + await getConversationEventStore().loadHistory(dispatchConversationId) + ).filter((event) => event.data.type.startsWith("turn_")), + ).toEqual([ + expect.objectContaining({ + data: expect.objectContaining({ type: "turn_started" }), + }), + ]); const rerunGenerate = vi.fn(async () => { throw new Error("must not regenerate a delivered dispatch"); }); - await runAgentDispatchSlice( + await processAgentDispatchCallback( { id: created.record.id, - expectedVersion: created.record.version, + expectedVersion: awaitingRepair!.version, + kind: "delivery", }, { agentRunner: { run: rerunGenerate } }, ); expect(rerunGenerate).not.toHaveBeenCalled(); expect(getCapturedSlackApiCalls("chat.postMessage")).toHaveLength(1); + await expect(getDispatchRecord(created.record.id)).resolves.toMatchObject({ + attempt: 1, + status: "completed", + resultMessageTs: "1700000000.000004", + }); + const lifecycle = ( + await getConversationEventStore().loadHistory(dispatchConversationId) + ).filter((event) => event.data.type.startsWith("turn_")); + expect(lifecycle.at(-1)?.data).toMatchObject({ + type: "turn_completed", + turnId: `dispatch:${created.record.id}`, + outcome: "success", + }); + }); + + it("recovers an aged side-effect completion before a normal retry", async () => { + const created = await createOrGetDispatch({ + plugin: "scheduler", + nowMs: Date.now() - 25 * 60 * 60 * 1000, + options: { + idempotencyKey: "run-terminal-projection-write-fail", + destination: slackAddress(), + destinationVisibility: "private", + input: "Run the scheduled task.", + source: slackSource(), + }, + }); + const state = getStateAdapter(); + await state.connect(); + const ready = await getDispatchRecord(created.record.id); + if (!ready) throw new Error("Expected dispatch record to exist"); + const originalSet = state.set.bind(state); + let injectedFailures = 0; + const setSpy = vi + .spyOn(state, "set") + .mockImplementation(async (key, value, ttlMs) => { + const dispatch = parseDispatchRecord(value); + if ( + String(key) === getDispatchStorageKey(created.record.id) && + dispatch?.status === "completed" + ) { + injectedFailures += 1; + throw new Error("dispatch state store unavailable"); + } + return originalSet(key, value, ttlMs); + }); + const sideEffectReply = createReply(); + sideEffectReply.deliveryPlan = { + mode: "channel_only", + postThreadText: false, + }; + + try { + await processAgentDispatchCallback( + { + id: created.record.id, + expectedVersion: ready.version, + }, + { + agentRunner: { + run: async () => completedAgentRun(sideEffectReply), + }, + }, + ); + } finally { + setSpy.mockRestore(); + } + + expect(injectedFailures).toBe(1); + expect(getCapturedSlackApiCalls("chat.postMessage")).toHaveLength(0); + await expect(getDispatchRecord(created.record.id)).resolves.toMatchObject({ + attempt: 1, + maxAttempts: 5, + status: "running", + }); + + const originalFetch = global.fetch; + const fetchMock = vi.fn( + async (..._args: Parameters) => + new Response("Accepted", { status: 202 }), + ); + global.fetch = fetchMock as typeof fetch; + const delivery = new RecoverableSlackDeliveryService(getSqlExecutor(), { + post: postRecoverableSlackMessage, + reconcile: reconcileRecoverableSlackMessage, + }); + try { + await expect( + recoverStaleDispatches({ + nowMs: Date.now() + 10 * 60 * 1000, + recoverableSlackDelivery: delivery, + }), + ).resolves.toBe(1); + } finally { + global.fetch = originalFetch; + } + const callbackRequest = fetchMock.mock.calls.at(-1); + const callback = JSON.parse(String(callbackRequest?.[1]?.body)) as { + id: string; + expectedVersion: number; + kind: "delivery"; + }; + expect(callback).toMatchObject({ id: created.record.id, kind: "delivery" }); + const rerun = vi.fn(async () => completedAgentRun(sideEffectReply)); + await processAgentDispatchCallback(callback, { + agentRunner: { run: rerun }, + }); + + expect(rerun).not.toHaveBeenCalled(); + await expect(getDispatchRecord(created.record.id)).resolves.toMatchObject({ + attempt: 1, + status: "completed", + }); + const lifecycle = ( + await getConversationEventStore().loadHistory( + getDispatchConversationId(created.record), + ) + ).filter((event) => event.data.type.startsWith("turn_")); + expect(lifecycle.at(-1)?.data).toMatchObject({ + type: "turn_completed", + turnId: `dispatch:${created.record.id}`, + outcome: "no_reply", + }); }); - it("completes the session record after delivering a failed dispatch fallback", async () => { + it("fails the session record after delivering a failed dispatch fallback", async () => { queueSlackApiResponse("chat.postMessage", { body: chatPostMessageOk({ channel: "C123", @@ -738,7 +1051,7 @@ describe("agent dispatch runner", () => { }), ); - await runAgentDispatchSlice( + await processAgentDispatchCallback( { id: created.record.id, expectedVersion: created.record.version, @@ -758,16 +1071,18 @@ describe("agent dispatch runner", () => { ).resolves.toMatchObject({ conversationId: dispatchConversationId, sessionId: `dispatch:${created.record.id}`, - state: "completed", + state: "failed", surface: "api", }); const lifecycle = ( await getConversationEventStore().loadHistory(dispatchConversationId) ).filter((event) => event.data.type.startsWith("turn_")); + expect(lifecycle).toHaveLength(2); expect(lifecycle.at(-1)?.data).toMatchObject({ type: "turn_failed", turnId: `dispatch:${created.record.id}`, failureCode: "model_execution_failed", + eventId: expect.any(String), }); }); @@ -789,7 +1104,7 @@ describe("agent dispatch runner", () => { source: slackSource(), }, }); - await runAgentDispatchSlice( + await processAgentDispatchCallback( { id: created.record.id, expectedVersion: created.record.version, @@ -825,7 +1140,7 @@ describe("agent dispatch runner", () => { const rerunGenerate = vi.fn(async () => { throw new Error("must not regenerate a delivered dispatch"); }); - await runAgentDispatchSlice( + await processAgentDispatchCallback( { id: created.record.id, expectedVersion: reverted.version, @@ -841,6 +1156,78 @@ describe("agent dispatch runner", () => { }); }); + it("records a persistence failure when a delivered marker has no lifecycle terminal", async () => { + const created = await createOrGetDispatch({ + plugin: "scheduler", + nowMs: Date.parse("2026-05-26T12:00:00.000Z"), + options: { + idempotencyKey: "run-delivered-without-terminal", + destination: slackAddress(), + destinationVisibility: "private", + input: "Run the scheduled task.", + source: slackSource(), + }, + }); + const conversationId = getDispatchConversationId(created.record); + const conversation = coerceThreadConversationState({}); + conversation.messages.push( + { + id: `dispatch:${created.record.id}:user`, + role: "user", + text: "Run the scheduled task.", + createdAtMs: Date.parse("2026-05-26T12:00:00.000Z"), + author: { userName: "system:scheduler", isBot: true }, + meta: { replied: true }, + }, + { + id: `assistant:dispatch:${created.record.id}`, + role: "assistant", + text: "A delivered fallback whose model outcome is unknown.", + createdAtMs: Date.parse("2026-05-26T12:00:01.000Z"), + author: { userName: "junior", isBot: true }, + meta: { + replied: true, + slackTs: "1700000000.000007", + }, + }, + ); + await persistConversationMessages({ conversation, conversationId }); + + const rerunGenerate = vi.fn(async () => { + throw new Error("must not regenerate a delivered dispatch"); + }); + await processAgentDispatchCallback( + { + id: created.record.id, + expectedVersion: created.record.version, + }, + { agentRunner: { run: rerunGenerate } }, + ); + + expect(rerunGenerate).not.toHaveBeenCalled(); + expect(getCapturedSlackApiCalls("chat.postMessage")).toHaveLength(0); + await expect(getDispatchRecord(created.record.id)).resolves.toMatchObject({ + status: "completed", + resultMessageTs: "1700000000.000007", + }); + const lifecycle = ( + await getConversationEventStore().loadHistory(conversationId) + ).filter((event) => event.data.type.startsWith("turn_")); + expect(lifecycle.map((event) => event.data)).toEqual([ + { + type: "turn_started", + turnId: `dispatch:${created.record.id}`, + inputMessageIds: [`dispatch:${created.record.id}:user`], + surface: "api", + }, + { + type: "turn_failed", + turnId: `dispatch:${created.record.id}`, + failureCode: "persistence_failed", + }, + ]); + }); + it("does not burn an attempt when the destination conversation is busy", async () => { const created = await createOrGetDispatch({ plugin: "scheduler", @@ -862,7 +1249,7 @@ describe("agent dispatch runner", () => { expect(lock).toBeTruthy(); try { - await runAgentDispatchSlice( + await processAgentDispatchCallback( { id: created.record.id, expectedVersion: created.record.version, diff --git a/packages/junior/tests/integration/heartbeat.test.ts b/packages/junior/tests/integration/heartbeat.test.ts index 739e05abe..a125b01bd 100644 --- a/packages/junior/tests/integration/heartbeat.test.ts +++ b/packages/junior/tests/integration/heartbeat.test.ts @@ -684,6 +684,53 @@ describe("plugin heartbeat", () => { }); }); + it("schedules due delivery recovery independently from model attempts and leases", async () => { + const fetchMock = mockDispatchCallbackFetch(originalFetch); + const created = await createOrGetDispatch({ + plugin: "scheduler", + nowMs: Date.parse("2026-05-26T12:00:00.000Z"), + options: { + idempotencyKey: "run-delivery-recovery", + destination: SLACK_DESTINATION, + destinationVisibility: "private", + input: "Run the scheduled task.", + source: SLACK_SOURCE, + }, + }); + await withDispatchLock(created.record.id, async (state) => { + const record = await state.get( + getDispatchStorageKey(created.record.id), + ); + if (!record) throw new Error("Expected dispatch record to exist"); + await updateDispatchRecord(state, { + ...record, + attempt: record.maxAttempts, + leaseExpiresAtMs: Date.parse("2026-05-26T13:00:00.000Z"), + nextCallbackAtMs: Date.parse("2026-05-26T12:04:00.000Z"), + nextCallbackKind: "delivery", + status: "awaiting_resume", + }); + }); + + await expect( + recoverStaleDispatches({ + nowMs: Date.parse("2026-05-26T12:05:00.000Z"), + }), + ).resolves.toBe(1); + await expect(getDispatchRecord(created.record.id)).resolves.toMatchObject({ + attempt: created.record.maxAttempts, + status: "awaiting_resume", + nextCallbackKind: "delivery", + }); + const callbackRequest = fetchMock.mock.calls.find(([input]) => + String(input).includes("/api/internal/agent-dispatch"), + ); + expect(JSON.parse(String(callbackRequest?.[1]?.body))).toMatchObject({ + id: created.record.id, + kind: "delivery", + }); + }); + it("fails stale dispatches when the locked row no longer parses", async () => { const created = await createOrGetDispatch({ plugin: "scheduler", diff --git a/packages/junior/tests/integration/mcp-oauth-callback-slack.test.ts b/packages/junior/tests/integration/mcp-oauth-callback-slack.test.ts index dfff57291..de9945ac0 100644 --- a/packages/junior/tests/integration/mcp-oauth-callback-slack.test.ts +++ b/packages/junior/tests/integration/mcp-oauth-callback-slack.test.ts @@ -8,6 +8,7 @@ import { import { EVAL_MCP_AUTH_CODE, EVAL_MCP_AUTH_PROVIDER, + readEvalMcpAuthorizationCodeExchanges, } from "../msw/handlers/eval-mcp-auth"; import { getCapturedSlackApiCalls, @@ -17,6 +18,7 @@ import { createPluginAppFixture, type PluginAppFixture, } from "../fixtures/plugin-app"; +import { createConversationWorkQueueTestAdapter } from "../fixtures/conversation-work"; import { completedAgentRun } from "@/chat/runtime/agent-run-outcome"; import { hydrateConversationMessages, @@ -560,6 +562,120 @@ describe("mcp oauth callback slack integration", () => { ); }); + it("does not consume the MCP code while conversation ownership is busy", async () => { + const threadId = "slack:C123:1700000000.014"; + const sessionId = "turn_user-14"; + const source = slackSource("1700000000.014"); + await stateAdapterModule.getStateAdapter().set(`thread-state:${threadId}`, { + conversation: { + messages: [ + { + id: "user-14", + role: "user", + text: "what did i say about the budget?", + createdAtMs: 1, + author: { userId: "U123", userName: "dcramer" }, + meta: { slackTs: "1700000000.0141" }, + }, + ], + processing: { + pendingAuth: { + kind: "mcp", + provider: EVAL_MCP_AUTH_PROVIDER, + actorId: "U123", + sessionId, + linkSentAtMs: 1, + }, + }, + }, + }); + await seedVisibleTranscriptFromThreadState( + stateAdapterModule.getStateAdapter(), + threadId, + ); + await createAwaitingMcpTurnRecord({ + conversationId: threadId, + sessionId, + source, + text: "what did i say about the budget?", + threadTs: "1700000000.014", + }); + const authProvider = await createPendingAuthSession({ + conversationId: threadId, + sessionId, + userMessage: "what did i say about the budget?", + channelId: "C123", + threadTs: "1700000000.014", + }); + + const adapter = stateAdapterModule.getStateAdapter(); + const queue = createConversationWorkQueueTestAdapter(); + const { acquireActiveLock } = await import("@/chat/state/locks"); + const busyLock = await acquireActiveLock(adapter, threadId); + if (!busyLock) throw new Error("Expected active conversation lock"); + const blockedResponse = + await mcpOauthCallbackHarnessModule.runMcpOauthCallbackRoute({ + provider: EVAL_MCP_AUTH_PROVIDER, + state: authProvider.authSessionId, + code: EVAL_MCP_AUTH_CODE, + agentRunner: testAgentRunner, + agentContinueOptions: { queue, state: adapter }, + }); + + expect(blockedResponse.status).toBe(500); + expect(readEvalMcpAuthorizationCodeExchanges()).toBe(0); + expect(executeAgentRunMock).not.toHaveBeenCalled(); + await expect( + mcpAuthStoreModule.getMcpAuthSession(authProvider.authSessionId), + ).resolves.toBeDefined(); + await expect( + mcpAuthStoreModule.getMcpStoredOAuthCredentials( + "U123", + EVAL_MCP_AUTH_PROVIDER, + ), + ).resolves.toEqual( + expect.not.objectContaining({ + authorizationCompletionId: expect.any(String), + tokens: expect.any(Object), + }), + ); + expect(queue.sentRecords()).toHaveLength(0); + await adapter.releaseLock(busyLock); + + const response = + await mcpOauthCallbackHarnessModule.runMcpOauthCallbackRoute({ + provider: EVAL_MCP_AUTH_PROVIDER, + state: authProvider.authSessionId, + code: EVAL_MCP_AUTH_CODE, + agentRunner: testAgentRunner, + agentContinueOptions: { queue, state: adapter }, + }); + + expect(response.status).toBe(200); + expect(readEvalMcpAuthorizationCodeExchanges()).toBe(1); + await expect( + mcpAuthStoreModule.getMcpStoredOAuthCredentials( + "U123", + EVAL_MCP_AUTH_PROVIDER, + ), + ).resolves.toMatchObject({ + authorizationCompletionId: expect.any(String), + tokens: expect.any(Object), + }); + expect(queue.sentRecords()).toHaveLength(1); + + expect(executeAgentRunMock).toHaveBeenCalledTimes(1); + expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([ + expect.objectContaining({ + params: expect.objectContaining({ + channel: "C123", + thread_ts: "1700000000.014", + text: "The budget deadline you mentioned earlier was Friday.", + }), + }), + ]); + }); + it("fails MCP OAuth resume when stored actor team mismatches destination", async () => { const threadId = "slack:C123:1700000000.006"; const sessionId = "turn_user-6"; @@ -958,6 +1074,80 @@ describe("mcp oauth callback slack integration", () => { expect(getCapturedSlackApiCalls("chat.postMessage")).toHaveLength(0); }); + it("cleans completed session state at the MCP handler gate", async () => { + const conversationId = "conversation-mcp-completed"; + const threadId = "slack:C123:1700000000.008"; + const sessionId = "turn_user-8"; + await stateAdapterModule.getStateAdapter().set(`thread-state:${threadId}`, { + conversation: { + messages: [ + { + id: "user-8", + role: "user", + text: "list mcp data", + createdAtMs: 1, + author: { userId: "U123", userName: "dcramer" }, + }, + { + id: `assistant:${sessionId}`, + role: "assistant", + text: "Already delivered", + createdAtMs: 2, + author: { isBot: true, userName: "junior" }, + meta: { replied: true }, + }, + ], + processing: { + activeTurnId: sessionId, + pendingAuth: { + kind: "mcp", + provider: EVAL_MCP_AUTH_PROVIDER, + actorId: "U123", + sessionId, + linkSentAtMs: 1, + }, + }, + }, + }); + await seedVisibleTranscriptFromThreadState( + stateAdapterModule.getStateAdapter(), + threadId, + ); + await turnSessionStoreModule.upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + sessionId, + sliceId: 2, + state: "completed", + destination: SLACK_DESTINATION, + source: slackSource("1700000000.008"), + piMessages: [], + actor: { platform: "slack", teamId: "T123", userId: "U123" }, + }); + const authProvider = await createPendingAuthSession({ + conversationId, + sessionId, + userMessage: "list mcp data", + channelId: "C123", + threadTs: "1700000000.008", + }); + const response = + await mcpOauthCallbackHarnessModule.runMcpOauthCallbackRoute({ + provider: EVAL_MCP_AUTH_PROVIDER, + state: authProvider.authSessionId, + code: EVAL_MCP_AUTH_CODE, + agentRunner: testAgentRunner, + }); + + expect(response.status).toBe(200); + expect(executeAgentRunMock).not.toHaveBeenCalled(); + const repaired = await stateAdapterModule.getStateAdapter().get<{ + conversation?: { processing?: { activeTurnId?: string } }; + }>(`thread-state:${threadId}`); + expect(repaired?.conversation?.processing?.activeTurnId).toBeUndefined(); + expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([]); + }); + it("does not resume MCP OAuth with a mismatched stored actor", async () => { const sessionId = "turn_user-7"; await stateAdapterModule diff --git a/packages/junior/tests/integration/oauth-callback-slack.test.ts b/packages/junior/tests/integration/oauth-callback-slack.test.ts index 7ca692f7e..ea40bfa66 100644 --- a/packages/junior/tests/integration/oauth-callback-slack.test.ts +++ b/packages/junior/tests/integration/oauth-callback-slack.test.ts @@ -9,6 +9,7 @@ import { createPluginAppFixture, type PluginAppFixture, } from "../fixtures/plugin-app"; +import { createConversationWorkQueueTestAdapter } from "../fixtures/conversation-work"; import { completedAgentRun } from "@/chat/runtime/agent-run-outcome"; import { hydrateConversationMessages, @@ -505,6 +506,399 @@ describe("oauth callback slack integration", () => { ]); }); + it("converges concurrent callbacks for the same OAuth-paused turn", async () => { + const conversationId = "slack:C123:1700000000.016"; + const sessionId = "turn_msg_concurrent"; + const state = "eval-oauth-concurrent-state"; + const source = slackSource("1700000000.016"); + await turnSessionStoreModule.upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + sessionId, + sliceId: 2, + state: "awaiting_resume", + destination: SLACK_DESTINATION, + source, + piMessages: [], + resumeReason: "auth", + resumedFromSliceId: 1, + actor: { + platform: "slack", + teamId: "T123", + userId: "U123", + userName: "dcramer", + }, + }); + await stateAdapterModule.getStateAdapter().set(`oauth-state:${state}`, { + userId: "U123", + provider: "eval-oauth", + channelId: "C123", + destination: SLACK_DESTINATION, + source, + threadTs: "1700000000.016", + pendingMessage: "list my sentry issues", + resumeConversationId: conversationId, + resumeSessionId: sessionId, + }); + await stateAdapterModule + .getStateAdapter() + .set(`thread-state:${conversationId}`, { + conversation: { + messages: [ + { + id: "msg.concurrent", + role: "user", + text: "list my sentry issues", + createdAtMs: 1, + author: { userId: "U123", userName: "dcramer" }, + }, + ], + processing: { + pendingAuth: { + kind: "plugin", + provider: "eval-oauth", + actorId: "U123", + sessionId, + linkSentAtMs: 1, + }, + }, + }, + }); + await seedVisibleTranscriptFromThreadState( + stateAdapterModule.getStateAdapter(), + conversationId, + ); + executeAgentRunMock.mockResolvedValue( + completedAgentRun({ + text: "Here are your Sentry issues.", + piMessages: [ + { + role: "user", + content: [{ type: "text", text: "list my sentry issues" }], + timestamp: 1, + }, + { + role: "assistant", + content: [{ type: "text", text: "Here are your Sentry issues." }], + timestamp: 2, + }, + ] as any, + diagnostics: makeDiagnostics(), + }), + ); + + const queue = createConversationWorkQueueTestAdapter(); + const beginCallback = async () => { + const callbacks: Array<() => Promise | void> = []; + const { GET } = await import("@/handlers/oauth-callback"); + const { ConversationTurnLifecycleService } = + await import("@/chat/conversations/turn-lifecycle"); + const { getConversationEventStore } = await import("@/chat/db"); + const response = await GET( + new Request( + `https://junior.example.com/api/oauth/callback/eval-oauth?state=${state}&code=eval-oauth-code`, + ), + "eval-oauth", + (task) => { + callbacks.push(typeof task === "function" ? task : () => task); + }, + { + agentRunner: testAgentRunner, + agentContinueOptions: { + queue, + state: stateAdapterModule.getStateAdapter(), + }, + turnLifecycle: new ConversationTurnLifecycleService( + getConversationEventStore(), + ), + }, + ); + return { callbacks, response }; + }; + + const attempts = await Promise.all([beginCallback(), beginCallback()]); + expect(attempts.map(({ response }) => response.status)).toEqual([200, 200]); + await Promise.all( + attempts.flatMap(({ callbacks }) => + callbacks.map((callback) => callback()), + ), + ); + + expect(executeAgentRunMock).toHaveBeenCalledTimes(1); + expect(getCapturedSlackApiCalls("chat.postMessage")).toHaveLength(1); + await expect( + turnSessionStoreModule.getAgentTurnSessionRecord( + conversationId, + sessionId, + ), + ).resolves.toMatchObject({ state: "completed" }); + const persisted = await stateAdapterModule.getStateAdapter().get<{ + conversation?: { + processing?: { activeTurnId?: string; pendingAuth?: unknown }; + }; + }>(`thread-state:${conversationId}`); + expect(persisted?.conversation?.processing).toMatchObject({ + activeTurnId: undefined, + pendingAuth: undefined, + }); + }); + + it("recovers a committed OAuth credential when activation stays busy", async () => { + const conversationId = "slack:C123:1700000000.014"; + const sessionId = "turn_msg_14"; + const source = slackSource("1700000000.014"); + await turnSessionStoreModule.upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + sessionId, + sliceId: 2, + state: "awaiting_resume", + destination: SLACK_DESTINATION, + source, + piMessages: [], + resumeReason: "auth", + resumedFromSliceId: 1, + actor: { + platform: "slack", + teamId: "T123", + userId: "U123", + userName: "dcramer", + }, + }); + await stateAdapterModule + .getStateAdapter() + .set("oauth-state:eval-oauth-busy-state", { + userId: "U123", + provider: "eval-oauth", + channelId: "C123", + destination: SLACK_DESTINATION, + source, + threadTs: "1700000000.014", + pendingMessage: "list my sentry issues", + resumeConversationId: conversationId, + resumeSessionId: sessionId, + }); + await stateAdapterModule + .getStateAdapter() + .set(`thread-state:${conversationId}`, { + conversation: { + messages: [ + { + id: "msg.14", + role: "user", + text: "list my sentry issues", + createdAtMs: 1, + author: { userId: "U123", userName: "dcramer" }, + meta: { slackTs: "1700000000.0141" }, + }, + ], + processing: { + pendingAuth: { + kind: "plugin", + provider: "eval-oauth", + actorId: "U123", + sessionId, + linkSentAtMs: 1, + }, + }, + }, + }); + await seedVisibleTranscriptFromThreadState( + stateAdapterModule.getStateAdapter(), + conversationId, + ); + + const adapter = stateAdapterModule.getStateAdapter(); + const queue = createConversationWorkQueueTestAdapter(); + const sessionRecord = + await turnSessionStoreModule.getAgentTurnSessionRecord( + conversationId, + sessionId, + ); + if (!sessionRecord) throw new Error("Expected an auth-paused session"); + await turnSessionStoreModule.prepareAgentTurnAuthorizationRecovery({ + authorizationCompletionId: + turnSessionStoreModule.createAgentTurnAuthorizationCompletionId({ + attemptId: "eval-oauth-busy-state", + authorizationKind: "plugin", + provider: "eval-oauth", + }), + authorizationKind: "plugin", + conversationId, + expectedVersion: sessionRecord.version, + provider: "eval-oauth", + sessionId, + userId: "U123", + }); + const { recoverAuthorizationCompletedAgentTurns } = + await import("@/chat/services/agent-continue"); + await expect( + recoverAuthorizationCompletedAgentTurns({ queue, state: adapter }), + ).resolves.toBe(0); + expect(queue.sentRecords()).toHaveLength(0); + const originalSet = adapter.set.bind(adapter); + const { acquireActiveLock } = await import("@/chat/state/locks"); + let releaseBusyLock: Promise | undefined; + const setSpy = vi + .spyOn(adapter, "set") + .mockImplementation( + async (key: string, value: unknown, ttlMs?: number) => { + await originalSet(key, value, ttlMs); + if (key === "oauth-token:U123:eval-oauth" && !releaseBusyLock) { + const busyLock = await acquireActiveLock(adapter, conversationId); + if (!busyLock) throw new Error("Expected callback race lock"); + releaseBusyLock = new Promise((resolve) => { + setTimeout(() => { + void adapter.releaseLock(busyLock).then(resolve); + }, 350); + }); + } + }, + ); + await expect( + oauthCallbackHarnessModule.runOauthCallbackRoute({ + provider: "eval-oauth", + state: "eval-oauth-busy-state", + code: "eval-oauth-code", + agentRunner: testAgentRunner, + agentContinueOptions: { queue, state: adapter }, + }), + ).rejects.toThrow("OAuth turn changed while activating callback recovery"); + setSpy.mockRestore(); + await releaseBusyLock; + + expect(executeAgentRunMock).not.toHaveBeenCalled(); + await expect( + adapter.get("oauth-state:eval-oauth-busy-state"), + ).resolves.not.toBeNull(); + await expect( + capabilitiesFactoryModule + .createUserTokenStore() + .get("U123", "eval-oauth"), + ).resolves.toMatchObject({ accessToken: "eval-oauth-access-token" }); + await expect( + turnSessionStoreModule.getAgentTurnSessionRecord( + conversationId, + sessionId, + ), + ).resolves.toMatchObject({ + authorizationRecovery: { active: false }, + }); + await expect( + turnSessionStoreModule.getAgentTurnSessionRecord( + conversationId, + sessionId, + ), + ).resolves.toMatchObject({ resumeReason: "auth" }); + expect(queue.sentRecords()).toHaveLength(0); + for (let index = 0; index < 5_001; index += 1) { + await adapter.appendToList( + "junior:agent_turn_session:index", + { + conversationId: `unrelated:${index}`, + cumulativeDurationMs: 0, + lastProgressAtMs: index, + sessionId: `unrelated-session:${index}`, + sliceId: 1, + startedAtMs: index, + state: "completed", + updatedAtMs: index, + version: 1, + }, + { maxLength: 5_000 }, + ); + } + await expect( + adapter.getList("junior:agent_turn_session:index"), + ).resolves.not.toContainEqual( + expect.objectContaining({ conversationId, sessionId }), + ); + let workRequestFailed = false; + const failingWorkState = new Proxy(adapter, { + get(target, property, receiver) { + if (property === "set") { + return async (key: string, value: unknown, ttlMs?: number) => { + if ( + !workRequestFailed && + key === `junior:conversation:${conversationId}` + ) { + workRequestFailed = true; + throw new Error("conversation work store unavailable"); + } + return await target.set(key, value, ttlMs); + }; + } + const value = Reflect.get(target, property, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + await expect( + recoverAuthorizationCompletedAgentTurns({ + queue, + state: failingWorkState, + }), + ).resolves.toBe(0); + await expect( + turnSessionStoreModule.getAgentTurnSessionRecord( + conversationId, + sessionId, + ), + ).resolves.toMatchObject({ + authorizationRecovery: { active: true }, + }); + await expect( + recoverAuthorizationCompletedAgentTurns({ + queue, + state: adapter, + }), + ).resolves.toBe(1); + const { listAuthorizationRecoveries } = + await import("@/chat/state/authorization-recovery-index"); + await expect(listAuthorizationRecoveries(adapter)).resolves.toEqual([]); + await expect( + turnSessionStoreModule.getAgentTurnSessionRecord( + conversationId, + sessionId, + ), + ).resolves.toMatchObject({ resumeReason: "auth" }); + expect(queue.sentRecords()).toHaveLength(1); + + const { resumeAwaitingSlackContinuation } = + await import("@/chat/runtime/agent-continue-runner"); + const { processConversationQueueMessage } = + await import("@/chat/task-execution/vercel-callback"); + const { ConversationTurnLifecycleService } = + await import("@/chat/conversations/turn-lifecycle"); + const { getConversationEventStore } = await import("@/chat/db"); + const turnLifecycle = new ConversationTurnLifecycleService( + getConversationEventStore(), + ); + await expect( + processConversationQueueMessage(queue.takeMessage(), { + queue, + state: adapter, + run: async ({ conversationId: queuedConversationId }) => { + await resumeAwaitingSlackContinuation(queuedConversationId, { + agentRunner: testAgentRunner, + turnLifecycle, + }); + return { status: "completed" }; + }, + }), + ).resolves.toEqual({ status: "completed" }); + + expect(executeAgentRunMock).toHaveBeenCalledTimes(1); + expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([ + expect.objectContaining({ + params: expect.objectContaining({ + channel: "C123", + thread_ts: "1700000000.014", + text: "Here are your Sentry issues.", + }), + }), + ]); + }); + it("fails a session-recorded OAuth resume with mismatched actor team", async () => { const conversationId = "slack:C123:1700000000.012"; const sessionId = "turn_msg_12"; @@ -949,4 +1343,142 @@ describe("oauth callback slack integration", () => { expect(executeAgentRunMock).not.toHaveBeenCalled(); expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([]); }); + + it("cleans completed session state at the OAuth handler gate", async () => { + const conversationId = "slack:C123:1700000000.013"; + const sessionId = "turn_msg_13"; + await turnSessionStoreModule.upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + sessionId, + sliceId: 2, + state: "completed", + destination: SLACK_DESTINATION, + source: slackSource("1700000000.013"), + piMessages: [], + actor: { platform: "slack", teamId: "T123", userId: "U123" }, + }); + await stateAdapterModule + .getStateAdapter() + .set("oauth-state:eval-oauth-completed-state", { + userId: "U123", + provider: "eval-oauth", + channelId: "C123", + destination: SLACK_DESTINATION, + source: slackSource("1700000000.013"), + threadTs: "1700000000.013", + pendingMessage: "list my sentry issues", + resumeConversationId: conversationId, + resumeSessionId: sessionId, + }); + await stateAdapterModule + .getStateAdapter() + .set(`thread-state:${conversationId}`, { + conversation: { + messages: [ + { + id: "msg.13", + role: "user", + text: "list my sentry issues", + createdAtMs: 1, + author: { userId: "U123", userName: "dcramer" }, + }, + { + id: `assistant:${sessionId}`, + role: "assistant", + text: "Already delivered", + createdAtMs: 2, + author: { isBot: true, userName: "junior" }, + meta: { replied: true }, + }, + ], + processing: { activeTurnId: sessionId }, + }, + }); + await seedVisibleTranscriptFromThreadState( + stateAdapterModule.getStateAdapter(), + conversationId, + ); + const response = await oauthCallbackHarnessModule.runOauthCallbackRoute({ + provider: "eval-oauth", + state: "eval-oauth-completed-state", + code: "eval-oauth-code", + agentRunner: testAgentRunner, + }); + + expect(response.status).toBe(200); + expect(executeAgentRunMock).not.toHaveBeenCalled(); + const repaired = await stateAdapterModule.getStateAdapter().get<{ + conversation?: { processing?: { activeTurnId?: string } }; + }>(`thread-state:${conversationId}`); + expect(repaired?.conversation?.processing?.activeTurnId).toBeUndefined(); + expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([]); + }); + + it("cleans a terminal auth session when its stale continuation is delivered", async () => { + const conversationId = "slack:C123:1700000000.015"; + const sessionId = "turn_msg_stale_auth"; + const session = await turnSessionStoreModule.upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + sessionId, + sliceId: 2, + state: "failed", + destination: SLACK_DESTINATION, + source: slackSource("1700000000.015"), + piMessages: [], + actor: { platform: "slack", teamId: "T123", userId: "U123" }, + }); + await stateAdapterModule + .getStateAdapter() + .set(`thread-state:${conversationId}`, { + conversation: { + messages: [], + processing: { + activeTurnId: sessionId, + pendingAuth: { + kind: "plugin", + provider: "eval-oauth", + actorId: "U123", + sessionId, + linkSentAtMs: 1, + }, + }, + }, + }); + + const { continueSlackAgentRun } = + await import("@/chat/runtime/agent-continue-runner"); + const { ConversationTurnLifecycleService } = + await import("@/chat/conversations/turn-lifecycle"); + const { getConversationEventStore } = await import("@/chat/db"); + await expect( + continueSlackAgentRun( + { + conversationId, + destination: SLACK_DESTINATION, + expectedVersion: session.version, + sessionId, + }, + { + agentRunner: testAgentRunner, + turnLifecycle: new ConversationTurnLifecycleService( + getConversationEventStore(), + ), + }, + ), + ).resolves.toBe(false); + + const persisted = await stateAdapterModule.getStateAdapter().get<{ + conversation?: { + processing?: { activeTurnId?: string; pendingAuth?: unknown }; + }; + }>(`thread-state:${conversationId}`); + expect(persisted?.conversation?.processing).toMatchObject({ + activeTurnId: undefined, + pendingAuth: undefined, + }); + expect(executeAgentRunMock).not.toHaveBeenCalled(); + expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([]); + }); }); diff --git a/packages/junior/tests/integration/oauth-resume-slack.test.ts b/packages/junior/tests/integration/oauth-resume-slack.test.ts index 7a2cb8388..9d35a6595 100644 --- a/packages/junior/tests/integration/oauth-resume-slack.test.ts +++ b/packages/junior/tests/integration/oauth-resume-slack.test.ts @@ -4,9 +4,25 @@ import { getSlackContinuationMarker, getSlackInterruptionMarker, } from "@/chat/slack/output"; -import { disconnectStateAdapter } from "@/chat/state/adapter"; +import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; +import { acquireActiveLock } from "@/chat/state/locks"; import { getCapturedSlackApiCalls } from "../msw/handlers/slack-api"; import { completedAgentRun } from "@/chat/runtime/agent-run-outcome"; +import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; +import { + getConversationEventStore, + getConversationMessageStore, + getSqlExecutor, +} from "@/chat/db"; +import { + RecoverableSlackDeliveryService, + type RecoverableSlackDeliveryPort, +} from "@/chat/slack/recoverable-delivery"; +import { postRecoverableSlackMessage } from "@/chat/slack/outbound"; +import { GET as heartbeat } from "@/handlers/heartbeat"; +import { createWaitUntilCollector } from "../fixtures/wait-until"; +import { getAgentTurnSessionRecord } from "@/chat/state/turn-session"; +import { upsertAgentTurnSessionRecord } from "@/chat/state/turn-session"; function makeDiagnostics( outcome: "success" | "execution_failure" | "provider_error" = "success", @@ -48,16 +64,144 @@ function expectBlocksIncludeConversationId( expect(JSON.stringify(params.blocks)).toContain(conversationId); } +async function runRecoverableSchedulingCase(args: { + deliveryOutcome: "accepted" | "failed"; + modelOutcome: "success" | "execution_failure"; + suffix: string; +}) { + const { resumeSlackTurn } = await import("@/chat/runtime/slack-resume"); + const conversationId = `conversation-schedule-${args.suffix}`; + const turnId = `turn-schedule-${args.suffix}`; + const messageId = `message-schedule-${args.suffix}`; + const threadTs = `1700000000.${args.suffix}`; + await upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + sessionId: turnId, + sliceId: 2, + state: "awaiting_resume", + cumulativeDurationMs: 1_000, + cumulativeUsage: { totalTokens: 1_000 }, + piMessages: [], + resumeReason: "auth", + source: testSlackSource(threadTs), + }); + await getConversationMessageStore().record(conversationId, [ + { + messageId, + role: "user", + text: "continue this turn", + createdAtMs: Date.now(), + }, + ]); + const post: RecoverableSlackDeliveryPort["post"] = + args.deliveryOutcome === "accepted" + ? postRecoverableSlackMessage + : async () => ({ + outcome: "definitive_failure", + reason: "api_rejected", + }); + const schedule = vi.fn(async () => undefined); + const run = vi.fn(async () => + completedAgentRun({ + text: "done", + diagnostics: makeDiagnostics(args.modelOutcome, { + durationMs: 500, + usage: { outputTokens: 7 }, + }), + }), + ); + await resumeSlackTurn({ + messageText: "continue this turn", + channelId: "C123", + threadTs, + inputMessageIds: [messageId], + sliceId: 2, + recoverableSlackDelivery: new RecoverableSlackDeliveryService( + getSqlExecutor(), + { post, reconcile: vi.fn() }, + ), + turnLifecycle: new ConversationTurnLifecycleService( + getConversationEventStore(), + ), + replyContext: { + routing: { + credentialContext: { + actor: { type: "user", userId: "U123" }, + }, + destination: TEST_SLACK_DESTINATION, + source: testSlackSource(threadTs), + actor: { platform: "slack", teamId: "T123", userId: "U123" }, + correlation: { conversationId, turnId }, + }, + }, + agentRunner: { run }, + scheduleSessionCompletedPluginTasks: schedule, + }); + return { conversationId, run, schedule, turnId }; +} + describe("oauth resume slack integration", () => { beforeEach(async () => { process.env.JUNIOR_STATE_ADAPTER = "memory"; - vi.resetModules(); await disconnectStateAdapter(); }); afterEach(async () => { await disconnectStateAdapter(); delete process.env.JUNIOR_STATE_ADAPTER; + delete process.env.JUNIOR_SCHEDULER_SECRET; + vi.useRealTimers(); + }); + + it("posts the safe fallback when failure-state persistence fails", async () => { + const { resumeSlackTurn } = await import("@/chat/runtime/slack-resume"); + const conversationId = "slack:T123:C123:1700000000.009"; + const turnId = "turn_1700000000_009"; + + await expect( + resumeSlackTurn({ + messageText: "Resume the failed turn", + channelId: "C123", + threadTs: "1700000000.009", + inputMessageIds: ["msg.9"], + lifecycleCorrelation: { conversationId, turnId }, + turnLifecycle: new ConversationTurnLifecycleService( + getConversationEventStore(), + ), + replyContext: { + routing: { + credentialContext: { + actor: { type: "user", userId: "U123" }, + }, + correlation: { conversationId, turnId }, + destination: TEST_SLACK_DESTINATION, + source: testSlackSource("1700000000.009"), + actor: { platform: "slack", teamId: "T123", userId: "U123" }, + }, + }, + agentRunner: { + run: async () => { + throw new Error("resume failed"); + }, + }, + onFailure: async () => { + throw new Error("failure state unavailable"); + }, + }), + ).rejects.toThrow("failure state unavailable"); + + expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([ + expect.objectContaining({ + params: expect.objectContaining({ + channel: "C123", + thread_ts: "1700000000.009", + text: expect.stringContaining( + "I ran into an internal error while processing that. Reference: `event_id=", + ), + }), + }), + ]); }); it("posts resumed status updates through the Slack MSW harness", async () => { @@ -146,73 +290,8 @@ describe("oauth resume slack integration", () => { ]); }, 10_000); - it("posts the safe fallback when failure-state persistence fails", async () => { - const { resumeSlackTurn } = await import("@/chat/runtime/slack-resume"); - const { getConversationEventStore } = await import("@/chat/db"); - const conversationId = "slack:T123:C123:1700000000.009"; - const turnId = "turn_1700000000_009"; - - await expect( - resumeSlackTurn({ - messageText: "Resume the failed turn", - channelId: "C123", - threadTs: "1700000000.009", - inputMessageIds: ["msg.9"], - lifecycleCorrelation: { conversationId, turnId }, - replyContext: { - routing: { - credentialContext: { - actor: { type: "user", userId: "U123" }, - }, - correlation: { conversationId, turnId }, - destination: TEST_SLACK_DESTINATION, - source: testSlackSource("1700000000.009"), - actor: { platform: "slack", teamId: "T123", userId: "U123" }, - }, - }, - agentRunner: { - run: async () => { - throw new Error("resume failed"); - }, - }, - onFailure: async () => { - throw new Error("failure state unavailable"); - }, - }), - ).rejects.toThrow("failure state unavailable"); - - expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([ - expect.objectContaining({ - params: expect.objectContaining({ - channel: "C123", - thread_ts: "1700000000.009", - text: expect.stringContaining( - "I ran into an internal error while processing that. Reference: `event_id=", - ), - }), - }), - ]); - - const lifecycle = ( - await getConversationEventStore().loadHistory(conversationId) - ).filter((event) => event.data.type.startsWith("turn_")); - expect(lifecycle.map((event) => event.data)).toEqual([ - expect.objectContaining({ - type: "turn_started", - turnId, - inputMessageIds: ["msg.9"], - surface: "slack", - }), - expect.objectContaining({ - type: "turn_failed", - turnId, - failureCode: "persistence_failed", - eventId: expect.stringMatching(/^[a-f0-9]{32}$/i), - }), - ]); - }); - - it("uses correlation IDs for resumed reply footers", async () => { + it("recovers a resumed pending delivery on heartbeat without rerunning or reposting", async () => { + process.env.JUNIOR_SCHEDULER_SECRET = "heartbeat-secret"; const { resumeAuthorizedRequest } = await import("@/chat/runtime/slack-resume"); const { upsertAgentTurnSessionRecord } = @@ -232,12 +311,55 @@ describe("oauth resume slack integration", () => { }, source: testSlackSource("1700000000.007"), }); + await expect( + getAgentTurnSessionRecord("conversation-1", "turn-1"), + ).resolves.toMatchObject({ + state: "awaiting_resume", + sliceId: 2, + cumulativeDurationMs: 1_000, + }); + await getConversationMessageStore().record("conversation-1", [ + { + messageId: "message-turn-1", + role: "user", + text: "continue this turn", + createdAtMs: Date.now(), + }, + ]); + const recoverableSlackDelivery = new RecoverableSlackDeliveryService( + getSqlExecutor(), + { + post: postRecoverableSlackMessage, + reconcile: vi.fn(), + }, + ); + const run = vi.fn(async () => + completedAgentRun({ + text: "done", + diagnostics: makeDiagnostics("success", { + durationMs: 500, + usage: { + outputTokens: 7, + }, + }), + }), + ); + const onSuccess = vi.fn(async () => { + throw new Error("repair crashed"); + }); + const onRecoveredSuccess = vi.fn(async () => undefined); - await resumeAuthorizedRequest({ + const resumeArgs: Parameters[0] = { messageText: "continue this turn", channelId: "C123", threadTs: "1700000000.007", connectedText: "", + inputMessageIds: ["message-turn-1"], + sliceId: 2, + recoverableSlackDelivery, + turnLifecycle: new ConversationTurnLifecycleService( + getConversationEventStore(), + ), replyContext: { routing: { credentialContext: { @@ -252,19 +374,59 @@ describe("oauth resume slack integration", () => { }, }, }, - agentRunner: { - run: async () => - completedAgentRun({ - text: "done", - diagnostics: makeDiagnostics("success", { - durationMs: 500, - usage: { - outputTokens: 7, - }, - }), - }), - }, + agentRunner: { run }, + onSuccess, + onRecoveredSuccess, + }; + + await resumeAuthorizedRequest(resumeArgs); + await expect( + getAgentTurnSessionRecord("conversation-1", "turn-1"), + ).resolves.toMatchObject({ + state: "awaiting_resume", + sliceId: 2, + cumulativeDurationMs: 1_000, }); + const state = getStateAdapter(); + await state.connect(); + const activeRuntimeLock = await acquireActiveLock(state, "conversation-1"); + expect(activeRuntimeLock).not.toBeNull(); + try { + const blockedWaitUntil = createWaitUntilCollector(); + const blockedResponse = await heartbeat( + new Request("https://example.invalid/api/internal/heartbeat", { + headers: { authorization: "Bearer heartbeat-secret" }, + }), + blockedWaitUntil.fn, + { recoverableSlackDelivery }, + ); + expect(blockedResponse.status).toBe(202); + await blockedWaitUntil.flush(); + + const blockedLifecycle = ( + await getConversationEventStore().loadHistory("conversation-1") + ).filter((event) => event.data.type.startsWith("turn_")); + expect(blockedLifecycle.map((event) => event.data.type)).toEqual([ + "turn_started", + ]); + } finally { + await state.releaseLock(activeRuntimeLock!); + } + + const recoveryWaitUntil = createWaitUntilCollector(); + const recoveryResponse = await heartbeat( + new Request("https://example.invalid/api/internal/heartbeat", { + headers: { authorization: "Bearer heartbeat-secret" }, + }), + recoveryWaitUntil.fn, + { recoverableSlackDelivery }, + ); + expect(recoveryResponse.status).toBe(202); + await recoveryWaitUntil.flush(); + + expect(run).toHaveBeenCalledTimes(1); + expect(onSuccess).toHaveBeenCalledTimes(1); + expect(onRecoveredSuccess).not.toHaveBeenCalled(); expect(getCapturedSlackApiCalls("chat.postMessage")).toEqual([ expect.objectContaining({ @@ -290,6 +452,113 @@ describe("oauth resume slack integration", () => { }), }), ]); + const lifecycle = ( + await getConversationEventStore().loadHistory("conversation-1") + ).filter((event) => event.data.type.startsWith("turn_")); + expect(lifecycle.map((event) => event.data)).toEqual([ + { + type: "turn_started", + turnId: "turn-1", + inputMessageIds: ["message-turn-1"], + surface: "slack", + }, + { type: "turn_completed", turnId: "turn-1", outcome: "success" }, + ]); + await expect( + getAgentTurnSessionRecord("conversation-1", "turn-1"), + ).resolves.toMatchObject({ + state: "completed", + sliceId: 2, + cumulativeDurationMs: 1_500, + cumulativeUsage: { + totalTokens: 1_007, + }, + }); + }); + + it("schedules completed plugins once only for successful recoverable resumes", async () => { + const success = await runRecoverableSchedulingCase({ + deliveryOutcome: "accepted", + modelOutcome: "success", + suffix: "021", + }); + const modelFailure = await runRecoverableSchedulingCase({ + deliveryOutcome: "accepted", + modelOutcome: "execution_failure", + suffix: "022", + }); + const deliveryFailure = await runRecoverableSchedulingCase({ + deliveryOutcome: "failed", + modelOutcome: "success", + suffix: "023", + }); + + expect(success.run).toHaveBeenCalledOnce(); + expect(success.schedule).toHaveBeenCalledOnce(); + expect(success.schedule).toHaveBeenCalledWith({ + conversationId: success.conversationId, + sessionId: success.turnId, + }); + expect(modelFailure.run).toHaveBeenCalledOnce(); + expect(modelFailure.schedule).not.toHaveBeenCalled(); + expect(deliveryFailure.run).toHaveBeenCalledOnce(); + expect(deliveryFailure.schedule).not.toHaveBeenCalled(); + await expect( + getAgentTurnSessionRecord( + deliveryFailure.conversationId, + deliveryFailure.turnId, + ), + ).resolves.toMatchObject({ + cumulativeDurationMs: 1_500, + cumulativeUsage: { totalTokens: 1_007 }, + sliceId: 2, + state: "failed", + }); + }); + + it("records one terminal failure when a correlated resume fails", async () => { + const { resumeAuthorizedRequest } = + await import("@/chat/runtime/slack-resume"); + const conversationId = "conversation-resume-failure"; + const turnId = "turn-resume-failure"; + + await resumeAuthorizedRequest({ + messageText: "continue this turn", + channelId: "C123", + threadTs: "1700000000.009", + connectedText: "", + inputMessageIds: ["message-resume-failure"], + turnLifecycle: new ConversationTurnLifecycleService( + getConversationEventStore(), + ), + replyContext: { + routing: { + credentialContext: { + actor: { type: "user", userId: "U123" }, + }, + destination: TEST_SLACK_DESTINATION, + source: testSlackSource("1700000000.009"), + actor: { platform: "slack", teamId: "T123", userId: "U123" }, + correlation: { conversationId, turnId }, + }, + }, + agentRunner: { + run: async () => { + throw new Error("resume failed"); + }, + }, + }); + + const lifecycle = ( + await getConversationEventStore().loadHistory(conversationId) + ).filter((event) => event.data.type.startsWith("turn_")); + expect(lifecycle).toHaveLength(2); + expect(lifecycle.at(-1)?.data).toMatchObject({ + type: "turn_failed", + turnId, + failureCode: "agent_run_failed", + eventId: expect.any(String), + }); }); it("posts resumed auth pause notices with the conversation footer", async () => { @@ -301,6 +570,10 @@ describe("oauth resume slack integration", () => { channelId: "C123", threadTs: "1700000000.008", connectedText: "", + inputMessageIds: ["message-auth-pause"], + turnLifecycle: new ConversationTurnLifecycleService( + getConversationEventStore(), + ), replyContext: { routing: { credentialContext: { @@ -337,6 +610,10 @@ describe("oauth resume slack integration", () => { getCapturedSlackApiCalls("chat.postMessage")[0]!.params, "conversation-auth-pause", ); + const lifecycle = ( + await getConversationEventStore().loadHistory("conversation-auth-pause") + ).filter((event) => event.data.type.startsWith("turn_")); + expect(lifecycle.map((event) => event.data.type)).toEqual(["turn_started"]); }); it("chunks long resumed replies into explicit continuation messages", async () => { @@ -390,12 +667,18 @@ describe("oauth resume slack integration", () => { it("marks resumed provider-error partial replies as interrupted", async () => { const { resumeAuthorizedRequest } = await import("@/chat/runtime/slack-resume"); + const conversationId = "conversation-provider-error"; + const turnId = "turn-provider-error"; await resumeAuthorizedRequest({ messageText: "Continue the original request", channelId: "C123", threadTs: "1700000000.003", connectedText: "Connected. Continuing...", + inputMessageIds: ["message-provider-error"], + turnLifecycle: new ConversationTurnLifecycleService( + getConversationEventStore(), + ), replyContext: { routing: { credentialContext: { @@ -404,6 +687,7 @@ describe("oauth resume slack integration", () => { destination: TEST_SLACK_DESTINATION, source: testSlackSource("1700000000.003"), actor: { platform: "slack", teamId: "T123", userId: "U123" }, + correlation: { conversationId, turnId }, }, }, agentRunner: { @@ -426,6 +710,15 @@ describe("oauth resume slack integration", () => { getSlackInterruptionMarker().trim(), ); expect(postCalls[1]?.params.text).not.toContain("event_id="); + const lifecycle = ( + await getConversationEventStore().loadHistory(conversationId) + ).filter((event) => event.data.type.startsWith("turn_")); + expect(lifecycle.at(-1)?.data).toMatchObject({ + type: "turn_failed", + turnId, + failureCode: "model_execution_failed", + eventId: expect.any(String), + }); }); it("replaces resumed execution-failure replies before Slack planning", async () => { diff --git a/packages/junior/tests/integration/runtime/agent-run-agent-continue.test.ts b/packages/junior/tests/integration/runtime/agent-run-agent-continue.test.ts index 65593aa9a..089753f2c 100644 --- a/packages/junior/tests/integration/runtime/agent-run-agent-continue.test.ts +++ b/packages/junior/tests/integration/runtime/agent-run-agent-continue.test.ts @@ -446,6 +446,40 @@ describe("executeAgentRun agent continuation", () => { ); }); + it("uses the runtime-owned timeout instead of module configuration", async () => { + const replyPromise = executeAgentRun({ + input: { messageText: "help me" }, + routing: { + destination: TEST_DESTINATION, + source: TEST_SOURCE, + actor: TEST_ACTOR, + correlation: { + conversationId: "conversation-runtime-timeout", + turnId: "turn-runtime-timeout", + channelId: "C123", + threadTs: "1712345.0007", + }, + }, + policy: { turnTimeoutMs: 25_000 }, + }); + + await vi.advanceTimersByTimeAsync(10_000); + expect(promptAborted.value).toBe(false); + + await vi.advanceTimersByTimeAsync(15_000); + const outcome = await replyPromise; + + expect(promptAborted.value).toBe(true); + expect(outcome.status).toBe("suspended"); + const sessionRecord = await getAgentTurnSessionRecord( + "conversation-runtime-timeout", + "turn-runtime-timeout", + ); + expect(sessionRecord?.errorMessage).toBe( + "Agent turn timed out after 25000ms", + ); + }); + it("persists omitted-image context in the session-recorded Pi user message", async () => { const replyPromise = executeAgentRun({ input: { diff --git a/packages/junior/tests/integration/slack/assistant-thread-contract.test.ts b/packages/junior/tests/integration/slack/assistant-thread-contract.test.ts index 469ebb523..dcf5e4e42 100644 --- a/packages/junior/tests/integration/slack/assistant-thread-contract.test.ts +++ b/packages/junior/tests/integration/slack/assistant-thread-contract.test.ts @@ -19,6 +19,7 @@ import { flattenAgentRunRequestForTest } from "../../fixtures/agent-runner"; const SIGNING_SECRET = "test-signing-secret"; const BOT_USER_ID = "U0BOT"; const DM_CHANNEL_ID = "D12345"; +const DM_WITHOUT_THREAD_CHANNEL_ID = "DNOEXPLICITTHREAD"; const DM_THREAD_TS = "1700000000.000001"; const CHANNEL_ID = "C12345"; const CHANNEL_ROOT_TS = "1700000200.000200"; @@ -28,12 +29,12 @@ const slackWebhookClient = createSlackWebhookTestClient({ function createDirectMessageRequest( text: string, - options?: { threadTs?: string }, + options?: { channel?: string; threadTs?: string }, ): Request { return slackWebhookClient.event( slackEventsApiEnvelope({ eventType: "message", - channel: DM_CHANNEL_ID, + channel: options?.channel ?? DM_CHANNEL_ID, ts: "1700000100.000100", text, ...(options?.threadTs ? { threadTs: options.threadTs } : {}), @@ -387,7 +388,9 @@ describe("Slack contract: assistant-thread delivery", () => { const waitUntil = slackWebhookClient.waitUntil(); const response = await handleChatSdkPlatformWebhook( - createDirectMessageRequest("How do I debug memory leaks in Node?"), + createDirectMessageRequest("How do I debug memory leaks in Node?", { + channel: DM_WITHOUT_THREAD_CHANNEL_ID, + }), "slack", waitUntil.fn, bot, @@ -396,6 +399,12 @@ describe("Slack contract: assistant-thread delivery", () => { expect(response.status).toBe(200); await waitUntil.flush(); - expect(slackApiOutbox.calls("assistant.threads.setTitle")).toEqual([]); + expect( + slackApiOutbox + .calls("assistant.threads.setTitle") + .filter( + (call) => call.params.channel_id === DM_WITHOUT_THREAD_CHANNEL_ID, + ), + ).toEqual([]); }); }); diff --git a/packages/junior/tests/integration/slack/bot-handlers.test.ts b/packages/junior/tests/integration/slack/bot-handlers.test.ts index f68806c7f..759267090 100644 --- a/packages/junior/tests/integration/slack/bot-handlers.test.ts +++ b/packages/junior/tests/integration/slack/bot-handlers.test.ts @@ -20,7 +20,9 @@ import { type ConversationMessage, } from "@/chat/state/conversation"; import { + activateAgentTurnAuthorizationRecovery, getAgentTurnSessionRecord, + prepareAgentTurnAuthorizationRecovery, upsertAgentTurnSessionRecord, } from "@/chat/state/turn-session"; import { @@ -35,6 +37,7 @@ import { } from "../../fixtures/slack-harness"; import { createTestChatRuntime } from "../../fixtures/chat-runtime"; import { flattenAgentRunRequestForTest } from "../../fixtures/agent-runner"; +import { getConversationEventStore } from "@/chat/db"; const emptyThreadReplies = async () => []; @@ -204,6 +207,34 @@ function turnPiMessages(text: string) { ]; } +async function seedStartedTurn(args: { + conversationId: string; + messageId: string; + turnId: string; +}): Promise { + await getConversationEventStore().append(args.conversationId, [ + { + idempotencyKey: `turn:${args.turnId}:started`, + createdAtMs: 1, + data: { + type: "turn_started", + turnId: args.turnId, + inputMessageIds: [args.messageId], + surface: "slack", + }, + }, + ]); +} + +async function loadTurnLifecycle(conversationId: string, turnId: string) { + return (await getConversationEventStore().loadHistory(conversationId)).filter( + (event) => + event.data.type.startsWith("turn_") && + "turnId" in event.data && + event.data.turnId === turnId, + ); +} + // ── Tests ──────────────────────────────────────────────────────────── describe("bot handlers (integration)", () => { @@ -1254,7 +1285,7 @@ describe("bot handlers (integration)", () => { }, }), ); - await upsertAgentTurnSessionRecord({ + const parkedSession = await upsertAgentTurnSessionRecord({ modelId: "test/model", conversationId, sessionId: activeSessionId, @@ -1263,6 +1294,27 @@ describe("bot handlers (integration)", () => { resumeReason: "auth", piMessages: turnPiMessages("please use notion"), }); + await seedStartedTurn({ + conversationId, + messageId: "msg-original", + turnId: activeSessionId, + }); + const prepared = await prepareAgentTurnAuthorizationRecovery({ + authorizationCompletionId: "lost-callback-receipt", + authorizationKind: "plugin", + conversationId, + expectedVersion: parkedSession.version, + provider: "notion", + sessionId: activeSessionId, + userId: "U-test", + }); + if (!prepared) throw new Error("Expected prepared auth recovery"); + await activateAgentTurnAuthorizationRecovery({ + authorizationCompletionId: "lost-callback-receipt", + conversationId, + expectedVersion: prepared.version, + sessionId: activeSessionId, + }); const { slackRuntime } = createRuntime({ services: { replyExecutor: { @@ -1271,9 +1323,27 @@ describe("bot handlers (integration)", () => { }, }); + const threadState = createAwaitingContinuationState({ activeSessionId }); + const processing = threadState.conversation + .processing as typeof threadState.conversation.processing & { + pendingAuth?: { + actorId: string; + kind: "plugin"; + linkSentAtMs: number; + provider: string; + sessionId: string; + }; + }; + processing.pendingAuth = { + actorId: "U-test", + kind: "plugin", + linkSentAtMs: 1, + provider: "notion", + sessionId: activeSessionId, + }; const thread = createTestThread({ id: conversationId, - state: createAwaitingContinuationState({ activeSessionId }), + state: threadState, }); await slackRuntime.handleNewMention( @@ -1305,10 +1375,123 @@ describe("bot handlers (integration)", () => { const state = thread.getState(); const conversation = ( state as { - conversation?: { processing?: { activeTurnId?: string } }; + conversation?: { + processing?: { activeTurnId?: string; pendingAuth?: unknown }; + }; } ).conversation; expect(conversation?.processing?.activeTurnId).toBeUndefined(); + expect(conversation?.processing?.pendingAuth).toBeUndefined(); + await expect( + loadTurnLifecycle(conversationId, activeSessionId), + ).resolves.toEqual([ + expect.objectContaining({ + data: expect.objectContaining({ type: "turn_started" }), + }), + expect.objectContaining({ + data: expect.objectContaining({ + type: "turn_failed", + failureCode: "agent_run_failed", + }), + }), + ]); + }); + + it("does not abandon an auth turn while its resume lock completes", async () => { + const conversationId = "slack:C0AUTHRACE:1700000000.000"; + const activeSessionId = "turn_msg-auth-race"; + const destination = slackDestination("C0AUTHRACE"); + const source = createSlackSourceForTest("C0AUTHRACE"); + const session = await upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + sessionId: activeSessionId, + sliceId: 1, + state: "awaiting_resume", + resumeReason: "auth", + destination, + source, + piMessages: turnPiMessages("please use notion"), + }); + await seedStartedTurn({ + conversationId, + messageId: "msg-original", + turnId: activeSessionId, + }); + const executeAgentRun = vi.fn().mockResolvedValue( + completedAgentRun({ + text: "The prior turn already completed.", + diagnostics: { + assistantMessageCount: 1, + modelId: "test-model", + outcome: "success" as const, + toolCalls: [], + toolErrorCount: 0, + toolResultCount: 0, + usedPrimaryText: true, + }, + }), + ); + const { slackRuntime } = createRuntime({ + services: { + replyExecutor: { agentRunner: { run: executeAgentRun } }, + }, + }); + const thread = createTestThread({ + id: conversationId, + state: createAwaitingContinuationState({ activeSessionId }), + }); + const followUp = createTestMessage({ + id: "msg-auth-race-follow-up", + threadId: conversationId, + text: "any update?", + isMention: true, + }); + + const state = getStateAdapter(); + await state.connect(); + const lock = await acquireActiveLock(state, conversationId); + expect(lock).not.toBeNull(); + try { + await expect( + slackRuntime.handleNewMention(thread, followUp, { destination }), + ).rejects.toThrow("Turn input is deferred until the active resume ends"); + await upsertAgentTurnSessionRecord({ + modelId: "test/model", + conversationId, + sessionId: activeSessionId, + sliceId: 1, + state: "completed", + destination, + source, + piMessages: session.piMessages, + }); + await getConversationEventStore().append(conversationId, [ + { + idempotencyKey: `turn:${activeSessionId}:terminal`, + createdAtMs: 2, + data: { + type: "turn_completed", + turnId: activeSessionId, + outcome: "success", + }, + }, + ]); + } finally { + await state.releaseLock(lock!); + } + + await slackRuntime.handleNewMention(thread, followUp, { destination }); + + await expect( + getAgentTurnSessionRecord(conversationId, activeSessionId), + ).resolves.toMatchObject({ state: "completed" }); + expect(executeAgentRun).toHaveBeenCalledOnce(); + expect( + (await loadTurnLifecycle(conversationId, activeSessionId)).map( + (event) => event.data.type, + ), + ).toEqual(["turn_started", "turn_completed"]); }); it("appends a parked-conversation follow-up to the event log before consuming it", async () => { @@ -1770,7 +1953,7 @@ describe("bot handlers (integration)", () => { expect(thread.posts).toEqual([]); }); - it("posts the failure fallback after ack even when the attempt is not final", async () => { + it("keeps the inbox pending after Pi commits input until delivery terminalizes", async () => { const ack = vi.fn().mockResolvedValue(undefined); const { slackRuntime } = createRuntime({ services: { @@ -1809,12 +1992,8 @@ describe("bot handlers (integration)", () => { }, ); - expect(ack).toHaveBeenCalledOnce(); - expect(thread.posts).toEqual([ - expect.stringContaining( - "I ran into an internal error while processing that.", - ), - ]); + expect(ack).not.toHaveBeenCalled(); + expect(thread.posts).toEqual([]); }); it("posts the failure fallback on the final delivery attempt", async () => { @@ -1881,6 +2060,11 @@ describe("bot handlers (integration)", () => { resumeReason: "timeout", piMessages: turnPiMessages("please keep working"), }); + await seedStartedTurn({ + conversationId, + messageId: "msg-original", + turnId: activeSessionId, + }); const { slackRuntime } = createRuntime({ services: { replyExecutor: { @@ -1922,6 +2106,19 @@ describe("bot handlers (integration)", () => { } ).conversation; expect(conversation?.processing?.activeTurnId).toBeUndefined(); + await expect( + loadTurnLifecycle(conversationId, activeSessionId), + ).resolves.toEqual([ + expect.objectContaining({ + data: expect.objectContaining({ type: "turn_started" }), + }), + expect.objectContaining({ + data: expect.objectContaining({ + type: "turn_failed", + failureCode: "persistence_failed", + }), + }), + ]); }); it("reschedules an awaiting continuation for repeated delivery of the active message", async () => { @@ -3145,65 +3342,4 @@ describe("bot handlers (integration)", () => { expect(capturedContexts).toHaveLength(1); expect(capturedContexts[0]).toBeUndefined(); }); - - it("multi-turn state continuity: second turn sees first turn's conversation state", async () => { - let turnCount = 0; - const { slackRuntime } = createRuntime({ - services: { - replyExecutor: { - agentRunner: { - run: async () => { - turnCount += 1; - return completedAgentRun({ - text: `reply-${turnCount}`, - diagnostics: { - assistantMessageCount: 1, - modelId: "test-model", - outcome: "success" as const, - toolCalls: [], - toolErrorCount: 0, - toolResultCount: 0, - usedPrimaryText: true, - }, - }); - }, - }, - }, - }, - }); - - const thread = createTestThread({ id: "slack:C0MULTI:1700000000.000" }); - - await slackRuntime.handleNewMention( - thread, - createTestMessage({ - id: "msg-t1", - threadId: "slack:C0MULTI:1700000000.000", - text: "first turn", - isMention: true, - }), - { destination: createTestDestination(thread) }, - ); - - const conv1 = await loadVisibleConversation(thread); - expect(conv1).toBeDefined(); - const messageCountAfterFirst = conv1?.messages?.length ?? 0; - - await slackRuntime.handleNewMention( - thread, - createTestMessage({ - id: "msg-t2", - threadId: "slack:C0MULTI:1700000000.000", - text: "second turn", - isMention: true, - }), - { destination: createTestDestination(thread) }, - ); - - const conv2 = await loadVisibleConversation(thread); - expect(conv2).toBeDefined(); - expect(conv2?.messages?.length ?? 0).toBeGreaterThan( - messageCountAfterFirst, - ); - }); }); diff --git a/packages/junior/tests/integration/slack/finalized-reply-behavior.test.ts b/packages/junior/tests/integration/slack/finalized-reply-behavior.test.ts index d371f2927..b2275c0e6 100644 --- a/packages/junior/tests/integration/slack/finalized-reply-behavior.test.ts +++ b/packages/junior/tests/integration/slack/finalized-reply-behavior.test.ts @@ -12,6 +12,15 @@ import { } from "../../fixtures/slack-harness"; import { completedAgentRun } from "@/chat/runtime/agent-run-outcome"; import { flattenAgentRunRequestForTest } from "../../fixtures/agent-runner"; +import { TurnInputDeferredError } from "@/chat/runtime/turn"; +import { listAgentTurnSessionSummariesForConversation } from "@/chat/state/turn-session"; +import { getAgentTurnSessionRecord } from "@/chat/state/turn-session"; +import { getConversationEventStore, getSqlExecutor } from "@/chat/db"; +import { RecoverableSlackDeliveryService } from "@/chat/slack/recoverable-delivery"; +import { getPersistedThreadState } from "@/chat/runtime/thread-state"; +import { coerceThreadConversationState } from "@/chat/state/conversation"; +import { hydrateConversationMessages } from "@/chat/conversations/visible-messages"; +import { parseSlackMessageTs } from "@/chat/slack/timestamp"; function toPostedText(value: unknown): string { if (typeof value === "string") { @@ -323,4 +332,403 @@ describe("Slack behavior: finalized thread replies", () => { ); expect(turnLifecycle.complete).not.toHaveBeenCalled(); }); + + it("recovers a pending concrete Slack delivery without rerunning Pi", async () => { + const run = vi.fn(async () => + completedAgentRun({ + text: "Recovered exactly once", + piMessages: [ + { + role: "assistant", + content: [{ type: "text", text: "Recovered exactly once" }], + }, + ] as never, + diagnostics: makeDiagnostics(), + }), + ); + let advances = 0; + const scheduleSessionCompletedPluginTasks = vi.fn(); + const recoverableSlackDelivery = new RecoverableSlackDeliveryService( + getSqlExecutor(), + { + post: vi.fn(async () => { + advances += 1; + if (advances === 1) { + return { + outcome: "retryable_absence" as const, + reason: "rate_limited" as const, + retryAtMs: Date.now(), + }; + } + return { + outcome: "accepted" as const, + ts: parseSlackMessageTs("1700006008.001")!, + }; + }), + reconcile: vi.fn(), + }, + ); + const { slackRuntime } = createTestChatRuntime({ + services: { + replyExecutor: { + agentRunner: { run }, + recoverableSlackDelivery, + scheduleSessionCompletedPluginTasks, + }, + }, + }); + const thread = createTestThread({ + id: "slack:C0FINAL:1700006008.000", + }); + (thread as unknown as { adapter: { name: string } }).adapter = { + name: "slack", + }; + const message = createTestMessage({ + id: "m-final-recovery", + text: "<@U0APP> recover me", + isMention: true, + threadId: thread.id, + }); + + await expect( + slackRuntime.handleNewMention(thread, message, { + destination: createTestDestination(thread), + isFinalAttempt: false, + }), + ).rejects.toBeInstanceOf(TurnInputDeferredError); + await expect( + slackRuntime.handleNewMention(thread, message, { + destination: createTestDestination(thread), + isFinalAttempt: false, + }), + ).resolves.toBeUndefined(); + + expect(run).toHaveBeenCalledTimes(1); + expect(advances).toBe(2); + expect(scheduleSessionCompletedPluginTasks).toHaveBeenCalledOnce(); + expect( + (await listAgentTurnSessionSummariesForConversation(thread.id)).find( + (summary) => summary.sessionId === "turn_m-final-recovery", + )?.state, + ).toBe("completed"); + await expect( + getAgentTurnSessionRecord(thread.id, "turn_m-final-recovery"), + ).resolves.toMatchObject({ state: "completed" }); + const lifecycle = ( + await getConversationEventStore().loadHistory(thread.id) + ).filter((event) => event.data.type.startsWith("turn_")); + expect(lifecycle.map((event) => event.data)).toEqual([ + { + type: "turn_started", + turnId: "turn_m-final-recovery", + inputMessageIds: ["m-final-recovery"], + surface: "slack", + }, + { + type: "turn_completed", + turnId: "turn_m-final-recovery", + outcome: "success", + }, + ]); + const persistedConversation = coerceThreadConversationState( + await getPersistedThreadState(thread.id), + ); + await hydrateConversationMessages({ + conversation: persistedConversation, + conversationId: thread.id, + }); + expect(persistedConversation.processing.activeTurnId).toBeUndefined(); + expect( + persistedConversation.messages.find( + (entry) => entry.id === "m-final-recovery", + )?.meta?.replied, + ).toBe(true); + expect(thread.posts).toEqual([]); + }); + + it("recovers an older delivery before newer input and partially acknowledges it", async () => { + const run = vi.fn(); + const thread = createTestThread({ + id: "slack:C0FINAL:1700006009.000", + }); + const destination = createTestDestination(thread); + if (destination.platform !== "slack") { + throw new Error("Expected Slack destination"); + } + const older = { + conversationId: thread.id, + deliveryId: "slack:turn_old-message", + turnId: "turn_old-message", + command: { + completion: { + inputMessageIds: ["old-message"], + model: { modelId: "fake-agent-model" }, + sliceId: 1, + terminal: { outcome: "success" }, + }, + session: { + source: { + platform: "slack", + teamId: destination.teamId, + channelId: destination.channelId, + messageTs: "1700006009.000", + threadTs: "1700006009.000", + }, + destination, + startedAtMs: 1_000, + }, + }, + } as never; + const ack = vi.fn(); + const ackMessageIds = vi.fn(); + const { slackRuntime } = createTestChatRuntime({ + services: { + replyExecutor: { + agentRunner: { run }, + recoverableSlackDelivery: { + loadByConversation: vi.fn(async () => older), + loadByTurn: vi.fn(async () => older), + listDue: vi.fn(async () => []), + loadTerminalOutcome: vi.fn(async () => undefined), + createIntent: vi.fn(), + advance: vi.fn(async () => ({ outcome: "accepted" as const })), + }, + scheduleSessionCompletedPluginTasks: vi.fn(), + }, + }, + }); + const next = createTestMessage({ + id: "new-message", + text: "<@U0APP> newer input", + isMention: true, + threadId: thread.id, + }); + + const error = await slackRuntime + .handleNewMention(thread, next, { + destination, + isFinalAttempt: false, + ack, + ackMessageIds, + }) + .catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(TurnInputDeferredError); + expect((error as TurnInputDeferredError).immediate).toBe(true); + expect(run).not.toHaveBeenCalled(); + expect(ack).not.toHaveBeenCalled(); + expect(ackMessageIds).toHaveBeenCalledWith(["old-message"]); + }); + + it("acknowledges an accepted terminal even when plugin repair fails", async () => { + const thread = createTestThread({ + id: "slack:C0FINAL:1700006013.000", + }); + const ack = vi.fn(); + const run = vi.fn(); + const scheduleSessionCompletedPluginTasks = vi.fn(async () => { + throw new Error("plugin queue unavailable"); + }); + const { slackRuntime } = createTestChatRuntime({ + services: { + replyExecutor: { + agentRunner: { run }, + recoverableSlackDelivery: { + loadByConversation: vi.fn(async () => undefined), + loadByTurn: vi.fn(async () => undefined), + listDue: vi.fn(async () => []), + loadTerminalOutcome: vi.fn(async () => ({ + deliveryOutcome: "accepted" as const, + modelSucceeded: true, + })), + createIntent: vi.fn(), + advance: vi.fn(), + }, + scheduleSessionCompletedPluginTasks, + }, + }, + }); + + await expect( + slackRuntime.handleNewMention( + thread, + createTestMessage({ + id: "prior-terminal", + text: "<@U0APP> already delivered", + isMention: true, + threadId: thread.id, + }), + { destination: createTestDestination(thread), ack }, + ), + ).resolves.toBeUndefined(); + + expect(run).not.toHaveBeenCalled(); + expect(scheduleSessionCompletedPluginTasks).toHaveBeenCalledWith({ + conversationId: thread.id, + sessionId: "turn_prior-terminal", + }); + expect( + scheduleSessionCompletedPluginTasks.mock.invocationCallOrder[0], + ).toBeLessThan(ack.mock.invocationCallOrder[0]!); + }); + + it("repairs the failed outcome after a definitive Slack rejection", async () => { + const ack = vi.fn(); + const recoverableSlackDelivery = new RecoverableSlackDeliveryService( + getSqlExecutor(), + { + post: vi.fn(async () => ({ + outcome: "definitive_failure" as const, + reason: "api_rejected" as const, + })), + reconcile: vi.fn(), + }, + ); + const { slackRuntime } = createTestChatRuntime({ + services: { + replyExecutor: { + agentRunner: { + run: async () => + completedAgentRun({ + text: "Slack will reject this", + diagnostics: makeDiagnostics(), + }), + }, + recoverableSlackDelivery, + }, + }, + }); + const thread = createTestThread({ + id: "slack:C0FINAL:1700006014.000", + }); + (thread.adapter as { name?: string }).name = "slack"; + + await expect( + slackRuntime.handleNewMention( + thread, + createTestMessage({ + id: "definitive-failure", + text: "<@U0APP> reply", + isMention: true, + threadId: thread.id, + }), + { destination: createTestDestination(thread), ack }, + ), + ).resolves.toBeUndefined(); + + expect(ack).toHaveBeenCalledOnce(); + expect(thread.posts).toEqual([]); + expect( + (await listAgentTurnSessionSummariesForConversation(thread.id)).find( + (summary) => summary.sessionId === "turn_definitive-failure", + )?.state, + ).toBe("failed"); + }); + + it("defers callback failure after durable intent without failing the turn", async () => { + const lifecycle = { + start: vi.fn(), + complete: vi.fn(), + fail: vi.fn(), + }; + let intent: unknown; + const recoverableSlackDelivery = { + loadByConversation: vi.fn(async () => undefined), + loadByTurn: vi.fn(async () => intent as never), + listDue: vi.fn(async () => []), + loadTerminalOutcome: vi.fn(async () => undefined), + createIntent: vi.fn(async (args) => { + intent = { + ...args, + nextAttemptAtMs: Date.now(), + command: args.command, + }; + return intent as never; + }), + advance: vi.fn(), + }; + const { slackRuntime } = createTestChatRuntime({ + services: { + replyExecutor: { + agentRunner: { + run: async () => + completedAgentRun({ + text: "reply", + diagnostics: makeDiagnostics(), + }), + }, + recoverableSlackDelivery, + turnLifecycle: lifecycle, + }, + }, + }); + const thread = createTestThread({ + id: "slack:C0FINAL:1700006010.000", + }); + (thread as unknown as { adapter: { name: string } }).adapter = { + name: "slack", + }; + + await expect( + slackRuntime.handleNewMention( + thread, + createTestMessage({ + id: "intent-callback-failure", + text: "<@U0APP> reply", + isMention: true, + threadId: thread.id, + }), + { + destination: createTestDestination(thread), + isFinalAttempt: false, + beforeFirstResponsePost: async () => { + throw new Error("status cleanup failed"); + }, + }, + ), + ).rejects.toBeInstanceOf(TurnInputDeferredError); + + expect(recoverableSlackDelivery.createIntent).toHaveBeenCalledOnce(); + expect(recoverableSlackDelivery.advance).not.toHaveBeenCalled(); + expect(lifecycle.fail).not.toHaveBeenCalled(); + expect(thread.posts).toEqual([]); + }); + + it("acknowledges canvas recovery after a terminal run failure", async () => { + const ack = vi.fn(); + const { slackRuntime } = createTestChatRuntime({ + services: { + replyExecutor: { + agentRunner: { + run: async (request) => { + await request.durability?.onArtifactStateUpdated?.({ + lastCanvasId: "F_CANVAS_RECOVERY", + lastCanvasUrl: "https://slack.example/docs/T/F_CANVAS_RECOVERY", + }); + throw new Error("run interrupted after canvas creation"); + }, + }, + }, + }, + }); + const thread = createTestThread({ + id: "slack:C0FINAL:1700006011.000", + }); + + await slackRuntime.handleNewMention( + thread, + createTestMessage({ + id: "canvas-recovery", + text: "<@U0APP> create a canvas", + isMention: true, + threadId: thread.id, + }), + { destination: createTestDestination(thread), ack }, + ); + + expect(ack).toHaveBeenCalledOnce(); + expect(thread.posts.map(toPostedText).join("\n")).toContain( + "https://slack.example/docs/T/F_CANVAS_RECOVERY", + ); + }); }); diff --git a/packages/junior/tests/integration/slack/outbound-installation-token-contract.test.ts b/packages/junior/tests/integration/slack/outbound-installation-token-contract.test.ts index eaf6d1536..eaa45d2fa 100644 --- a/packages/junior/tests/integration/slack/outbound-installation-token-contract.test.ts +++ b/packages/junior/tests/integration/slack/outbound-installation-token-contract.test.ts @@ -1,6 +1,11 @@ -import { beforeEach, describe, expect, it } from "vitest"; +import type { SlackAdapter } from "@chat-adapter/slack"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { runWithSlackInstallationToken } from "@/chat/slack/client"; -import { postSlackMessage } from "@/chat/slack/outbound"; +import { + createSlackDeliveryLocator, + postSlackMessage, +} from "@/chat/slack/outbound"; +import { createInstallationBoundRecoverableSlackDeliveryPort } from "@/chat/app/services"; import { getCapturedSlackApiCalls, resetSlackApiMockState, @@ -40,4 +45,57 @@ describe("Slack contract: outbound installation token", () => { expect(calls).toHaveLength(1); expect(calls[0]?.headers.authorization).toBe("Bearer xoxb-test-token"); }); + + it("binds recoverable post and reconciliation to the persisted workspace", async () => { + const resolveTokenForTeam = vi.fn(async (teamId: string) => ({ + botUserId: "UDEST", + token: `xoxb-${teamId}`, + })); + const adapter = { + initialize: vi.fn(async () => undefined), + requestContext: { + run: async (_context: unknown, task: () => Promise) => + await task(), + }, + resolveTokenForTeam, + } as unknown as SlackAdapter; + const port = createInstallationBoundRecoverableSlackDeliveryPort({ + getSlackAdapter: () => adapter, + }); + const metadata = { locator: createSlackDeliveryLocator(), partIndex: 0 }; + + await expect( + port.post({ + channelId: "C123", + metadata, + teamId: "TDEST", + text: "recover in the destination workspace", + threadTs: "1718123456.000000", + }), + ).resolves.toMatchObject({ outcome: "accepted" }); + await expect( + port.reconcile({ + channelId: "C123", + metadata, + oldestTs: "1718123400.000000", + teamId: "TDEST", + threadTs: "1718123456.000000", + }), + ).resolves.toEqual({ outcome: "confirmed_absent" }); + + expect(resolveTokenForTeam).toHaveBeenCalledTimes(2); + expect(resolveTokenForTeam).toHaveBeenNthCalledWith(1, "TDEST", undefined); + expect(resolveTokenForTeam).toHaveBeenNthCalledWith(2, "TDEST", undefined); + const providerCalls = [ + ...getCapturedSlackApiCalls("chat.postMessage"), + ...getCapturedSlackApiCalls("auth.test"), + ...getCapturedSlackApiCalls("conversations.replies"), + ]; + expect(providerCalls).toHaveLength(3); + expect(providerCalls.map((call) => call.headers.authorization)).toEqual([ + "Bearer xoxb-TDEST", + "Bearer xoxb-TDEST", + "Bearer xoxb-TDEST", + ]); + }); }); diff --git a/packages/junior/tests/integration/slack/recoverable-outbound-contract.test.ts b/packages/junior/tests/integration/slack/recoverable-outbound-contract.test.ts new file mode 100644 index 000000000..10c461b59 --- /dev/null +++ b/packages/junior/tests/integration/slack/recoverable-outbound-contract.test.ts @@ -0,0 +1,281 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { + parseSlackDeliveryLocator, + postRecoverableSlackMessage, + reconcileRecoverableSlackMessage, + type SlackDeliveryMetadata, +} from "@/chat/slack/outbound"; +import { + getCapturedSlackApiCalls, + queueSlackApiError, + queueSlackApiResponse, + queueSlackRateLimit, + resetSlackApiMockState, +} from "../../msw/handlers/slack-api"; + +const THREAD_TS = "1700000000.000100"; +const OLDEST_TS = "1700000001.000100"; +const DELIVERY_TS = "1700000002.000100"; +const DEFAULT_POSTED_TS = "1700000000.100"; + +function deliveryMetadata(partIndex = 0): SlackDeliveryMetadata { + const locator = parseSlackDeliveryLocator("abcdefghijklmnopqrstuv"); + if (!locator) throw new Error("Test delivery locator must be valid"); + return { locator, partIndex }; +} + +function recoveredMessage( + input: { + metadata?: Record; + botId?: string; + userId?: string; + ts?: string; + } = {}, +): Record { + return { + ts: input.ts ?? DELIVERY_TS, + thread_ts: THREAD_TS, + text: "delivered text is unrelated to the marker", + ...(input.botId === undefined ? { bot_id: "B_TEST_BOT" } : {}), + ...(input.botId ? { bot_id: input.botId } : {}), + ...(input.userId ? { user: input.userId } : {}), + metadata: input.metadata ?? { + event_type: "junior_delivery", + event_payload: { + locator: "abcdefghijklmnopqrstuv", + part_index: 0, + }, + }, + }; +} + +describe("Slack contract: recoverable outbound delivery", () => { + beforeEach(() => { + process.env.SLACK_BOT_TOKEN = + process.env.SLACK_BOT_TOKEN ?? "xoxb-test-token"; + resetSlackApiMockState(); + }); + + it("posts once with only the fixed public-safe delivery metadata", async () => { + const result = await postRecoverableSlackMessage({ + channelId: "slack:C123", + threadTs: THREAD_TS, + text: "hello", + metadata: deliveryMetadata(2), + }); + + expect(result).toEqual({ outcome: "accepted", ts: DEFAULT_POSTED_TS }); + const calls = getCapturedSlackApiCalls("chat.postMessage"); + expect(calls).toHaveLength(1); + expect(calls[0]?.params).toEqual({ + channel: "C123", + thread_ts: THREAD_TS, + text: "hello", + metadata: { + event_type: "junior_delivery", + event_payload: { + locator: "abcdefghijklmnopqrstuv", + part_index: 2, + }, + }, + }); + expect(JSON.stringify(calls[0]?.params.metadata)).not.toMatch( + /conversation|channel|thread|turn|message|text|authorization/i, + ); + }); + + it.each([ + ["Slack 5xx", { status: 503, body: "unavailable" }, "server_error"], + ["connection reset", { networkError: true }, "transport_error"], + ] as const)( + "does not retry an ambiguous %s write", + async (_label, response, reason) => { + queueSlackApiResponse("chat.postMessage", response); + const result = await postRecoverableSlackMessage({ + channelId: "C123", + threadTs: THREAD_TS, + text: "hello", + metadata: deliveryMetadata(), + }); + + expect(result).toEqual({ outcome: "uncertain", reason }); + expect(getCapturedSlackApiCalls("chat.postMessage")).toHaveLength(1); + }, + ); + + it("classifies an explicit Slack rejection as definitive without retrying", async () => { + queueSlackApiError("chat.postMessage", { error: "not_in_channel" }); + + const result = await postRecoverableSlackMessage({ + channelId: "C123", + text: "hello", + metadata: deliveryMetadata(), + }); + + expect(result).toEqual({ + outcome: "definitive_failure", + reason: "api_rejected", + }); + expect(getCapturedSlackApiCalls("chat.postMessage")).toHaveLength(1); + }); + + it("reconciles an exact marker authored by the current Slack bot", async () => { + queueSlackApiResponse("conversations.replies", { + body: { ok: true, messages: [recoveredMessage()] }, + }); + + await expect( + reconcileRecoverableSlackMessage({ + channelId: "slack:C123", + threadTs: THREAD_TS, + oldestTs: OLDEST_TS, + metadata: deliveryMetadata(), + }), + ).resolves.toEqual({ outcome: "accepted", ts: DELIVERY_TS }); + + expect( + getCapturedSlackApiCalls("conversations.replies")[0]?.params, + ).toEqual({ + channel: "C123", + ts: THREAD_TS, + oldest: OLDEST_TS, + inclusive: "true", + include_all_metadata: "true", + limit: "15", + }); + }); + + it("ignores a forged human marker and metadata with extra payload fields", async () => { + queueSlackApiResponse("conversations.replies", { + body: { + ok: true, + messages: [ + recoveredMessage({ botId: "", userId: "U_HUMAN" }), + recoveredMessage({ + metadata: { + event_type: "junior_delivery", + event_payload: { + locator: "abcdefghijklmnopqrstuv", + part_index: 0, + version: 1, + conversation_id: "private-value", + }, + }, + }), + ], + }, + }); + + await expect( + reconcileRecoverableSlackMessage({ + channelId: "C123", + threadTs: THREAD_TS, + oldestTs: OLDEST_TS, + metadata: deliveryMetadata(), + }), + ).resolves.toEqual({ outcome: "confirmed_absent" }); + }); + + it("returns a cursor for a later rate-limited page invocation", async () => { + queueSlackApiResponse("conversations.replies", { + body: { + ok: true, + messages: [], + response_metadata: { next_cursor: "page-2" }, + }, + }); + await expect( + reconcileRecoverableSlackMessage({ + channelId: "C123", + threadTs: THREAD_TS, + oldestTs: OLDEST_TS, + metadata: deliveryMetadata(), + }), + ).resolves.toEqual({ outcome: "continue", nextCursor: "page-2" }); + expect(getCapturedSlackApiCalls("conversations.replies")).toHaveLength(1); + + queueSlackApiResponse("conversations.replies", { + body: { ok: true, messages: [recoveredMessage()] }, + }); + await expect( + reconcileRecoverableSlackMessage({ + channelId: "C123", + threadTs: THREAD_TS, + oldestTs: OLDEST_TS, + metadata: deliveryMetadata(), + cursor: "page-2", + }), + ).resolves.toEqual({ outcome: "accepted", ts: DELIVERY_TS }); + expect(getCapturedSlackApiCalls("conversations.replies")).toHaveLength(2); + expect( + getCapturedSlackApiCalls("conversations.replies")[1]?.params.cursor, + ).toBe("page-2"); + }); + + it("carries Retry-After when reconciliation is rate limited", async () => { + const before = Date.now(); + queueSlackRateLimit("conversations.replies"); + + const result = await reconcileRecoverableSlackMessage({ + channelId: "C123", + threadTs: THREAD_TS, + oldestTs: OLDEST_TS, + metadata: deliveryMetadata(), + }); + + expect(result).toMatchObject({ outcome: "retryable" }); + expect( + result.outcome === "retryable" ? result.retryAtMs : undefined, + ).toBeGreaterThanOrEqual(before); + expect(getCapturedSlackApiCalls("conversations.replies")).toHaveLength(1); + }); + + it("fails closed when reconciliation is missing scope", async () => { + queueSlackApiError("conversations.replies", { + error: "missing_scope", + needed: "channels:history", + }); + await expect( + reconcileRecoverableSlackMessage({ + channelId: "C123", + threadTs: THREAD_TS, + oldestTs: OLDEST_TS, + metadata: deliveryMetadata(), + }), + ).resolves.toEqual({ + outcome: "unresolved", + reason: "permanent_provider_error", + providerErrorCode: "missing_scope", + }); + }); + + it("confirms absence only after a successful final page with no match", async () => { + queueSlackApiResponse("conversations.replies", { + body: { ok: true, messages: [] }, + }); + + await expect( + reconcileRecoverableSlackMessage({ + channelId: "C123", + threadTs: THREAD_TS, + oldestTs: OLDEST_TS, + metadata: deliveryMetadata(), + }), + ).resolves.toEqual({ outcome: "confirmed_absent" }); + }); + + it("fails closed when Slack reports more pages without a cursor", async () => { + queueSlackApiResponse("conversations.replies", { + body: { ok: true, messages: [], has_more: true }, + }); + + await expect( + reconcileRecoverableSlackMessage({ + channelId: "C123", + threadTs: THREAD_TS, + oldestTs: OLDEST_TS, + metadata: deliveryMetadata(), + }), + ).resolves.toEqual({ outcome: "unresolved" }); + }); +}); diff --git a/packages/junior/tests/msw/handlers/eval-mcp-auth.ts b/packages/junior/tests/msw/handlers/eval-mcp-auth.ts index 6ff491daf..517502a38 100644 --- a/packages/junior/tests/msw/handlers/eval-mcp-auth.ts +++ b/packages/junior/tests/msw/handlers/eval-mcp-auth.ts @@ -12,6 +12,7 @@ const EVAL_MCP_TOKEN_ENDPOINT = `${EVAL_MCP_AUTH_ORIGIN}/oauth/token`; const EVAL_MCP_REGISTRATION_ENDPOINT = `${EVAL_MCP_AUTH_ORIGIN}/oauth/register`; const EVAL_MCP_ACCESS_TOKEN = "eval-auth-access-token"; const EVAL_MCP_SESSION_ID = "eval-auth-session"; +let authorizationCodeExchanges = 0; function unauthorizedResponse() { return new HttpResponse(null, { @@ -35,7 +36,14 @@ function jsonRpcResult(id: unknown, result: unknown, headers?: HeadersInit) { ); } -export function resetEvalMcpAuthMockState(): void {} +export function resetEvalMcpAuthMockState(): void { + authorizationCodeExchanges = 0; +} + +/** Return the number of authorization-code token exchanges. */ +export function readEvalMcpAuthorizationCodeExchanges(): number { + return authorizationCodeExchanges; +} export const evalMcpAuthHandlers = [ http.get( @@ -360,6 +368,7 @@ export const evalMcpAuthHandlers = [ const bodyText = await request.text(); const params = new URLSearchParams(bodyText); const code = params.get("code"); + authorizationCodeExchanges += 1; if (code !== EVAL_MCP_AUTH_CODE) { return HttpResponse.json( { diff --git a/packages/junior/tests/msw/handlers/slack-api.ts b/packages/junior/tests/msw/handlers/slack-api.ts index a52388085..c43ddab59 100644 --- a/packages/junior/tests/msw/handlers/slack-api.ts +++ b/packages/junior/tests/msw/handlers/slack-api.ts @@ -69,6 +69,7 @@ export interface SlackMockHttpResponse { status?: number; headers?: Record; body?: Record | string; + networkError?: boolean; } export interface CapturedSlackApiCall { @@ -270,6 +271,9 @@ function dequeueResponse(key: string): SlackMockHttpResponse | undefined { } function toHttpResponse(response: SlackMockHttpResponse): Response { + if (response.networkError) { + return HttpResponse.error(); + } const status = response.status ?? 200; const headers = response.headers; diff --git a/packages/junior/tests/unit/build/nitro-plugin-module.test.ts b/packages/junior/tests/unit/build/nitro-plugin-module.test.ts index b00f44baf..1531de26e 100644 --- a/packages/junior/tests/unit/build/nitro-plugin-module.test.ts +++ b/packages/junior/tests/unit/build/nitro-plugin-module.test.ts @@ -6,6 +6,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { PLUGIN_TASK_QUEUE_TOPIC } from "@/chat/plugins/task-queue"; import { DEFAULT_CONVERSATION_WORK_QUEUE_TOPIC } from "@/chat/task-execution/vercel-queue"; import { + JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE, JUNIOR_CONVERSATION_WORK_CALLBACK_ROUTE, JUNIOR_HEARTBEAT_CRON_SCHEDULE, JUNIOR_HEARTBEAT_ROUTE, @@ -107,6 +108,11 @@ describe("juniorNitro plugin modules", () => { expect(vercel.functions).toEqual({ maxDuration: 300, }); + expect( + vercel.functionRules?.[JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE], + ).toEqual({ + maxDuration: 300, + }); expect( vercel.functionRules?.[JUNIOR_CONVERSATION_WORK_CALLBACK_ROUTE], ).toEqual({ @@ -155,6 +161,9 @@ describe("juniorNitro plugin modules", () => { memory: 1024, }, functionRules: { + [JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE]: { + memory: 1536, + }, [JUNIOR_CONVERSATION_WORK_CALLBACK_ROUTE]: { memory: 2048, experimentalTriggers: [ @@ -179,7 +188,7 @@ describe("juniorNitro plugin modules", () => { }, }; - juniorNitro({ maxDuration: 300 }).nitro.setup(nitro); + juniorNitro().nitro.setup(nitro); const vercel = getVercelOptions(nitro); expect(vercel.config?.crons).toEqual([ @@ -196,6 +205,12 @@ describe("juniorNitro plugin modules", () => { maxDuration: 120, memory: 1024, }); + expect( + vercel.functionRules?.[JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE], + ).toEqual({ + maxDuration: 120, + memory: 1536, + }); expect( vercel.functionRules?.[JUNIOR_CONVERSATION_WORK_CALLBACK_ROUTE], ).toEqual({ @@ -312,7 +327,7 @@ describe("juniorNitro plugin modules", () => { ]); }); - it("preserves Vercel max function duration settings", () => { + it("rejects non-numeric Vercel max function duration settings", () => { const virtual: Record Promise) | string> = {}; const nitro = { hooks: { @@ -332,18 +347,85 @@ describe("juniorNitro plugin modules", () => { }, }; - juniorNitro().nitro.setup(nitro); - const vercel = getVercelOptions(nitro); + expect(() => juniorNitro().nitro.setup(nitro)).toThrow( + 'Junior requires a numeric Vercel maxDuration so runtime leases match the host window. Configure juniorNitro({ maxDuration: }) instead of maxDuration: "max".', + ); + }); - expect( - vercel.functionRules?.[JUNIOR_CONVERSATION_WORK_CALLBACK_ROUTE] - ?.maxDuration, - ).toBe("max"); - expect( - vercel.functionRules?.[JUNIOR_PLUGIN_TASK_CALLBACK_ROUTE]?.maxDuration, - ).toBe("max"); + it("rejects conflicting explicit and existing function durations", () => { + const nitro = { + hooks: { hook() {} }, + options: { + output: { serverDir: "/tmp/junior-output" }, + rootDir: "/tmp/junior-app", + vercel: { functions: { maxDuration: 120 } }, + virtual: {}, + }, + }; + + expect(() => juniorNitro({ maxDuration: 500 }).nitro.setup(nitro)).toThrow( + "juniorNitro({ maxDuration: 500 }) conflicts with Nitro Vercel functions.maxDuration 120. Configure the duration once.", + ); + }); + + it("rejects a conflicting agent dispatch function duration", () => { + const nitro = { + hooks: { hook() {} }, + options: { + output: { serverDir: "/tmp/junior-output" }, + rootDir: "/tmp/junior-app", + vercel: { + functions: { maxDuration: 300 }, + functionRules: { + [JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE]: { maxDuration: 120 }, + }, + }, + virtual: {}, + }, + }; + + expect(() => juniorNitro().nitro.setup(nitro)).toThrow( + `Vercel function rule ${JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE} maxDuration 120 conflicts with Junior's configured duration 300. Configure the duration once with juniorNitro({ maxDuration }).`, + ); }); + it.each([120, 500])( + "injects one custom %ds duration into deployment and runtime config", + async (maxDuration) => { + const virtual: Record Promise) | string> = {}; + const nitro = { + hooks: { hook() {} }, + options: { + output: { serverDir: "/tmp/junior-output" }, + rootDir: "/tmp/junior-app", + vercel: {}, + virtual, + }, + }; + + juniorNitro({ maxDuration }).nitro.setup(nitro); + const vercel = getVercelOptions(nitro); + expect(vercel.functions).toEqual({ maxDuration }); + expect( + vercel.functionRules?.[JUNIOR_AGENT_DISPATCH_CALLBACK_ROUTE] + ?.maxDuration, + ).toBe(maxDuration); + expect( + vercel.functionRules?.[JUNIOR_CONVERSATION_WORK_CALLBACK_ROUTE] + ?.maxDuration, + ).toBe(maxDuration); + expect( + vercel.functionRules?.[JUNIOR_PLUGIN_TASK_CALLBACK_ROUTE]?.maxDuration, + ).toBe(maxDuration); + + const template = virtual["#junior/config"]; + expect(typeof template).toBe("function"); + await expect((template as () => Promise)()).resolves.toContain( + `export const functionMaxDurationSeconds = ${maxDuration};`, + ); + }, + ); + it("loads plugin modules lazily when virtual config is rendered", async () => { const tempRoot = await makeTempDir(); await fs.writeFile( diff --git a/packages/junior/tests/unit/handlers/oauth-resume.test.ts b/packages/junior/tests/unit/handlers/oauth-resume.test.ts index 6e3464651..c2ed7dba8 100644 --- a/packages/junior/tests/unit/handlers/oauth-resume.test.ts +++ b/packages/junior/tests/unit/handlers/oauth-resume.test.ts @@ -259,11 +259,18 @@ describe("resumeAuthorizedRequest", () => { it("schedules plugin tasks after a successful resumed turn", async () => { const scheduleSessionCompletedPluginTasks = vi.fn(async () => undefined); + const turnLifecycle = { + start: vi.fn(async () => undefined), + complete: vi.fn(async () => undefined), + fail: vi.fn(async () => undefined), + }; await resumeSlackTurn({ messageText: "continue this turn", channelId: "C-test", threadTs: "1700000000.0006", + inputMessageIds: ["1700000000.0006"], + turnLifecycle, replyContext: { routing: { credentialContext: { @@ -300,6 +307,9 @@ describe("resumeAuthorizedRequest", () => { conversationId: "slack:T-test:C-test:1700000000.0006", sessionId: "turn_1700000000_0006", }); + expect(turnLifecycle.start).toHaveBeenCalledOnce(); + expect(turnLifecycle.complete).toHaveBeenCalledOnce(); + expect(turnLifecycle.fail).not.toHaveBeenCalled(); }); it("releases the thread lock before scheduling another timeout slice", async () => { diff --git a/packages/junior/tests/unit/runtime/agent-dispatch-signing.test.ts b/packages/junior/tests/unit/runtime/agent-dispatch-signing.test.ts index dca196b1b..bc499228d 100644 --- a/packages/junior/tests/unit/runtime/agent-dispatch-signing.test.ts +++ b/packages/junior/tests/unit/runtime/agent-dispatch-signing.test.ts @@ -28,6 +28,7 @@ describe("agent dispatch callback signing", () => { await scheduleDispatchCallback({ id: "dispatch_123", expectedVersion: 3, + kind: "delivery", }); expect(fetchMock).toHaveBeenCalledTimes(1); @@ -42,6 +43,7 @@ describe("agent dispatch callback signing", () => { await expect(verifyDispatchCallbackRequest(request)).resolves.toEqual({ id: "dispatch_123", expectedVersion: 3, + kind: "delivery", }); }); diff --git a/packages/junior/tests/unit/runtime/delivered-turn-state.test.ts b/packages/junior/tests/unit/runtime/delivered-turn-state.test.ts index 1b0c85323..e9a5af64a 100644 --- a/packages/junior/tests/unit/runtime/delivered-turn-state.test.ts +++ b/packages/junior/tests/unit/runtime/delivered-turn-state.test.ts @@ -1,5 +1,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { buildDeliveredTurnStatePatch } from "@/chat/runtime/delivered-turn-state"; +import { + buildDeliveredTurnStatePatch, + buildRecoveredDeliveredTurnStatePatch, +} from "@/chat/runtime/delivered-turn-state"; import { coerceThreadArtifactsState } from "@/chat/state/artifacts"; import { coerceThreadConversationState } from "@/chat/state/conversation"; @@ -58,26 +61,38 @@ describe("delivered turn state", () => { ]); }); - it("does not invent an assistant message for intentional silence", () => { - const state = buildDeliveredTurnStatePatch({ - artifacts: coerceThreadArtifactsState({}), - conversation: coerceThreadConversationState({}), - reply: { - text: "", - deliveryPlan: { mode: "thread", postThreadText: false }, - diagnostics: { - assistantMessageCount: 1, - modelId: "test/model", - outcome: "success", - toolCalls: [], - toolErrorCount: 0, - toolResultCount: 0, - usedPrimaryText: true, + it("repairs derived conversation state from a canonical terminal", () => { + const conversation = coerceThreadConversationState({ + conversation: { + processing: { + activeTurnId: "turn-1", + pendingAuth: { + kind: "plugin", + provider: "github", + actorId: "U123", + sessionId: "turn-1", + linkSentAtMs: 1, + }, }, }, - sessionId: "turn_silent", + }); + conversation.messages.push({ + id: "message-1", + role: "user", + text: "continue", + createdAtMs: 1, }); - expect(state.conversation.messages).toEqual([]); + const repaired = buildRecoveredDeliveredTurnStatePatch({ + conversation, + sessionId: "turn-1", + inputMessageIds: ["message-1"], + }); + + expect(repaired.conversation.processing).toMatchObject({ + activeTurnId: undefined, + pendingAuth: undefined, + }); + expect(repaired.conversation.messages[0]?.meta?.replied).toBe(true); }); }); diff --git a/packages/junior/tests/unit/slack/recoverable-outbound-classification.test.ts b/packages/junior/tests/unit/slack/recoverable-outbound-classification.test.ts new file mode 100644 index 000000000..5c99eec69 --- /dev/null +++ b/packages/junior/tests/unit/slack/recoverable-outbound-classification.test.ts @@ -0,0 +1,56 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + postMessage: vi.fn(), +})); + +vi.mock("@/chat/slack/client", () => ({ + SlackActionError: class SlackActionError extends Error {}, + getSlackClient: () => ({ + chat: { postMessage: mocks.postMessage }, + }), + normalizeSlackConversationId: (value: string) => value, + withSlackRetries: vi.fn(), +})); + +import { + parseSlackDeliveryLocator, + postRecoverableSlackMessage, + type SlackDeliveryMetadata, +} from "@/chat/slack/outbound"; + +function deliveryMetadata(): SlackDeliveryMetadata { + const locator = parseSlackDeliveryLocator("abcdefghijklmnopqrstuv"); + if (!locator) throw new Error("Test delivery locator must be valid"); + return { locator, partIndex: 0 }; +} + +describe("recoverable Slack post failure classification", () => { + beforeEach(() => { + mocks.postMessage.mockReset(); + }); + + it.each([ + ["timeout", { code: "ETIMEDOUT", message: "request timed out" }], + ["reset", { code: "ECONNRESET", message: "socket hang up" }], + ["socket hangup", new Error("socket hang up")], + ])( + "makes one transport attempt for an ambiguous %s", + async (_name, error) => { + mocks.postMessage.mockRejectedValueOnce(error); + + await expect( + postRecoverableSlackMessage({ + channelId: "C123", + threadTs: "1700000000.000100", + text: "hello", + metadata: deliveryMetadata(), + }), + ).resolves.toEqual({ + outcome: "uncertain", + reason: "transport_error", + }); + expect(mocks.postMessage).toHaveBeenCalledTimes(1); + }, + ); +}); diff --git a/packages/junior/tests/unit/slack/tool-registration.test.ts b/packages/junior/tests/unit/slack/tool-registration.test.ts index 25cdc6319..98bf47d09 100644 --- a/packages/junior/tests/unit/slack/tool-registration.test.ts +++ b/packages/junior/tests/unit/slack/tool-registration.test.ts @@ -140,6 +140,22 @@ describe("Slack tool registration", () => { expect(tools).toHaveProperty("slackThreadRead"); }); + it("omits sendMessage when runtime delivery owns an internal dispatch", () => { + const tools = createTools( + [], + {}, + { + ...ctx("C12345"), + conversationId: "agent-dispatch:dispatch-123", + surface: "api", + }, + ); + + expect(tools).not.toHaveProperty("sendMessage"); + expect(tools).toHaveProperty("slackChannelListMessages"); + expect(tools).toHaveProperty("slackThreadRead"); + }); + it("registers delivery tools from assistant context channel in DM turns", () => { const tools = createTools( [], diff --git a/packages/junior/tests/unit/state/authorization-recovery-index.test.ts b/packages/junior/tests/unit/state/authorization-recovery-index.test.ts new file mode 100644 index 000000000..b654026c0 --- /dev/null +++ b/packages/junior/tests/unit/state/authorization-recovery-index.test.ts @@ -0,0 +1,81 @@ +import { createMemoryState } from "@chat-adapter/state-memory"; +import type { StateAdapter } from "chat"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + AUTHORIZATION_RECOVERY_INDEX_MAX_ENTRIES, + listAuthorizationRecoveries, + registerAuthorizationRecovery, +} from "@/chat/state/authorization-recovery-index"; + +const INDEX_KEY = "junior:agent_turn_authorization_recovery:index"; + +describe("authorization recovery index", () => { + let state: StateAdapter; + + beforeEach(async () => { + state = createMemoryState(); + await state.connect(); + }); + + afterEach(async () => { + await state.disconnect(); + }); + + it("rejects a new entry at capacity without evicting live entries", async () => { + const registeredAtMs = Date.now(); + await state.set( + INDEX_KEY, + Array.from( + { length: AUTHORIZATION_RECOVERY_INDEX_MAX_ENTRIES }, + (_, index) => ({ + authorizationCompletionId: `completion-${index}`, + conversationId: `conversation-${index}`, + registeredAtMs, + sessionId: `session-${index}`, + }), + ), + ); + + await registerAuthorizationRecovery(state, { + authorizationCompletionId: "completion-0", + conversationId: "conversation-0", + registeredAtMs: registeredAtMs + 1, + sessionId: "session-0", + }); + await expect( + registerAuthorizationRecovery(state, { + authorizationCompletionId: "overflow-completion", + conversationId: "overflow-conversation", + registeredAtMs, + sessionId: "overflow-session", + }), + ).rejects.toThrow("Authorization recovery index is at capacity"); + + const entries = await listAuthorizationRecoveries(state); + expect(entries).toHaveLength(AUTHORIZATION_RECOVERY_INDEX_MAX_ENTRIES); + expect(entries[0]).toMatchObject({ registeredAtMs: registeredAtMs + 1 }); + expect(entries).not.toContainEqual( + expect.objectContaining({ + authorizationCompletionId: "overflow-completion", + }), + ); + }); + + it("rejects malformed or oversized persisted indexes", async () => { + await state.set( + INDEX_KEY, + Array.from( + { length: AUTHORIZATION_RECOVERY_INDEX_MAX_ENTRIES + 1 }, + () => ({}), + ), + ); + await expect(listAuthorizationRecoveries(state)).rejects.toThrow( + "Authorization recovery index exceeds capacity", + ); + + await state.set(INDEX_KEY, [{}]); + await expect(listAuthorizationRecoveries(state)).rejects.toThrow( + "Authorization recovery index is malformed", + ); + }); +});