diff --git a/packages/junior-evals/evals/core/lifecycle-and-resilience.eval.ts b/packages/junior-evals/evals/core/lifecycle-and-resilience.eval.ts index d37e61ed7..f050384f5 100644 --- a/packages/junior-evals/evals/core/lifecycle-and-resilience.eval.ts +++ b/packages/junior-evals/evals/core/lifecycle-and-resilience.eval.ts @@ -59,6 +59,47 @@ describeEval("Lifecycle and Resilience", slackEvals, (it) => { expect(visibleThreadReplies(result.session)).toHaveLength(1); }); + it("when xAI returns a generic 503, explain the temporary outage without leaking provider details", async ({ + run, + }) => { + const productionServiceUnavailableError = + '503 {"type":"error","error":{"type":"service_unavailable_error","message":"Service temporarily unavailable"},"providerMetadata":{"gateway":{"originalModelId":"xai/grok-4.5","resolvedProvider":"xai","fallbacksAvailable":[],"modelAttemptCount":1,"totalProviderAttemptCount":1}}}'; + const result = await run({ + overrides: { + reply_results: [ + { + assistant_message_count: 0, + error_message: productionServiceUnavailableError, + outcome: "provider_error", + stop_reason: "error", + text: "", + }, + ], + }, + initialEvents: [mention("Create a Linear ticket for this incident")], + criteria: rubric({ + pass: [ + "The reply neutrally explains that the model provider had a temporary problem and asks the user to try again.", + "The reply includes a reference event ID for support.", + ], + fail: [ + "Do not expose xAI, Grok, gateway metadata, raw JSON, HTTP status codes, or provider exception text.", + "Do not describe the failure as an unexplained internal error.", + ], + }), + }); + + const replies = visibleThreadReplies(result.session); + expect(replies).toHaveLength(1); + const text = textContent(replies[0]?.content); + expect(text).toContain("temporary connection problem"); + expect(text).not.toContain("capacity"); + expect(text).toMatch(/event_id=[a-f0-9]+/); + expect(text).not.toContain("xai"); + expect(text).not.toContain("service_unavailable_error"); + expect(text).not.toContain("503"); + }); + it("when a short reply is interrupted by the provider, keep the partial answer in one marked post", async ({ run, }) => { diff --git a/packages/junior-evals/src/behavior-harness.ts b/packages/junior-evals/src/behavior-harness.ts index 1f107c2b7..0b0c3809c 100644 --- a/packages/junior-evals/src/behavior-harness.ts +++ b/packages/junior-evals/src/behavior-harness.ts @@ -52,6 +52,7 @@ import { appendAndEnqueueInboundMessage } from "@/chat/task-execution/store"; import { executeAgentRun } from "@/chat/agent"; import { completedAgentRun } from "@/chat/runtime/agent-run-outcome"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; +import { createProviderError } from "@/chat/services/provider-error"; import { addAgentTurnUsage, type AgentTurnUsage } from "@/chat/usage"; import { resumeAwaitingSlackContinuation } from "@/chat/runtime/agent-continue-runner"; import { scheduleAgentContinue } from "@/chat/services/agent-continue"; @@ -1927,6 +1928,15 @@ function buildRuntimeServices( ...(replyResult.stop_reason ? { stopReason: replyResult.stop_reason } : {}), + ...(replyResult.outcome === "provider_error" && + replyResult.error_message + ? { + providerError: createProviderError( + replyResult.error_message, + { modelId: "eval-reply-result" }, + ), + } + : {}), toolCalls: replyResult.tool_calls ?? [], toolErrorCount: replyResult.tool_error_count ?? 0, toolResultCount: replyResult.tool_result_count ?? 0, diff --git a/packages/junior/src/chat/agent/index.ts b/packages/junior/src/chat/agent/index.ts index 87fbd2f48..157dbae45 100644 --- a/packages/junior/src/chat/agent/index.ts +++ b/packages/junior/src/chat/agent/index.ts @@ -9,7 +9,6 @@ * stay outside this module. */ import { Agent, type AgentLoopTurnUpdate } from "@earendil-works/pi-agent-core"; -import { isRetryableAssistantError } from "@earendil-works/pi-ai"; import type { FileUpload } from "chat"; import { botConfig } from "@/chat/config"; import { @@ -64,10 +63,8 @@ import { shouldEmitDevAgentTrace } from "@/chat/runtime/dev-agent-trace"; import { isTurnInputCommitLostError } from "@/chat/runtime/turn"; import type { AgentRunOutcome } from "@/chat/runtime/agent-run-outcome"; import { buildTurnResult } from "@/chat/services/turn-result"; -import { - isProviderRetryError, - nextProviderRetry, -} from "@/chat/services/provider-retry"; +import { nextProviderRetry } from "@/chat/services/provider-retry"; +import { isProviderRetryError } from "@/chat/services/provider-error"; import { configuredTurnReasoningLevel, selectTurnReasoningLevel, @@ -1022,10 +1019,8 @@ async function executeAgentRunInPrivacyContext( const providerRetry = nextProviderRetry({ attempt, + failure: lastAssistant, messages: agent!.state.messages, - retryableFailure: - lastAssistant !== undefined && - isRetryableAssistantError(lastAssistant), }); if (!providerRetry) { break; diff --git a/packages/junior/src/chat/pi/client.ts b/packages/junior/src/chat/pi/client.ts index 3ee714f00..4876466b1 100644 --- a/packages/junior/src/chat/pi/client.ts +++ b/packages/junior/src/chat/pi/client.ts @@ -44,7 +44,7 @@ import { import { createProviderError, isProviderRetryError, -} from "@/chat/services/provider-retry"; +} from "@/chat/services/provider-error"; import { hasCompactedConversationContext } from "@/chat/services/context-compaction-marker"; const GATEWAY_PROVIDER = "vercel-ai-gateway" as const; diff --git a/packages/junior/src/chat/runtime/slack-runtime.ts b/packages/junior/src/chat/runtime/slack-runtime.ts index 92d5b59d8..e317edaff 100644 --- a/packages/junior/src/chat/runtime/slack-runtime.ts +++ b/packages/junior/src/chat/runtime/slack-runtime.ts @@ -9,7 +9,7 @@ import type { Message, MessageContext, Thread } from "chat"; import type { Destination } from "@sentry/junior-plugin-api"; import { getSubscribedReplyPreflightDecision } from "@/chat/services/subscribed-decision"; -import { isProviderRetryError } from "@/chat/services/provider-retry"; +import { isProviderRetryError } from "@/chat/services/provider-error"; import { AuthorizationFlowDisabledError } from "@/chat/services/auth-pause"; import { SlackActionError } from "@/chat/slack/client"; import { diff --git a/packages/junior/src/chat/services/provider-error.ts b/packages/junior/src/chat/services/provider-error.ts new file mode 100644 index 000000000..40caa2392 --- /dev/null +++ b/packages/junior/src/chat/services/provider-error.ts @@ -0,0 +1,244 @@ +export type ProviderErrorKind = + | "auth" + | "permission" + | "rate_limit" + | "capacity" + | "timeout" + | "network" + | "server" + | "invalid_request" + | "quota" + | "content_policy" + | "unknown"; + +interface ProviderErrorContext { + modelId?: string; +} + +interface ProviderErrorDetails extends ProviderErrorContext { + kind: ProviderErrorKind; + status?: number; + retryable: boolean; + retryAfterMs?: number; + fallbackEligible: boolean; +} + +/** Structured failure produced at an AI provider boundary. */ +export class ProviderError extends Error { + readonly code = "provider_error"; + readonly kind: ProviderErrorKind; + readonly status?: number; + readonly modelId?: string; + readonly retryable: boolean; + readonly retryAfterMs?: number; + readonly fallbackEligible: boolean; + + constructor(message: string, details: ProviderErrorDetails, cause?: unknown) { + super(`AI provider error: ${message || "Unknown provider error"}`, { + cause, + }); + this.name = "ProviderError"; + this.kind = details.kind; + this.status = details.status; + this.modelId = details.modelId; + this.retryable = details.retryable; + this.retryAfterMs = details.retryAfterMs; + this.fallbackEligible = details.fallbackEligible; + } +} + +type ErrorShape = Error & { + status?: unknown; + statusCode?: unknown; + responseHeaders?: unknown; + response?: { headers?: unknown }; +}; + +function providerMessage(error: unknown): string { + return (error instanceof Error ? error.message : String(error)).trim(); +} + +function extractStatus(error: unknown, message: string): number | undefined { + if (error instanceof Error) { + const shaped = error as ErrorShape; + if (typeof shaped.status === "number") return shaped.status; + if (typeof shaped.statusCode === "number") return shaped.statusCode; + } + const match = message.match( + /^(?:Error:\s*)?([45]\d\d)\b|\bstatus(?:Code)?["'=:\s]+([45]\d\d)\b/i, + ); + const status = match?.[1] ?? match?.[2]; + return status ? Number(status) : undefined; +} + +function extractField(message: string, field: string): string | undefined { + return message.match(new RegExp(`"${field}"\\s*:\\s*"([^"]+)"`, "i"))?.[1]; +} + +function extractRetryAfterMs( + error: unknown, + message: string, +): number | undefined { + const headers = + error instanceof Error + ? ((error as ErrorShape).responseHeaders ?? + (error as ErrorShape).response?.headers) + : undefined; + const retryAfter = + headers instanceof Headers + ? headers.get("retry-after") + : headers && typeof headers === "object" + ? Object.entries(headers as Record).find( + ([name]) => name.toLowerCase() === "retry-after", + )?.[1] + : undefined; + const raw = + typeof retryAfter === "string" || typeof retryAfter === "number" + ? String(retryAfter) + : message.match(/"retry-after"\s*:\s*"?([^",}]+)"?/i)?.[1]; + if (!raw) return undefined; + const seconds = Number(raw); + if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1_000; + const date = Date.parse(raw); + return Number.isNaN(date) ? undefined : Math.max(0, date - Date.now()); +} + +function classifyProviderError( + status: number | undefined, + message: string, +): ProviderErrorKind { + if (/content.?policy|safety|moderation/i.test(message)) + return "content_policy"; + if ( + /insufficient.?quota|quota exceeded|usage limit|monthly usage limit|available balance|out of budget|billing/i.test( + message, + ) + ) + return "quota"; + if ( + status === 401 || + /invalid.?api.?key|no api key|authentication|(?:invalid|missing|expired|revoked).{0,24}\bcredentials?\b|\bcredentials?\b.{0,24}(?:invalid|missing|expired|revoked)|\bno\b.{0,16}\bcredentials?\b/i.test( + message, + ) + ) + return "auth"; + if (status === 403 || /permission|forbidden|authorization/i.test(message)) + return "permission"; + if (status === 429 || /rate.?limit|too many requests/i.test(message)) + return "rate_limit"; + if (/overloaded|at capacity|capacity exceeded/i.test(message)) + return "capacity"; + if (/timed? out|timeout/i.test(message)) return "timeout"; + if ( + /network.?error|connection|fetch failed|socket|stream ended|ended before|ECONNRESET/i.test( + message, + ) + ) + return "network"; + if (status !== undefined && status >= 500) return "server"; + if ( + /service.?unavailable|internal server error|bad gateway|gateway timeout|upstream (?:request )?failed|unexpected error occurred/i.test( + message, + ) + ) + return "server"; + if (status !== undefined && status >= 400) return "invalid_request"; + if ( + /context.?length|context.?window|validation|bad request|unsupported model|invalid model|unknown (?:ai gateway )?model|mismatched api/i.test( + message, + ) + ) + return "invalid_request"; + return "unknown"; +} + +/** Normalize SDK, gateway, and Pi failures into one provider error contract. */ +export function createProviderError( + error: unknown, + context: ProviderErrorContext = {}, +): ProviderError { + if (error instanceof ProviderError) return error; + const message = providerMessage(error); + const status = extractStatus(error, message); + const kind = classifyProviderError(status, message); + return new ProviderError( + message, + { + kind, + status, + modelId: context.modelId ?? extractField(message, "originalModelId"), + retryable: + Boolean(message) && + ["rate_limit", "capacity", "timeout", "network", "server"].includes( + kind, + ), + retryAfterMs: extractRetryAfterMs(error, message), + fallbackEligible: ["capacity", "timeout", "network", "server"].includes( + kind, + ), + }, + error, + ); +} + +/** Return whether a provider-boundary error should be retried. */ +export function isProviderRetryError(error: unknown): error is ProviderError { + return error instanceof ProviderError && error.retryable; +} + +/** Classify a terminal Pi assistant error through the provider error contract. */ +export function isRetryableProviderMessage(message: { + stopReason?: string; + errorMessage?: string; +}): boolean { + return ( + message.stopReason === "error" && + Boolean(message.errorMessage) && + createProviderError(message.errorMessage).retryable + ); +} + +/** Return stable, sanitized copy suitable for a terminal user response. */ +export function getProviderErrorUserMessage(error: ProviderError): string { + switch (error.kind) { + case "capacity": + return "The selected model is temporarily unavailable because it is at capacity. Please try again shortly."; + case "rate_limit": { + const seconds = error.retryAfterMs + ? Math.max(1, Math.ceil(error.retryAfterMs / 1_000)) + : undefined; + return seconds + ? `The model is rate-limited. Please try again in about ${seconds} seconds.` + : "The model is rate-limited. Please try again shortly."; + } + case "timeout": + case "network": + case "server": + return "The model provider had a temporary connection problem. Please try again."; + case "auth": + case "permission": + return "The model provider rejected Junior's credentials. This needs an administrator or configuration fix."; + case "quota": + return "The model provider's usage quota has been exhausted. This needs an administrator or billing configuration fix."; + default: + return ""; + } +} + +/** Return safe structured fields for terminal failure telemetry. */ +export function getProviderErrorAttributes( + error: ProviderError, +): Record { + return { + "app.ai.provider_error.kind": error.kind, + "app.ai.provider_error.retryable": error.retryable, + "app.ai.provider_error.fallback_eligible": error.fallbackEligible, + ...(error.status !== undefined + ? { "app.ai.provider_error.status": error.status } + : {}), + ...(error.retryAfterMs !== undefined + ? { "app.ai.provider_error.retry_after_ms": error.retryAfterMs } + : {}), + ...(error.modelId ? { "gen_ai.request.model": error.modelId } : {}), + }; +} diff --git a/packages/junior/src/chat/services/provider-retry.ts b/packages/junior/src/chat/services/provider-retry.ts index a4bcbd418..ccd41b4fa 100644 --- a/packages/junior/src/chat/services/provider-retry.ts +++ b/packages/junior/src/chat/services/provider-retry.ts @@ -1,57 +1,35 @@ import type { PiMessage } from "@/chat/pi/messages"; +import { createProviderError } from "@/chat/services/provider-error"; import { getPiMessageRole, trimTrailingAssistantMessages, } from "@/chat/pi/transcript"; const PROVIDER_RETRY_DELAYS_MS = [2_000, 4_000, 8_000] as const; -const PROVIDER_ERROR_PREFIX = "AI provider error:"; -const PROVIDER_RETRY_ERROR_NAME = "ProviderRetryError"; -const NON_RETRYABLE_PROVIDER_ERROR_PATTERNS = [ - /invalid.?api.?key|no api key|authentication|authorization|permission|forbidden|(?:invalid|missing|expired|revoked).{0,24}\bcredentials?\b|\bcredentials?\b.{0,24}(?:invalid|missing|expired|revoked)|\bno\b.{0,16}\bcredentials?\b/i, - /context.?length|context.?window/i, - /content.?policy|validation|bad request|\b(?:400|401|403)\b/i, - /unsupported model|invalid model|unknown ai gateway model|unknown model|mismatched api/i, - /usage limit|monthly usage limit|available balance|insufficient.?quota|out of budget|quota exceeded|billing/i, -] as const; - -function providerMessage(error: unknown): string { - return (error instanceof Error ? error.message : String(error)).trim(); -} - -/** Mark failures that cross the AI provider boundary for shared retry handling. */ -export function createProviderError(error: unknown): Error { - const message = providerMessage(error); - const displayMessage = `${PROVIDER_ERROR_PREFIX} ${message || "Unknown provider error"}`; - const providerError = new Error(displayMessage, { cause: error }); - if (message && isRetryableProviderFailure(message)) { - providerError.name = PROVIDER_RETRY_ERROR_NAME; - } - return providerError; -} - -/** Return whether a provider-boundary error should be retried by the worker. */ -export function isProviderRetryError(error: unknown): boolean { - return error instanceof Error && error.name === PROVIDER_RETRY_ERROR_NAME; -} - -/** Classify upstream failures by exclusion so new transient provider errors retry. */ -function isRetryableProviderFailure(errorMessage: string): boolean { - return !NON_RETRYABLE_PROVIDER_ERROR_PATTERNS.some((pattern) => - pattern.test(errorMessage), - ); -} +const MAX_PROVIDER_RETRY_DELAY_MS = 60_000; /** Build the next provider retry step from Pi history, if the turn can resume. */ export function nextProviderRetry(args: { attempt: number; + failure?: { stopReason?: string; errorMessage?: string }; messages: PiMessage[]; - retryableFailure: boolean; }): { delayMs: number; messages: PiMessage[] } | undefined { - const delayMs = PROVIDER_RETRY_DELAYS_MS[args.attempt]; - if (delayMs === undefined || !args.retryableFailure) { + const backoffMs = PROVIDER_RETRY_DELAYS_MS[args.attempt]; + if ( + backoffMs === undefined || + args.failure?.stopReason !== "error" || + !args.failure.errorMessage + ) { return undefined; } + const providerError = createProviderError(args.failure.errorMessage); + if (!providerError.retryable) { + return undefined; + } + const delayMs = Math.min( + MAX_PROVIDER_RETRY_DELAY_MS, + Math.max(backoffMs, providerError.retryAfterMs ?? 0), + ); const messages = trimTrailingAssistantMessages(args.messages); if (messages.length === args.messages.length) { diff --git a/packages/junior/src/chat/services/subscribed-decision.ts b/packages/junior/src/chat/services/subscribed-decision.ts index 37fc95ee8..115889e7a 100644 --- a/packages/junior/src/chat/services/subscribed-decision.ts +++ b/packages/junior/src/chat/services/subscribed-decision.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { escapeXml } from "@/chat/xml"; -import { isProviderRetryError } from "@/chat/services/provider-retry"; +import { isProviderRetryError } from "@/chat/services/provider-error"; export enum SubscribedReplyReason { ThreadOptOut = "thread_opt_out", diff --git a/packages/junior/src/chat/services/turn-failure-response.ts b/packages/junior/src/chat/services/turn-failure-response.ts index b9c02a9b4..d4204d2b5 100644 --- a/packages/junior/src/chat/services/turn-failure-response.ts +++ b/packages/junior/src/chat/services/turn-failure-response.ts @@ -2,6 +2,11 @@ import type { LogContext } from "@/chat/logging"; import { buildTurnFailureResponse } from "@/chat/logging"; import { getInterruptionMarker } from "@/chat/interruption-marker"; import type { AgentRunResult } from "@/chat/services/turn-result"; +import { + getProviderErrorAttributes, + getProviderErrorUserMessage, + ProviderError, +} from "@/chat/services/provider-error"; type LogException = ( error: unknown, @@ -57,7 +62,10 @@ function getFailureCapture(reply: AgentRunResult): { reply.diagnostics.errorMessage ?? "Provider error without explicit message", ), - attributes: {}, + attributes: + reply.diagnostics.providerError instanceof ProviderError + ? getProviderErrorAttributes(reply.diagnostics.providerError) + : {}, body: "Agent turn failed with provider error", }; } @@ -137,11 +145,19 @@ export function finalizeFailedTurnReply(args: { ? args.reply.text.trim() : ""; + const providerUserMessage = + args.reply.diagnostics.providerError instanceof ProviderError + ? getProviderErrorUserMessage(args.reply.diagnostics.providerError) + : ""; + const failureText = providerUserMessage + ? `${providerUserMessage} Reference: \`event_id=${eventId}\`.` + : buildTurnFailureResponse(eventId); + return { ...args.reply, text: providerPartialText ? `${providerPartialText}${getInterruptionMarker()}` - : buildTurnFailureResponse(eventId), + : failureText, deliveryMode: "thread", deliveryPlan: { mode: "thread", diff --git a/packages/junior/src/chat/services/turn-result.ts b/packages/junior/src/chat/services/turn-result.ts index 6a12e433f..8ecc69d2d 100644 --- a/packages/junior/src/chat/services/turn-result.ts +++ b/packages/junior/src/chat/services/turn-result.ts @@ -7,6 +7,7 @@ import { } from "@/chat/no-reply"; import type { PiMessage } from "@/chat/pi/messages"; import type { TurnReasoningSelection } from "@/chat/services/turn-reasoning-level"; +import { createProviderError } from "@/chat/services/provider-error"; import type { AgentTurnUsage } from "@/chat/usage"; import type { ReplyDeliveryPlan } from "@/chat/services/reply-delivery-plan"; import type { ThreadArtifactsState } from "@/chat/state/artifacts"; @@ -387,7 +388,10 @@ export function buildTurnResult(input: TurnResultInput): AgentRunResult { usage, stopReason, errorMessage, - providerError: undefined, + providerError: + resolvedOutcome === "provider_error" && errorMessage + ? createProviderError(errorMessage, { modelId }) + : undefined, }; return { diff --git a/packages/junior/src/chat/task-execution/worker.ts b/packages/junior/src/chat/task-execution/worker.ts index fa95f24ff..e6aeba00f 100644 --- a/packages/junior/src/chat/task-execution/worker.ts +++ b/packages/junior/src/chat/task-execution/worker.ts @@ -3,7 +3,7 @@ import type { Destination } from "@sentry/junior-plugin-api"; import { sameDestination } from "@/chat/destination"; import { logException, logInfo, logWarn } from "@/chat/logging"; import type { ConversationStore } from "@/chat/conversations/store"; -import { isProviderRetryError } from "@/chat/services/provider-retry"; +import { isProviderRetryError } from "@/chat/services/provider-error"; import { ConversationQueueMessageRejectedError, type ConversationQueueMessage, diff --git a/packages/junior/tests/integration/runtime/agent-run-agent-continue.test.ts b/packages/junior/tests/integration/runtime/agent-run-agent-continue.test.ts index 42619a387..8a134e410 100644 --- a/packages/junior/tests/integration/runtime/agent-run-agent-continue.test.ts +++ b/packages/junior/tests/integration/runtime/agent-run-agent-continue.test.ts @@ -140,7 +140,8 @@ vi.mock("@earendil-works/pi-agent-core", () => { role: "assistant", content: [{ type: "text", text: "provider error" }], stopReason: "error", - errorMessage: "Provider returned error: 503 service unavailable", + errorMessage: + "Provider returned error: 503 service unavailable: capacity exceeded", }); promptSettled.value = true; return {}; diff --git a/packages/junior/tests/integration/slack/subscribed-message-behavior.test.ts b/packages/junior/tests/integration/slack/subscribed-message-behavior.test.ts index 2574f3490..406d13383 100644 --- a/packages/junior/tests/integration/slack/subscribed-message-behavior.test.ts +++ b/packages/junior/tests/integration/slack/subscribed-message-behavior.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { TurnInputCommitLostError } from "@/chat/runtime/turn"; import type { JuniorRuntimeServiceOverrides } from "@/chat/app/services"; -import { createProviderError } from "@/chat/services/provider-retry"; +import { createProviderError } from "@/chat/services/provider-error"; import { createTestChatRuntime } from "../../fixtures/chat-runtime"; import { createTestMessage, diff --git a/packages/junior/tests/unit/services/provider-retry.test.ts b/packages/junior/tests/unit/services/provider-retry.test.ts index d218aaaae..8f171706c 100644 --- a/packages/junior/tests/unit/services/provider-retry.test.ts +++ b/packages/junior/tests/unit/services/provider-retry.test.ts @@ -1,15 +1,15 @@ import { describe, expect, it } from "vitest"; -import { - isRetryableAssistantError, - type AssistantMessage, -} from "@earendil-works/pi-ai"; +import type { AssistantMessage } from "@earendil-works/pi-ai"; import { fauxAssistantMessage } from "@earendil-works/pi-ai/providers/faux"; import type { PiMessage } from "@/chat/pi/messages"; import { - nextProviderRetry, createProviderError, + getProviderErrorUserMessage, isProviderRetryError, -} from "@/chat/services/provider-retry"; + isRetryableProviderMessage, + ProviderError, +} from "@/chat/services/provider-error"; +import { nextProviderRetry } from "@/chat/services/provider-retry"; function assistantError(errorMessage: string | undefined): AssistantMessage { return fauxAssistantMessage([], { @@ -30,6 +30,13 @@ describe("provider retry helpers", () => { expect(error.message).toBe( "AI provider error: Anthropic stream ended before message_stop", ); + expect(error).toBeInstanceOf(ProviderError); + expect(error).toMatchObject({ + code: "provider_error", + kind: "network", + retryable: true, + fallbackEligible: true, + }); expect(isProviderRetryError(error)).toBe(true); expect(isProviderRetryError(createProviderError("invalid_api_key"))).toBe( false, @@ -51,7 +58,7 @@ describe("provider retry helpers", () => { nextProviderRetry({ attempt: 0, messages: [user, failedAssistant], - retryableFailure: isRetryableAssistantError(failedAssistant), + failure: failedAssistant, }), ).toEqual({ delayMs: 2_000, messages: [user] }); }); @@ -63,19 +70,84 @@ describe("provider retry helpers", () => { } as PiMessage; const failedAssistant = assistantError(XAI_SERVICE_UNAVAILABLE); - expect( - isProviderRetryError(createProviderError(XAI_SERVICE_UNAVAILABLE)), - ).toBe(true); + const providerError = createProviderError(XAI_SERVICE_UNAVAILABLE); + expect(providerError).toMatchObject({ + kind: "server", + status: 503, + modelId: "xai/grok-4.5", + retryable: true, + fallbackEligible: true, + }); + expect(isProviderRetryError(providerError)).toBe(true); expect( nextProviderRetry({ attempt: 0, messages: [user, failedAssistant], - retryableFailure: isRetryableAssistantError(failedAssistant), + failure: failedAssistant, }), ).toEqual({ delayMs: 2_000, messages: [user] }); }); + it("normalizes rate-limit hints and honors them when planning retries", () => { + const error = Object.assign(new Error("rate limited"), { + statusCode: 429, + responseHeaders: { "Retry-After": "30" }, + }); + + expect(createProviderError(error)).toMatchObject({ + kind: "rate_limit", + status: 429, + retryable: true, + retryAfterMs: 30_000, + fallbackEligible: false, + }); + + const user = { + role: "user", + content: [{ type: "text", text: "help" }], + } as PiMessage; + const failedAssistant = assistantError( + '429 too many requests {"retry-after":"30"}', + ); + expect( + nextProviderRetry({ + attempt: 0, + messages: [user, failedAssistant], + failure: failedAssistant, + }), + ).toEqual({ delayMs: 30_000, messages: [user] }); + }); + + it("retries common unstructured server failures", () => { + for (const message of [ + "Internal Server Error", + "Error: 502 Bad Gateway", + "upstream request failed", + "An unexpected error occurred", + ]) { + expect(createProviderError(message)).toMatchObject({ + kind: "server", + retryable: true, + fallbackEligible: true, + }); + } + expect(createProviderError("Gateway Timeout")).toMatchObject({ + kind: "timeout", + retryable: true, + fallbackEligible: true, + }); + }); + + it("does not claim a retry happened in terminal capacity copy", () => { + const message = getProviderErrorUserMessage( + createProviderError("Provider capacity exceeded"), + ); + + expect(message).toContain("at capacity"); + expect(message).not.toContain("retried"); + }); + it("does not retry explicit credential failures", () => { for (const message of [ "Missing AI gateway credentials (AI_GATEWAY_API_KEY or VERCEL_OIDC_TOKEN)", @@ -97,13 +169,13 @@ describe("provider retry helpers", () => { overrides: { attempt?: number; messages?: PiMessage[]; - retryableFailure?: boolean; + failure?: AssistantMessage; } = {}, ) => nextProviderRetry({ attempt: 0, messages: [user, failedAssistant], - retryableFailure: true, + failure: failedAssistant, ...overrides, }); @@ -115,10 +187,14 @@ describe("provider retry helpers", () => { '429 {"error":{"message":"Quota exceeded","type":"insufficient_quota"}}', ), ]) { - expect(isRetryableAssistantError(message)).toBe(false); + expect(isRetryableProviderMessage(message)).toBe(false); } - expect(retry({ retryableFailure: false })).toBeUndefined(); + expect( + retry({ failure: assistantError("400 bad request") }), + ).toBeUndefined(); expect(retry({ attempt: 3 })).toBeUndefined(); - expect(retry({ messages: [failedAssistant] })).toBeUndefined(); + expect( + retry({ messages: [failedAssistant], failure: undefined }), + ).toBeUndefined(); }); }); diff --git a/packages/junior/tests/unit/services/turn-failure-response.test.ts b/packages/junior/tests/unit/services/turn-failure-response.test.ts index b3d7a59ff..6543a2955 100644 --- a/packages/junior/tests/unit/services/turn-failure-response.test.ts +++ b/packages/junior/tests/unit/services/turn-failure-response.test.ts @@ -2,10 +2,12 @@ import { describe, expect, it, vi } from "vitest"; import { getInterruptionMarker } from "@/chat/interruption-marker"; import { finalizeFailedTurnReply } from "@/chat/services/turn-failure-response"; import type { AgentRunResult } from "@/chat/services/turn-result"; +import { createProviderError } from "@/chat/services/provider-error"; function providerErrorReply(args: { assistantMessageCount: number; errorMessage?: string; + providerError?: unknown; text: string; }): AgentRunResult { return { @@ -19,6 +21,7 @@ function providerErrorReply(args: { toolErrorCount: 0, usedPrimaryText: false, ...(args.errorMessage ? { errorMessage: args.errorMessage } : {}), + ...(args.providerError ? { providerError: args.providerError } : {}), }, }; } @@ -41,6 +44,35 @@ describe("finalizeFailedTurnReply", () => { expect(finalized.text).toContain("event_id=evt_123"); }); + it("explains recognized provider failures without exposing the payload", () => { + const logException = vi.fn().mockReturnValue("evt_503"); + const providerError = createProviderError( + '503 {"error":{"message":"Service temporarily unavailable"}}', + { modelId: "xai/grok-4.5" }, + ); + + const finalized = finalizeFailedTurnReply({ + reply: providerErrorReply({ + assistantMessageCount: 0, + errorMessage: providerError.message, + providerError, + text: "", + }), + logException, + context: {}, + }); + + expect(finalized.text).toContain("temporary connection problem"); + expect(finalized.text).not.toContain("capacity"); + expect(finalized.text).toContain("event_id=evt_503"); + expect(finalized.text).not.toContain("Service temporarily unavailable"); + expect(logException.mock.calls[0]?.[3]).toMatchObject({ + "app.ai.provider_error.kind": "server", + "app.ai.provider_error.status": 503, + "app.ai.provider_error.retryable": true, + }); + }); + it("delivers genuine model-authored partial text with the interruption marker", () => { const logException = vi.fn().mockReturnValue("evt_456");