From 16edb0c310677b268f05a5b9779e24f66c751c50 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:08:16 +0000 Subject: [PATCH 01/10] fix(agent): normalize provider failures Co-Authored-By: OpenAI GPT-5.4 Co-Authored-By: immutable dcramer --- .../src/chat/services/provider-error.ts | 215 ++++++++++++++++++ .../src/chat/services/provider-retry.ts | 37 +-- .../chat/services/turn-failure-response.ts | 19 +- .../junior/src/chat/services/turn-result.ts | 6 +- .../unit/services/provider-retry.test.ts | 36 ++- .../services/turn-failure-response.test.ts | 31 +++ 6 files changed, 305 insertions(+), 39 deletions(-) create mode 100644 packages/junior/src/chat/services/provider-error.ts diff --git a/packages/junior/src/chat/services/provider-error.ts b/packages/junior/src/chat/services/provider-error.ts new file mode 100644 index 000000000..9332fbbb0 --- /dev/null +++ b/packages/junior/src/chat/services/provider-error.ts @@ -0,0 +1,215 @@ +const MAX_PROVIDER_RESPONSE_CHARS = 4_000; + +export type ProviderErrorKind = + | "auth" + | "permission" + | "rate_limit" + | "capacity" + | "timeout" + | "network" + | "server" + | "invalid_request" + | "quota" + | "content_policy" + | "unknown"; + +export interface ProviderErrorContext { + modelId?: string; + provider?: string; +} + +export interface ProviderErrorDetails { + kind: ProviderErrorKind; + status?: number; + provider?: string; + modelId?: string; + retryable: boolean; + retryAfterMs?: number; + fallbackEligible: boolean; + providerResponse?: string; +} + +/** Structured failure produced at an AI provider boundary. */ +export class ProviderError extends Error { + readonly code = "provider_error"; + readonly kind: ProviderErrorKind; + readonly status?: number; + readonly provider?: string; + readonly modelId?: string; + readonly retryable: boolean; + readonly retryAfterMs?: number; + readonly fallbackEligible: boolean; + readonly providerResponse?: string; + readonly userMessage: string; + + constructor(message: string, details: ProviderErrorDetails, cause?: unknown) { + super(`AI provider error: ${message || "Unknown provider error"}`, { + cause, + }); + this.name = "ProviderError"; + this.kind = details.kind; + this.status = details.status; + this.provider = details.provider; + this.modelId = details.modelId; + this.retryable = details.retryable; + this.retryAfterMs = details.retryAfterMs; + this.fallbackEligible = details.fallbackEligible; + this.providerResponse = details.providerResponse; + this.userMessage = getProviderErrorUserMessage(this); + } +} + +type ErrorShape = Error & { + status?: unknown; + statusCode?: unknown; + response?: { headers?: unknown }; +}; + +function providerMessage(error: unknown): string { + return (error instanceof Error ? error.message : String(error)).trim(); +} + +function extractStatus(error: unknown, message: string): number | undefined { + if (error instanceof Error) { + const shaped = error as ErrorShape; + if (typeof shaped.status === "number") return shaped.status; + if (typeof shaped.statusCode === "number") return shaped.statusCode; + } + const match = message.match( + /^([45]\d\d)\b|\bstatus(?:Code)?["'=:\s]+([45]\d\d)\b/i, + ); + const status = match?.[1] ?? match?.[2]; + return status ? Number(status) : undefined; +} + +function extractField(message: string, field: string): string | undefined { + const match = message.match(new RegExp(`"${field}"\\s*:\\s*"([^"]+)"`, "i")); + return match?.[1]; +} + +function extractRetryAfterMs(error: unknown, message: string): number | undefined { + const responseHeaders = + error instanceof Error ? (error as ErrorShape).response?.headers : undefined; + const retryAfter = + responseHeaders instanceof Headers + ? responseHeaders.get("retry-after") + : responseHeaders && typeof responseHeaders === "object" + ? (responseHeaders as Record)["retry-after"] + : undefined; + const raw = typeof retryAfter === "string" || typeof retryAfter === "number" + ? String(retryAfter) + : message.match(/"retry-after"\s*:\s*"?([^",}]+)"?/i)?.[1]; + if (!raw) return undefined; + const seconds = Number(raw); + if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1_000; + const date = Date.parse(raw); + return Number.isNaN(date) ? undefined : Math.max(0, date - Date.now()); +} + +function classifyProviderError( + status: number | undefined, + message: string, +): ProviderErrorKind { + if (/content.?policy|safety|moderation/i.test(message)) return "content_policy"; + if (/insufficient.?quota|quota exceeded|usage limit|monthly usage limit|available balance|out of budget|billing/i.test(message)) return "quota"; + if (status === 401 || /invalid.?api.?key|no api key|authentication|missing credentials?|expired credentials?|revoked credentials?/i.test(message)) return "auth"; + if (status === 403 || /permission|forbidden|authorization/i.test(message)) return "permission"; + if (status === 429 || /rate.?limit|too many requests/i.test(message)) return "rate_limit"; + if (status === 503 || /overloaded|service.?unavailable|capacity/i.test(message)) return "capacity"; + if (/timed? out|timeout/i.test(message)) return "timeout"; + if (/network.?error|connection|fetch failed|socket|stream ended|ended before|ECONNRESET/i.test(message)) return "network"; + if (status !== undefined && status >= 500) return "server"; + if (status !== undefined && status >= 400) return "invalid_request"; + if (/context.?length|context.?window|validation|bad request|unsupported model|invalid model|unknown (?:ai gateway )?model|mismatched api/i.test(message)) return "invalid_request"; + return "unknown"; +} + +/** Normalize SDK, gateway, and Pi failures into one provider error contract. */ +export function createProviderError( + error: unknown, + context: ProviderErrorContext = {}, +): ProviderError { + if (error instanceof ProviderError) return error; + + const message = providerMessage(error); + const status = extractStatus(error, message); + const kind = classifyProviderError(status, message); + const retryable = [ + "rate_limit", + "capacity", + "timeout", + "network", + "server", + "unknown", + ].includes(kind); + const fallbackEligible = ["capacity", "timeout", "network", "server"].includes(kind); + const provider = context.provider ?? extractField(message, "resolvedProvider"); + const modelId = context.modelId ?? extractField(message, "originalModelId"); + + return new ProviderError( + message, + { + kind, + status, + provider, + modelId, + retryable: Boolean(message) && retryable, + retryAfterMs: extractRetryAfterMs(error, message), + fallbackEligible, + providerResponse: message + ? message.slice(0, MAX_PROVIDER_RESPONSE_CHARS) + : undefined, + }, + error, + ); +} + +/** Return stable, sanitized copy suitable for a terminal user response. */ +export function getProviderErrorUserMessage(error: ProviderError): string { + switch (error.kind) { + case "capacity": + return "The selected model is temporarily unavailable. I retried it, but it is still at capacity. Please try again shortly."; + case "rate_limit": { + const retrySeconds = error.retryAfterMs + ? Math.max(1, Math.ceil(error.retryAfterMs / 1_000)) + : undefined; + return retrySeconds + ? `The model is rate-limited. Please try again in about ${retrySeconds} seconds.` + : "The model is rate-limited. Please try again shortly."; + } + case "timeout": + case "network": + case "server": + return "The model provider had a temporary connection problem. Please try again."; + case "auth": + case "permission": + return "The model provider rejected Junior's credentials. This needs an administrator or configuration fix."; + case "quota": + return "The model provider's usage quota has been exhausted. This needs an administrator or billing configuration fix."; + default: + return ""; + } +} + +/** Return safe structured fields for terminal failure telemetry. */ +export function getProviderErrorAttributes( + error: ProviderError, +): Record { + return { + "app.ai.provider_error.kind": error.kind, + "app.ai.provider_error.retryable": error.retryable, + "app.ai.provider_error.fallback_eligible": error.fallbackEligible, + ...(error.status !== undefined + ? { "app.ai.provider_error.status": error.status } + : {}), + ...(error.retryAfterMs !== undefined + ? { "app.ai.provider_error.retry_after_ms": error.retryAfterMs } + : {}), + ...(error.provider + ? { "gen_ai.provider.name": error.provider } + : {}), + ...(error.modelId + ? { "gen_ai.request.model": error.modelId } + : {}), + }; +} diff --git a/packages/junior/src/chat/services/provider-retry.ts b/packages/junior/src/chat/services/provider-retry.ts index a4bcbd418..16c4f3a9d 100644 --- a/packages/junior/src/chat/services/provider-retry.ts +++ b/packages/junior/src/chat/services/provider-retry.ts @@ -1,45 +1,16 @@ import type { PiMessage } from "@/chat/pi/messages"; +import { ProviderError } from "@/chat/services/provider-error"; + +export { createProviderError } from "@/chat/services/provider-error"; import { getPiMessageRole, trimTrailingAssistantMessages, } from "@/chat/pi/transcript"; const PROVIDER_RETRY_DELAYS_MS = [2_000, 4_000, 8_000] as const; -const PROVIDER_ERROR_PREFIX = "AI provider error:"; -const PROVIDER_RETRY_ERROR_NAME = "ProviderRetryError"; -const NON_RETRYABLE_PROVIDER_ERROR_PATTERNS = [ - /invalid.?api.?key|no api key|authentication|authorization|permission|forbidden|(?:invalid|missing|expired|revoked).{0,24}\bcredentials?\b|\bcredentials?\b.{0,24}(?:invalid|missing|expired|revoked)|\bno\b.{0,16}\bcredentials?\b/i, - /context.?length|context.?window/i, - /content.?policy|validation|bad request|\b(?:400|401|403)\b/i, - /unsupported model|invalid model|unknown ai gateway model|unknown model|mismatched api/i, - /usage limit|monthly usage limit|available balance|insufficient.?quota|out of budget|quota exceeded|billing/i, -] as const; - -function providerMessage(error: unknown): string { - return (error instanceof Error ? error.message : String(error)).trim(); -} - -/** Mark failures that cross the AI provider boundary for shared retry handling. */ -export function createProviderError(error: unknown): Error { - const message = providerMessage(error); - const displayMessage = `${PROVIDER_ERROR_PREFIX} ${message || "Unknown provider error"}`; - const providerError = new Error(displayMessage, { cause: error }); - if (message && isRetryableProviderFailure(message)) { - providerError.name = PROVIDER_RETRY_ERROR_NAME; - } - return providerError; -} - /** Return whether a provider-boundary error should be retried by the worker. */ export function isProviderRetryError(error: unknown): boolean { - return error instanceof Error && error.name === PROVIDER_RETRY_ERROR_NAME; -} - -/** Classify upstream failures by exclusion so new transient provider errors retry. */ -function isRetryableProviderFailure(errorMessage: string): boolean { - return !NON_RETRYABLE_PROVIDER_ERROR_PATTERNS.some((pattern) => - pattern.test(errorMessage), - ); + return error instanceof ProviderError && error.retryable; } /** Build the next provider retry step from Pi history, if the turn can resume. */ diff --git a/packages/junior/src/chat/services/turn-failure-response.ts b/packages/junior/src/chat/services/turn-failure-response.ts index b9c02a9b4..38e6e3d5e 100644 --- a/packages/junior/src/chat/services/turn-failure-response.ts +++ b/packages/junior/src/chat/services/turn-failure-response.ts @@ -2,6 +2,10 @@ import type { LogContext } from "@/chat/logging"; import { buildTurnFailureResponse } from "@/chat/logging"; import { getInterruptionMarker } from "@/chat/interruption-marker"; import type { AgentRunResult } from "@/chat/services/turn-result"; +import { + getProviderErrorAttributes, + ProviderError, +} from "@/chat/services/provider-error"; type LogException = ( error: unknown, @@ -57,7 +61,10 @@ function getFailureCapture(reply: AgentRunResult): { reply.diagnostics.errorMessage ?? "Provider error without explicit message", ), - attributes: {}, + attributes: + reply.diagnostics.providerError instanceof ProviderError + ? getProviderErrorAttributes(reply.diagnostics.providerError) + : {}, body: "Agent turn failed with provider error", }; } @@ -137,11 +144,19 @@ export function finalizeFailedTurnReply(args: { ? args.reply.text.trim() : ""; + const providerUserMessage = + args.reply.diagnostics.providerError instanceof ProviderError + ? args.reply.diagnostics.providerError.userMessage + : ""; + const failureText = providerUserMessage + ? `${providerUserMessage} Reference: \`event_id=${eventId}\`.` + : buildTurnFailureResponse(eventId); + return { ...args.reply, text: providerPartialText ? `${providerPartialText}${getInterruptionMarker()}` - : buildTurnFailureResponse(eventId), + : failureText, deliveryMode: "thread", deliveryPlan: { mode: "thread", diff --git a/packages/junior/src/chat/services/turn-result.ts b/packages/junior/src/chat/services/turn-result.ts index 6a12e433f..8ecc69d2d 100644 --- a/packages/junior/src/chat/services/turn-result.ts +++ b/packages/junior/src/chat/services/turn-result.ts @@ -7,6 +7,7 @@ import { } from "@/chat/no-reply"; import type { PiMessage } from "@/chat/pi/messages"; import type { TurnReasoningSelection } from "@/chat/services/turn-reasoning-level"; +import { createProviderError } from "@/chat/services/provider-error"; import type { AgentTurnUsage } from "@/chat/usage"; import type { ReplyDeliveryPlan } from "@/chat/services/reply-delivery-plan"; import type { ThreadArtifactsState } from "@/chat/state/artifacts"; @@ -387,7 +388,10 @@ export function buildTurnResult(input: TurnResultInput): AgentRunResult { usage, stopReason, errorMessage, - providerError: undefined, + providerError: + resolvedOutcome === "provider_error" && errorMessage + ? createProviderError(errorMessage, { modelId }) + : undefined, }; return { diff --git a/packages/junior/tests/unit/services/provider-retry.test.ts b/packages/junior/tests/unit/services/provider-retry.test.ts index d218aaaae..35289c353 100644 --- a/packages/junior/tests/unit/services/provider-retry.test.ts +++ b/packages/junior/tests/unit/services/provider-retry.test.ts @@ -5,6 +5,7 @@ import { } from "@earendil-works/pi-ai"; import { fauxAssistantMessage } from "@earendil-works/pi-ai/providers/faux"; import type { PiMessage } from "@/chat/pi/messages"; +import { ProviderError } from "@/chat/services/provider-error"; import { nextProviderRetry, createProviderError, @@ -30,6 +31,13 @@ describe("provider retry helpers", () => { expect(error.message).toBe( "AI provider error: Anthropic stream ended before message_stop", ); + expect(error).toBeInstanceOf(ProviderError); + expect(error).toMatchObject({ + code: "provider_error", + kind: "network", + retryable: true, + fallbackEligible: true, + }); expect(isProviderRetryError(error)).toBe(true); expect(isProviderRetryError(createProviderError("invalid_api_key"))).toBe( false, @@ -63,9 +71,16 @@ describe("provider retry helpers", () => { } as PiMessage; const failedAssistant = assistantError(XAI_SERVICE_UNAVAILABLE); - expect( - isProviderRetryError(createProviderError(XAI_SERVICE_UNAVAILABLE)), - ).toBe(true); + const providerError = createProviderError(XAI_SERVICE_UNAVAILABLE); + expect(providerError).toMatchObject({ + kind: "capacity", + status: 503, + provider: "xai", + modelId: "xai/grok-4.5", + retryable: true, + fallbackEligible: true, + }); + expect(isProviderRetryError(providerError)).toBe(true); expect( nextProviderRetry({ @@ -76,6 +91,21 @@ describe("provider retry helpers", () => { ).toEqual({ delayMs: 2_000, messages: [user] }); }); + it("normalizes rate-limit hints", () => { + const error = Object.assign(new Error("429 too many requests"), { + status: 429, + response: { headers: { "retry-after": "30" } }, + }); + + expect(createProviderError(error)).toMatchObject({ + kind: "rate_limit", + status: 429, + retryable: true, + retryAfterMs: 30_000, + fallbackEligible: false, + }); + }); + it("does not retry explicit credential failures", () => { for (const message of [ "Missing AI gateway credentials (AI_GATEWAY_API_KEY or VERCEL_OIDC_TOKEN)", diff --git a/packages/junior/tests/unit/services/turn-failure-response.test.ts b/packages/junior/tests/unit/services/turn-failure-response.test.ts index b3d7a59ff..b46a445b1 100644 --- a/packages/junior/tests/unit/services/turn-failure-response.test.ts +++ b/packages/junior/tests/unit/services/turn-failure-response.test.ts @@ -2,10 +2,12 @@ import { describe, expect, it, vi } from "vitest"; import { getInterruptionMarker } from "@/chat/interruption-marker"; import { finalizeFailedTurnReply } from "@/chat/services/turn-failure-response"; import type { AgentRunResult } from "@/chat/services/turn-result"; +import { createProviderError } from "@/chat/services/provider-error"; function providerErrorReply(args: { assistantMessageCount: number; errorMessage?: string; + providerError?: unknown; text: string; }): AgentRunResult { return { @@ -19,6 +21,7 @@ function providerErrorReply(args: { toolErrorCount: 0, usedPrimaryText: false, ...(args.errorMessage ? { errorMessage: args.errorMessage } : {}), + ...(args.providerError ? { providerError: args.providerError } : {}), }, }; } @@ -41,6 +44,34 @@ describe("finalizeFailedTurnReply", () => { expect(finalized.text).toContain("event_id=evt_123"); }); + it("explains recognized provider failures without exposing the payload", () => { + const logException = vi.fn().mockReturnValue("evt_503"); + const providerError = createProviderError( + '503 {"error":{"message":"Service temporarily unavailable"}}', + { modelId: "xai/grok-4.5", provider: "xai" }, + ); + + const finalized = finalizeFailedTurnReply({ + reply: providerErrorReply({ + assistantMessageCount: 0, + errorMessage: providerError.message, + providerError, + text: "", + }), + logException, + context: {}, + }); + + expect(finalized.text).toContain("still at capacity"); + expect(finalized.text).toContain("event_id=evt_503"); + expect(finalized.text).not.toContain("Service temporarily unavailable"); + expect(logException.mock.calls[0]?.[3]).toMatchObject({ + "app.ai.provider_error.kind": "capacity", + "app.ai.provider_error.status": 503, + "app.ai.provider_error.retryable": true, + }); + }); + it("delivers genuine model-authored partial text with the interruption marker", () => { const logException = vi.fn().mockReturnValue("evt_456"); From b3969c79fc93073e430b0a778acadd69bae3d655 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:21:21 +0000 Subject: [PATCH 02/10] refactor(agent): centralize provider classification Co-Authored-By: OpenAI GPT-5.4 --- packages/junior/src/chat/agent/index.ts | 11 +- packages/junior/src/chat/pi/client.ts | 2 +- .../junior/src/chat/runtime/slack-runtime.ts | 2 +- .../src/chat/services/provider-error.ts | 110 +++++++----------- .../src/chat/services/provider-retry.ts | 16 ++- .../src/chat/services/subscribed-decision.ts | 2 +- .../chat/services/turn-failure-response.ts | 3 +- .../junior/src/chat/task-execution/worker.ts | 2 +- .../slack/subscribed-message-behavior.test.ts | 2 +- .../unit/services/provider-retry.test.ts | 26 ++--- 10 files changed, 74 insertions(+), 102 deletions(-) diff --git a/packages/junior/src/chat/agent/index.ts b/packages/junior/src/chat/agent/index.ts index 87fbd2f48..157dbae45 100644 --- a/packages/junior/src/chat/agent/index.ts +++ b/packages/junior/src/chat/agent/index.ts @@ -9,7 +9,6 @@ * stay outside this module. */ import { Agent, type AgentLoopTurnUpdate } from "@earendil-works/pi-agent-core"; -import { isRetryableAssistantError } from "@earendil-works/pi-ai"; import type { FileUpload } from "chat"; import { botConfig } from "@/chat/config"; import { @@ -64,10 +63,8 @@ import { shouldEmitDevAgentTrace } from "@/chat/runtime/dev-agent-trace"; import { isTurnInputCommitLostError } from "@/chat/runtime/turn"; import type { AgentRunOutcome } from "@/chat/runtime/agent-run-outcome"; import { buildTurnResult } from "@/chat/services/turn-result"; -import { - isProviderRetryError, - nextProviderRetry, -} from "@/chat/services/provider-retry"; +import { nextProviderRetry } from "@/chat/services/provider-retry"; +import { isProviderRetryError } from "@/chat/services/provider-error"; import { configuredTurnReasoningLevel, selectTurnReasoningLevel, @@ -1022,10 +1019,8 @@ async function executeAgentRunInPrivacyContext( const providerRetry = nextProviderRetry({ attempt, + failure: lastAssistant, messages: agent!.state.messages, - retryableFailure: - lastAssistant !== undefined && - isRetryableAssistantError(lastAssistant), }); if (!providerRetry) { break; diff --git a/packages/junior/src/chat/pi/client.ts b/packages/junior/src/chat/pi/client.ts index 3ee714f00..4876466b1 100644 --- a/packages/junior/src/chat/pi/client.ts +++ b/packages/junior/src/chat/pi/client.ts @@ -44,7 +44,7 @@ import { import { createProviderError, isProviderRetryError, -} from "@/chat/services/provider-retry"; +} from "@/chat/services/provider-error"; import { hasCompactedConversationContext } from "@/chat/services/context-compaction-marker"; const GATEWAY_PROVIDER = "vercel-ai-gateway" as const; diff --git a/packages/junior/src/chat/runtime/slack-runtime.ts b/packages/junior/src/chat/runtime/slack-runtime.ts index 92d5b59d8..e317edaff 100644 --- a/packages/junior/src/chat/runtime/slack-runtime.ts +++ b/packages/junior/src/chat/runtime/slack-runtime.ts @@ -9,7 +9,7 @@ import type { Message, MessageContext, Thread } from "chat"; import type { Destination } from "@sentry/junior-plugin-api"; import { getSubscribedReplyPreflightDecision } from "@/chat/services/subscribed-decision"; -import { isProviderRetryError } from "@/chat/services/provider-retry"; +import { isProviderRetryError } from "@/chat/services/provider-error"; import { AuthorizationFlowDisabledError } from "@/chat/services/auth-pause"; import { SlackActionError } from "@/chat/slack/client"; import { diff --git a/packages/junior/src/chat/services/provider-error.ts b/packages/junior/src/chat/services/provider-error.ts index 9332fbbb0..4b799e869 100644 --- a/packages/junior/src/chat/services/provider-error.ts +++ b/packages/junior/src/chat/services/provider-error.ts @@ -1,5 +1,3 @@ -const MAX_PROVIDER_RESPONSE_CHARS = 4_000; - export type ProviderErrorKind = | "auth" | "permission" @@ -13,20 +11,17 @@ export type ProviderErrorKind = | "content_policy" | "unknown"; -export interface ProviderErrorContext { +interface ProviderErrorContext { modelId?: string; provider?: string; } -export interface ProviderErrorDetails { +interface ProviderErrorDetails extends ProviderErrorContext { kind: ProviderErrorKind; status?: number; - provider?: string; - modelId?: string; retryable: boolean; retryAfterMs?: number; fallbackEligible: boolean; - providerResponse?: string; } /** Structured failure produced at an AI provider boundary. */ @@ -39,13 +34,9 @@ export class ProviderError extends Error { readonly retryable: boolean; readonly retryAfterMs?: number; readonly fallbackEligible: boolean; - readonly providerResponse?: string; - readonly userMessage: string; constructor(message: string, details: ProviderErrorDetails, cause?: unknown) { - super(`AI provider error: ${message || "Unknown provider error"}`, { - cause, - }); + super(`AI provider error: ${message || "Unknown provider error"}`, { cause }); this.name = "ProviderError"; this.kind = details.kind; this.status = details.status; @@ -54,8 +45,6 @@ export class ProviderError extends Error { this.retryable = details.retryable; this.retryAfterMs = details.retryAfterMs; this.fallbackEligible = details.fallbackEligible; - this.providerResponse = details.providerResponse; - this.userMessage = getProviderErrorUserMessage(this); } } @@ -83,22 +72,21 @@ function extractStatus(error: unknown, message: string): number | undefined { } function extractField(message: string, field: string): string | undefined { - const match = message.match(new RegExp(`"${field}"\\s*:\\s*"([^"]+)"`, "i")); - return match?.[1]; + return message.match(new RegExp(`"${field}"\\s*:\\s*"([^"]+)"`, "i"))?.[1]; } function extractRetryAfterMs(error: unknown, message: string): number | undefined { - const responseHeaders = - error instanceof Error ? (error as ErrorShape).response?.headers : undefined; + const headers = error instanceof Error ? (error as ErrorShape).response?.headers : undefined; const retryAfter = - responseHeaders instanceof Headers - ? responseHeaders.get("retry-after") - : responseHeaders && typeof responseHeaders === "object" - ? (responseHeaders as Record)["retry-after"] + headers instanceof Headers + ? headers.get("retry-after") + : headers && typeof headers === "object" + ? (headers as Record)["retry-after"] : undefined; - const raw = typeof retryAfter === "string" || typeof retryAfter === "number" - ? String(retryAfter) - : message.match(/"retry-after"\s*:\s*"?([^",}]+)"?/i)?.[1]; + const raw = + typeof retryAfter === "string" || typeof retryAfter === "number" + ? String(retryAfter) + : message.match(/"retry-after"\s*:\s*"?([^",}]+)"?/i)?.[1]; if (!raw) return undefined; const seconds = Number(raw); if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1_000; @@ -106,13 +94,10 @@ function extractRetryAfterMs(error: unknown, message: string): number | undefine return Number.isNaN(date) ? undefined : Math.max(0, date - Date.now()); } -function classifyProviderError( - status: number | undefined, - message: string, -): ProviderErrorKind { +function classifyProviderError(status: number | undefined, message: string): ProviderErrorKind { if (/content.?policy|safety|moderation/i.test(message)) return "content_policy"; if (/insufficient.?quota|quota exceeded|usage limit|monthly usage limit|available balance|out of budget|billing/i.test(message)) return "quota"; - if (status === 401 || /invalid.?api.?key|no api key|authentication|missing credentials?|expired credentials?|revoked credentials?/i.test(message)) return "auth"; + if (status === 401 || /invalid.?api.?key|no api key|authentication|(?:invalid|missing|expired|revoked).{0,24}\bcredentials?\b|\bcredentials?\b.{0,24}(?:invalid|missing|expired|revoked)|\bno\b.{0,16}\bcredentials?\b/i.test(message)) return "auth"; if (status === 403 || /permission|forbidden|authorization/i.test(message)) return "permission"; if (status === 429 || /rate.?limit|too many requests/i.test(message)) return "rate_limit"; if (status === 503 || /overloaded|service.?unavailable|capacity/i.test(message)) return "capacity"; @@ -130,51 +115,54 @@ export function createProviderError( context: ProviderErrorContext = {}, ): ProviderError { if (error instanceof ProviderError) return error; - const message = providerMessage(error); const status = extractStatus(error, message); const kind = classifyProviderError(status, message); - const retryable = [ - "rate_limit", - "capacity", - "timeout", - "network", - "server", - "unknown", - ].includes(kind); - const fallbackEligible = ["capacity", "timeout", "network", "server"].includes(kind); - const provider = context.provider ?? extractField(message, "resolvedProvider"); - const modelId = context.modelId ?? extractField(message, "originalModelId"); - return new ProviderError( message, { kind, status, - provider, - modelId, - retryable: Boolean(message) && retryable, + provider: context.provider ?? extractField(message, "resolvedProvider"), + modelId: context.modelId ?? extractField(message, "originalModelId"), + retryable: + Boolean(message) && + ["rate_limit", "capacity", "timeout", "network", "server"].includes(kind), retryAfterMs: extractRetryAfterMs(error, message), - fallbackEligible, - providerResponse: message - ? message.slice(0, MAX_PROVIDER_RESPONSE_CHARS) - : undefined, + fallbackEligible: ["capacity", "timeout", "network", "server"].includes(kind), }, error, ); } +/** Return whether a provider-boundary error should be retried. */ +export function isProviderRetryError(error: unknown): error is ProviderError { + return error instanceof ProviderError && error.retryable; +} + +/** Classify a terminal Pi assistant error through the provider error contract. */ +export function isRetryableProviderMessage(message: { + stopReason?: string; + errorMessage?: string; +}): boolean { + return ( + message.stopReason === "error" && + Boolean(message.errorMessage) && + createProviderError(message.errorMessage).retryable + ); +} + /** Return stable, sanitized copy suitable for a terminal user response. */ export function getProviderErrorUserMessage(error: ProviderError): string { switch (error.kind) { case "capacity": return "The selected model is temporarily unavailable. I retried it, but it is still at capacity. Please try again shortly."; case "rate_limit": { - const retrySeconds = error.retryAfterMs + const seconds = error.retryAfterMs ? Math.max(1, Math.ceil(error.retryAfterMs / 1_000)) : undefined; - return retrySeconds - ? `The model is rate-limited. Please try again in about ${retrySeconds} seconds.` + return seconds + ? `The model is rate-limited. Please try again in about ${seconds} seconds.` : "The model is rate-limited. Please try again shortly."; } case "timeout": @@ -192,24 +180,16 @@ export function getProviderErrorUserMessage(error: ProviderError): string { } /** Return safe structured fields for terminal failure telemetry. */ -export function getProviderErrorAttributes( - error: ProviderError, -): Record { +export function getProviderErrorAttributes(error: ProviderError): Record { return { "app.ai.provider_error.kind": error.kind, "app.ai.provider_error.retryable": error.retryable, "app.ai.provider_error.fallback_eligible": error.fallbackEligible, - ...(error.status !== undefined - ? { "app.ai.provider_error.status": error.status } - : {}), + ...(error.status !== undefined ? { "app.ai.provider_error.status": error.status } : {}), ...(error.retryAfterMs !== undefined ? { "app.ai.provider_error.retry_after_ms": error.retryAfterMs } : {}), - ...(error.provider - ? { "gen_ai.provider.name": error.provider } - : {}), - ...(error.modelId - ? { "gen_ai.request.model": error.modelId } - : {}), + ...(error.provider ? { "gen_ai.provider.name": error.provider } : {}), + ...(error.modelId ? { "gen_ai.request.model": error.modelId } : {}), }; } diff --git a/packages/junior/src/chat/services/provider-retry.ts b/packages/junior/src/chat/services/provider-retry.ts index 16c4f3a9d..4f749cb85 100644 --- a/packages/junior/src/chat/services/provider-retry.ts +++ b/packages/junior/src/chat/services/provider-retry.ts @@ -1,26 +1,24 @@ import type { PiMessage } from "@/chat/pi/messages"; -import { ProviderError } from "@/chat/services/provider-error"; - -export { createProviderError } from "@/chat/services/provider-error"; +import { isRetryableProviderMessage } from "@/chat/services/provider-error"; import { getPiMessageRole, trimTrailingAssistantMessages, } from "@/chat/pi/transcript"; const PROVIDER_RETRY_DELAYS_MS = [2_000, 4_000, 8_000] as const; -/** Return whether a provider-boundary error should be retried by the worker. */ -export function isProviderRetryError(error: unknown): boolean { - return error instanceof ProviderError && error.retryable; -} /** Build the next provider retry step from Pi history, if the turn can resume. */ export function nextProviderRetry(args: { attempt: number; + failure?: { stopReason?: string; errorMessage?: string }; messages: PiMessage[]; - retryableFailure: boolean; }): { delayMs: number; messages: PiMessage[] } | undefined { const delayMs = PROVIDER_RETRY_DELAYS_MS[args.attempt]; - if (delayMs === undefined || !args.retryableFailure) { + if ( + delayMs === undefined || + !args.failure || + !isRetryableProviderMessage(args.failure) + ) { return undefined; } diff --git a/packages/junior/src/chat/services/subscribed-decision.ts b/packages/junior/src/chat/services/subscribed-decision.ts index 37fc95ee8..115889e7a 100644 --- a/packages/junior/src/chat/services/subscribed-decision.ts +++ b/packages/junior/src/chat/services/subscribed-decision.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { escapeXml } from "@/chat/xml"; -import { isProviderRetryError } from "@/chat/services/provider-retry"; +import { isProviderRetryError } from "@/chat/services/provider-error"; export enum SubscribedReplyReason { ThreadOptOut = "thread_opt_out", diff --git a/packages/junior/src/chat/services/turn-failure-response.ts b/packages/junior/src/chat/services/turn-failure-response.ts index 38e6e3d5e..d4204d2b5 100644 --- a/packages/junior/src/chat/services/turn-failure-response.ts +++ b/packages/junior/src/chat/services/turn-failure-response.ts @@ -4,6 +4,7 @@ import { getInterruptionMarker } from "@/chat/interruption-marker"; import type { AgentRunResult } from "@/chat/services/turn-result"; import { getProviderErrorAttributes, + getProviderErrorUserMessage, ProviderError, } from "@/chat/services/provider-error"; @@ -146,7 +147,7 @@ export function finalizeFailedTurnReply(args: { const providerUserMessage = args.reply.diagnostics.providerError instanceof ProviderError - ? args.reply.diagnostics.providerError.userMessage + ? getProviderErrorUserMessage(args.reply.diagnostics.providerError) : ""; const failureText = providerUserMessage ? `${providerUserMessage} Reference: \`event_id=${eventId}\`.` diff --git a/packages/junior/src/chat/task-execution/worker.ts b/packages/junior/src/chat/task-execution/worker.ts index fa95f24ff..e6aeba00f 100644 --- a/packages/junior/src/chat/task-execution/worker.ts +++ b/packages/junior/src/chat/task-execution/worker.ts @@ -3,7 +3,7 @@ import type { Destination } from "@sentry/junior-plugin-api"; import { sameDestination } from "@/chat/destination"; import { logException, logInfo, logWarn } from "@/chat/logging"; import type { ConversationStore } from "@/chat/conversations/store"; -import { isProviderRetryError } from "@/chat/services/provider-retry"; +import { isProviderRetryError } from "@/chat/services/provider-error"; import { ConversationQueueMessageRejectedError, type ConversationQueueMessage, diff --git a/packages/junior/tests/integration/slack/subscribed-message-behavior.test.ts b/packages/junior/tests/integration/slack/subscribed-message-behavior.test.ts index 2574f3490..406d13383 100644 --- a/packages/junior/tests/integration/slack/subscribed-message-behavior.test.ts +++ b/packages/junior/tests/integration/slack/subscribed-message-behavior.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { TurnInputCommitLostError } from "@/chat/runtime/turn"; import type { JuniorRuntimeServiceOverrides } from "@/chat/app/services"; -import { createProviderError } from "@/chat/services/provider-retry"; +import { createProviderError } from "@/chat/services/provider-error"; import { createTestChatRuntime } from "../../fixtures/chat-runtime"; import { createTestMessage, diff --git a/packages/junior/tests/unit/services/provider-retry.test.ts b/packages/junior/tests/unit/services/provider-retry.test.ts index 35289c353..f0edfba5e 100644 --- a/packages/junior/tests/unit/services/provider-retry.test.ts +++ b/packages/junior/tests/unit/services/provider-retry.test.ts @@ -1,16 +1,14 @@ import { describe, expect, it } from "vitest"; -import { - isRetryableAssistantError, - type AssistantMessage, -} from "@earendil-works/pi-ai"; +import type { AssistantMessage } from "@earendil-works/pi-ai"; import { fauxAssistantMessage } from "@earendil-works/pi-ai/providers/faux"; import type { PiMessage } from "@/chat/pi/messages"; -import { ProviderError } from "@/chat/services/provider-error"; import { - nextProviderRetry, createProviderError, isProviderRetryError, -} from "@/chat/services/provider-retry"; + isRetryableProviderMessage, + ProviderError, +} from "@/chat/services/provider-error"; +import { nextProviderRetry } from "@/chat/services/provider-retry"; function assistantError(errorMessage: string | undefined): AssistantMessage { return fauxAssistantMessage([], { @@ -59,7 +57,7 @@ describe("provider retry helpers", () => { nextProviderRetry({ attempt: 0, messages: [user, failedAssistant], - retryableFailure: isRetryableAssistantError(failedAssistant), + failure: failedAssistant, }), ).toEqual({ delayMs: 2_000, messages: [user] }); }); @@ -86,7 +84,7 @@ describe("provider retry helpers", () => { nextProviderRetry({ attempt: 0, messages: [user, failedAssistant], - retryableFailure: isRetryableAssistantError(failedAssistant), + failure: failedAssistant, }), ).toEqual({ delayMs: 2_000, messages: [user] }); }); @@ -127,13 +125,13 @@ describe("provider retry helpers", () => { overrides: { attempt?: number; messages?: PiMessage[]; - retryableFailure?: boolean; + failure?: AssistantMessage; } = {}, ) => nextProviderRetry({ attempt: 0, messages: [user, failedAssistant], - retryableFailure: true, + failure: failedAssistant, ...overrides, }); @@ -145,10 +143,10 @@ describe("provider retry helpers", () => { '429 {"error":{"message":"Quota exceeded","type":"insufficient_quota"}}', ), ]) { - expect(isRetryableAssistantError(message)).toBe(false); + expect(isRetryableProviderMessage(message)).toBe(false); } - expect(retry({ retryableFailure: false })).toBeUndefined(); + expect(retry({ failure: assistantError("400 bad request") })).toBeUndefined(); expect(retry({ attempt: 3 })).toBeUndefined(); - expect(retry({ messages: [failedAssistant] })).toBeUndefined(); + expect(retry({ messages: [failedAssistant], failure: undefined })).toBeUndefined(); }); }); From b13de4e5bf428263f2d612327fe3979b5d98e079 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:41:55 +0000 Subject: [PATCH 03/10] test(evals): cover provider capacity failure Co-Authored-By: OpenAI Codex Co-Authored-By: immutable dcramer --- .../core/lifecycle-and-resilience.eval.ts | 40 +++++++++++++++++++ packages/junior-evals/src/behavior-harness.ts | 10 +++++ 2 files changed, 50 insertions(+) diff --git a/packages/junior-evals/evals/core/lifecycle-and-resilience.eval.ts b/packages/junior-evals/evals/core/lifecycle-and-resilience.eval.ts index d37e61ed7..a73d53e73 100644 --- a/packages/junior-evals/evals/core/lifecycle-and-resilience.eval.ts +++ b/packages/junior-evals/evals/core/lifecycle-and-resilience.eval.ts @@ -59,6 +59,46 @@ describeEval("Lifecycle and Resilience", slackEvals, (it) => { expect(visibleThreadReplies(result.session)).toHaveLength(1); }); + it("when xAI remains at capacity, explain the temporary outage without leaking provider details", async ({ + run, + }) => { + const productionCapacityError = + '503 {"type":"error","error":{"type":"service_unavailable_error","message":"Service temporarily unavailable"},"providerMetadata":{"gateway":{"originalModelId":"xai/grok-4.5","resolvedProvider":"xai","fallbacksAvailable":[],"modelAttemptCount":1,"totalProviderAttemptCount":1}}}'; + const result = await run({ + overrides: { + reply_results: [ + { + assistant_message_count: 0, + error_message: productionCapacityError, + outcome: "provider_error", + stop_reason: "error", + text: "", + }, + ], + }, + initialEvents: [mention("Create a Linear ticket for this incident")], + criteria: rubric({ + pass: [ + "The reply says the model is temporarily unavailable or at capacity and asks the user to try again shortly.", + "The reply includes a reference event ID for support.", + ], + fail: [ + "Do not expose xAI, Grok, gateway metadata, raw JSON, HTTP status codes, or provider exception text.", + "Do not describe the failure as an unexplained internal error.", + ], + }), + }); + + const replies = visibleThreadReplies(result.session); + expect(replies).toHaveLength(1); + const text = textContent(replies[0]?.content); + expect(text).toContain("still at capacity"); + expect(text).toMatch(/event_id=[a-f0-9]+/); + expect(text).not.toContain("xai"); + expect(text).not.toContain("service_unavailable_error"); + expect(text).not.toContain("503"); + }); + it("when a short reply is interrupted by the provider, keep the partial answer in one marked post", async ({ run, }) => { diff --git a/packages/junior-evals/src/behavior-harness.ts b/packages/junior-evals/src/behavior-harness.ts index 1f107c2b7..0b0c3809c 100644 --- a/packages/junior-evals/src/behavior-harness.ts +++ b/packages/junior-evals/src/behavior-harness.ts @@ -52,6 +52,7 @@ import { appendAndEnqueueInboundMessage } from "@/chat/task-execution/store"; import { executeAgentRun } from "@/chat/agent"; import { completedAgentRun } from "@/chat/runtime/agent-run-outcome"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; +import { createProviderError } from "@/chat/services/provider-error"; import { addAgentTurnUsage, type AgentTurnUsage } from "@/chat/usage"; import { resumeAwaitingSlackContinuation } from "@/chat/runtime/agent-continue-runner"; import { scheduleAgentContinue } from "@/chat/services/agent-continue"; @@ -1927,6 +1928,15 @@ function buildRuntimeServices( ...(replyResult.stop_reason ? { stopReason: replyResult.stop_reason } : {}), + ...(replyResult.outcome === "provider_error" && + replyResult.error_message + ? { + providerError: createProviderError( + replyResult.error_message, + { modelId: "eval-reply-result" }, + ), + } + : {}), toolCalls: replyResult.tool_calls ?? [], toolErrorCount: replyResult.tool_error_count ?? 0, toolResultCount: replyResult.tool_result_count ?? 0, From ba738acdb2252492b1b3d56b9a45b8d5ac7b54f3 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:11:37 +0000 Subject: [PATCH 04/10] fix(agent): harden provider retry policy Co-Authored-By: OpenAI Codex Co-Authored-By: immutable dcramer Co-Authored-By: Nick --- .../core/lifecycle-and-resilience.eval.ts | 11 +-- .../src/chat/services/provider-error.ts | 82 +++++++++++++++---- .../src/chat/services/provider-retry.ts | 19 +++-- .../unit/services/provider-retry.test.ts | 41 +++++++++- 4 files changed, 123 insertions(+), 30 deletions(-) diff --git a/packages/junior-evals/evals/core/lifecycle-and-resilience.eval.ts b/packages/junior-evals/evals/core/lifecycle-and-resilience.eval.ts index a73d53e73..f050384f5 100644 --- a/packages/junior-evals/evals/core/lifecycle-and-resilience.eval.ts +++ b/packages/junior-evals/evals/core/lifecycle-and-resilience.eval.ts @@ -59,17 +59,17 @@ describeEval("Lifecycle and Resilience", slackEvals, (it) => { expect(visibleThreadReplies(result.session)).toHaveLength(1); }); - it("when xAI remains at capacity, explain the temporary outage without leaking provider details", async ({ + it("when xAI returns a generic 503, explain the temporary outage without leaking provider details", async ({ run, }) => { - const productionCapacityError = + const productionServiceUnavailableError = '503 {"type":"error","error":{"type":"service_unavailable_error","message":"Service temporarily unavailable"},"providerMetadata":{"gateway":{"originalModelId":"xai/grok-4.5","resolvedProvider":"xai","fallbacksAvailable":[],"modelAttemptCount":1,"totalProviderAttemptCount":1}}}'; const result = await run({ overrides: { reply_results: [ { assistant_message_count: 0, - error_message: productionCapacityError, + error_message: productionServiceUnavailableError, outcome: "provider_error", stop_reason: "error", text: "", @@ -79,7 +79,7 @@ describeEval("Lifecycle and Resilience", slackEvals, (it) => { initialEvents: [mention("Create a Linear ticket for this incident")], criteria: rubric({ pass: [ - "The reply says the model is temporarily unavailable or at capacity and asks the user to try again shortly.", + "The reply neutrally explains that the model provider had a temporary problem and asks the user to try again.", "The reply includes a reference event ID for support.", ], fail: [ @@ -92,7 +92,8 @@ describeEval("Lifecycle and Resilience", slackEvals, (it) => { const replies = visibleThreadReplies(result.session); expect(replies).toHaveLength(1); const text = textContent(replies[0]?.content); - expect(text).toContain("still at capacity"); + expect(text).toContain("temporary connection problem"); + expect(text).not.toContain("capacity"); expect(text).toMatch(/event_id=[a-f0-9]+/); expect(text).not.toContain("xai"); expect(text).not.toContain("service_unavailable_error"); diff --git a/packages/junior/src/chat/services/provider-error.ts b/packages/junior/src/chat/services/provider-error.ts index 4b799e869..a797dc408 100644 --- a/packages/junior/src/chat/services/provider-error.ts +++ b/packages/junior/src/chat/services/provider-error.ts @@ -36,7 +36,9 @@ export class ProviderError extends Error { readonly fallbackEligible: boolean; constructor(message: string, details: ProviderErrorDetails, cause?: unknown) { - super(`AI provider error: ${message || "Unknown provider error"}`, { cause }); + super(`AI provider error: ${message || "Unknown provider error"}`, { + cause, + }); this.name = "ProviderError"; this.kind = details.kind; this.status = details.status; @@ -75,8 +77,14 @@ function extractField(message: string, field: string): string | undefined { return message.match(new RegExp(`"${field}"\\s*:\\s*"([^"]+)"`, "i"))?.[1]; } -function extractRetryAfterMs(error: unknown, message: string): number | undefined { - const headers = error instanceof Error ? (error as ErrorShape).response?.headers : undefined; +function extractRetryAfterMs( + error: unknown, + message: string, +): number | undefined { + const headers = + error instanceof Error + ? (error as ErrorShape).response?.headers + : undefined; const retryAfter = headers instanceof Headers ? headers.get("retry-after") @@ -94,18 +102,52 @@ function extractRetryAfterMs(error: unknown, message: string): number | undefine return Number.isNaN(date) ? undefined : Math.max(0, date - Date.now()); } -function classifyProviderError(status: number | undefined, message: string): ProviderErrorKind { - if (/content.?policy|safety|moderation/i.test(message)) return "content_policy"; - if (/insufficient.?quota|quota exceeded|usage limit|monthly usage limit|available balance|out of budget|billing/i.test(message)) return "quota"; - if (status === 401 || /invalid.?api.?key|no api key|authentication|(?:invalid|missing|expired|revoked).{0,24}\bcredentials?\b|\bcredentials?\b.{0,24}(?:invalid|missing|expired|revoked)|\bno\b.{0,16}\bcredentials?\b/i.test(message)) return "auth"; - if (status === 403 || /permission|forbidden|authorization/i.test(message)) return "permission"; - if (status === 429 || /rate.?limit|too many requests/i.test(message)) return "rate_limit"; - if (status === 503 || /overloaded|service.?unavailable|capacity/i.test(message)) return "capacity"; +function classifyProviderError( + status: number | undefined, + message: string, +): ProviderErrorKind { + if (/content.?policy|safety|moderation/i.test(message)) + return "content_policy"; + if ( + /insufficient.?quota|quota exceeded|usage limit|monthly usage limit|available balance|out of budget|billing/i.test( + message, + ) + ) + return "quota"; + if ( + status === 401 || + /invalid.?api.?key|no api key|authentication|(?:invalid|missing|expired|revoked).{0,24}\bcredentials?\b|\bcredentials?\b.{0,24}(?:invalid|missing|expired|revoked)|\bno\b.{0,16}\bcredentials?\b/i.test( + message, + ) + ) + return "auth"; + if (status === 403 || /permission|forbidden|authorization/i.test(message)) + return "permission"; + if (status === 429 || /rate.?limit|too many requests/i.test(message)) + return "rate_limit"; + if (/overloaded|at capacity|capacity exceeded/i.test(message)) + return "capacity"; if (/timed? out|timeout/i.test(message)) return "timeout"; - if (/network.?error|connection|fetch failed|socket|stream ended|ended before|ECONNRESET/i.test(message)) return "network"; + if ( + /network.?error|connection|fetch failed|socket|stream ended|ended before|ECONNRESET/i.test( + message, + ) + ) + return "network"; if (status !== undefined && status >= 500) return "server"; + if ( + /service.?unavailable|internal server error|upstream (?:request )?failed|unexpected error occurred/i.test( + message, + ) + ) + return "server"; if (status !== undefined && status >= 400) return "invalid_request"; - if (/context.?length|context.?window|validation|bad request|unsupported model|invalid model|unknown (?:ai gateway )?model|mismatched api/i.test(message)) return "invalid_request"; + if ( + /context.?length|context.?window|validation|bad request|unsupported model|invalid model|unknown (?:ai gateway )?model|mismatched api/i.test( + message, + ) + ) + return "invalid_request"; return "unknown"; } @@ -127,9 +169,13 @@ export function createProviderError( modelId: context.modelId ?? extractField(message, "originalModelId"), retryable: Boolean(message) && - ["rate_limit", "capacity", "timeout", "network", "server"].includes(kind), + ["rate_limit", "capacity", "timeout", "network", "server"].includes( + kind, + ), retryAfterMs: extractRetryAfterMs(error, message), - fallbackEligible: ["capacity", "timeout", "network", "server"].includes(kind), + fallbackEligible: ["capacity", "timeout", "network", "server"].includes( + kind, + ), }, error, ); @@ -180,12 +226,16 @@ export function getProviderErrorUserMessage(error: ProviderError): string { } /** Return safe structured fields for terminal failure telemetry. */ -export function getProviderErrorAttributes(error: ProviderError): Record { +export function getProviderErrorAttributes( + error: ProviderError, +): Record { return { "app.ai.provider_error.kind": error.kind, "app.ai.provider_error.retryable": error.retryable, "app.ai.provider_error.fallback_eligible": error.fallbackEligible, - ...(error.status !== undefined ? { "app.ai.provider_error.status": error.status } : {}), + ...(error.status !== undefined + ? { "app.ai.provider_error.status": error.status } + : {}), ...(error.retryAfterMs !== undefined ? { "app.ai.provider_error.retry_after_ms": error.retryAfterMs } : {}), diff --git a/packages/junior/src/chat/services/provider-retry.ts b/packages/junior/src/chat/services/provider-retry.ts index 4f749cb85..ccd41b4fa 100644 --- a/packages/junior/src/chat/services/provider-retry.ts +++ b/packages/junior/src/chat/services/provider-retry.ts @@ -1,11 +1,12 @@ import type { PiMessage } from "@/chat/pi/messages"; -import { isRetryableProviderMessage } from "@/chat/services/provider-error"; +import { createProviderError } from "@/chat/services/provider-error"; import { getPiMessageRole, trimTrailingAssistantMessages, } from "@/chat/pi/transcript"; const PROVIDER_RETRY_DELAYS_MS = [2_000, 4_000, 8_000] as const; +const MAX_PROVIDER_RETRY_DELAY_MS = 60_000; /** Build the next provider retry step from Pi history, if the turn can resume. */ export function nextProviderRetry(args: { @@ -13,14 +14,22 @@ export function nextProviderRetry(args: { failure?: { stopReason?: string; errorMessage?: string }; messages: PiMessage[]; }): { delayMs: number; messages: PiMessage[] } | undefined { - const delayMs = PROVIDER_RETRY_DELAYS_MS[args.attempt]; + const backoffMs = PROVIDER_RETRY_DELAYS_MS[args.attempt]; if ( - delayMs === undefined || - !args.failure || - !isRetryableProviderMessage(args.failure) + backoffMs === undefined || + args.failure?.stopReason !== "error" || + !args.failure.errorMessage ) { return undefined; } + const providerError = createProviderError(args.failure.errorMessage); + if (!providerError.retryable) { + return undefined; + } + const delayMs = Math.min( + MAX_PROVIDER_RETRY_DELAY_MS, + Math.max(backoffMs, providerError.retryAfterMs ?? 0), + ); const messages = trimTrailingAssistantMessages(args.messages); if (messages.length === args.messages.length) { diff --git a/packages/junior/tests/unit/services/provider-retry.test.ts b/packages/junior/tests/unit/services/provider-retry.test.ts index f0edfba5e..90e95f10d 100644 --- a/packages/junior/tests/unit/services/provider-retry.test.ts +++ b/packages/junior/tests/unit/services/provider-retry.test.ts @@ -71,7 +71,7 @@ describe("provider retry helpers", () => { const providerError = createProviderError(XAI_SERVICE_UNAVAILABLE); expect(providerError).toMatchObject({ - kind: "capacity", + kind: "server", status: 503, provider: "xai", modelId: "xai/grok-4.5", @@ -89,7 +89,7 @@ describe("provider retry helpers", () => { ).toEqual({ delayMs: 2_000, messages: [user] }); }); - it("normalizes rate-limit hints", () => { + it("normalizes rate-limit hints and honors them when planning retries", () => { const error = Object.assign(new Error("429 too many requests"), { status: 429, response: { headers: { "retry-after": "30" } }, @@ -102,6 +102,35 @@ describe("provider retry helpers", () => { retryAfterMs: 30_000, fallbackEligible: false, }); + + const user = { + role: "user", + content: [{ type: "text", text: "help" }], + } as PiMessage; + const failedAssistant = assistantError( + '429 too many requests {"retry-after":"30"}', + ); + expect( + nextProviderRetry({ + attempt: 0, + messages: [user, failedAssistant], + failure: failedAssistant, + }), + ).toEqual({ delayMs: 30_000, messages: [user] }); + }); + + it("retries common unstructured server failures", () => { + for (const message of [ + "Internal Server Error", + "upstream request failed", + "An unexpected error occurred", + ]) { + expect(createProviderError(message)).toMatchObject({ + kind: "server", + retryable: true, + fallbackEligible: true, + }); + } }); it("does not retry explicit credential failures", () => { @@ -145,8 +174,12 @@ describe("provider retry helpers", () => { ]) { expect(isRetryableProviderMessage(message)).toBe(false); } - expect(retry({ failure: assistantError("400 bad request") })).toBeUndefined(); + expect( + retry({ failure: assistantError("400 bad request") }), + ).toBeUndefined(); expect(retry({ attempt: 3 })).toBeUndefined(); - expect(retry({ messages: [failedAssistant], failure: undefined })).toBeUndefined(); + expect( + retry({ messages: [failedAssistant], failure: undefined }), + ).toBeUndefined(); }); }); From 7caec3d22fbaf5802e828d83ddc30f7d4f7b57b1 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:32:22 +0000 Subject: [PATCH 05/10] test(agent): update generic 503 expectations Co-Authored-By: OpenAI Codex --- .../junior/tests/unit/services/turn-failure-response.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/junior/tests/unit/services/turn-failure-response.test.ts b/packages/junior/tests/unit/services/turn-failure-response.test.ts index b46a445b1..8d36bb35b 100644 --- a/packages/junior/tests/unit/services/turn-failure-response.test.ts +++ b/packages/junior/tests/unit/services/turn-failure-response.test.ts @@ -62,11 +62,12 @@ describe("finalizeFailedTurnReply", () => { context: {}, }); - expect(finalized.text).toContain("still at capacity"); + expect(finalized.text).toContain("temporary connection problem"); + expect(finalized.text).not.toContain("capacity"); expect(finalized.text).toContain("event_id=evt_503"); expect(finalized.text).not.toContain("Service temporarily unavailable"); expect(logException.mock.calls[0]?.[3]).toMatchObject({ - "app.ai.provider_error.kind": "capacity", + "app.ai.provider_error.kind": "server", "app.ai.provider_error.status": 503, "app.ai.provider_error.retryable": true, }); From 4a6654c95dbe8af279fa7e680b94adf5b424ee79 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:39:10 +0000 Subject: [PATCH 06/10] test(runtime): Keep retry fixture explicitly transient --- .../tests/integration/runtime/agent-run-agent-continue.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/junior/tests/integration/runtime/agent-run-agent-continue.test.ts b/packages/junior/tests/integration/runtime/agent-run-agent-continue.test.ts index 42619a387..8a134e410 100644 --- a/packages/junior/tests/integration/runtime/agent-run-agent-continue.test.ts +++ b/packages/junior/tests/integration/runtime/agent-run-agent-continue.test.ts @@ -140,7 +140,8 @@ vi.mock("@earendil-works/pi-agent-core", () => { role: "assistant", content: [{ type: "text", text: "provider error" }], stopReason: "error", - errorMessage: "Provider returned error: 503 service unavailable", + errorMessage: + "Provider returned error: 503 service unavailable: capacity exceeded", }); promptSettled.value = true; return {}; From 9c8cade7a185532431806d81ae3a591f58d83ac0 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:53:46 +0000 Subject: [PATCH 07/10] fix(provider): Address telemetry and classification gaps Co-Authored-By: immutable dcramer Co-Authored-By: Nick --- .../src/chat/services/provider-error.ts | 10 +++++--- .../unit/services/provider-retry.test.ts | 25 +++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/packages/junior/src/chat/services/provider-error.ts b/packages/junior/src/chat/services/provider-error.ts index a797dc408..05d0a8c0f 100644 --- a/packages/junior/src/chat/services/provider-error.ts +++ b/packages/junior/src/chat/services/provider-error.ts @@ -67,7 +67,7 @@ function extractStatus(error: unknown, message: string): number | undefined { if (typeof shaped.statusCode === "number") return shaped.statusCode; } const match = message.match( - /^([45]\d\d)\b|\bstatus(?:Code)?["'=:\s]+([45]\d\d)\b/i, + /^(?:Error:\s*)?([45]\d\d)\b|\bstatus(?:Code)?["'=:\s]+([45]\d\d)\b/i, ); const status = match?.[1] ?? match?.[2]; return status ? Number(status) : undefined; @@ -136,7 +136,7 @@ function classifyProviderError( return "network"; if (status !== undefined && status >= 500) return "server"; if ( - /service.?unavailable|internal server error|upstream (?:request )?failed|unexpected error occurred/i.test( + /service.?unavailable|internal server error|bad gateway|gateway timeout|upstream (?:request )?failed|unexpected error occurred/i.test( message, ) ) @@ -202,7 +202,7 @@ export function isRetryableProviderMessage(message: { export function getProviderErrorUserMessage(error: ProviderError): string { switch (error.kind) { case "capacity": - return "The selected model is temporarily unavailable. I retried it, but it is still at capacity. Please try again shortly."; + return "The selected model is temporarily unavailable because it is at capacity. Please try again shortly."; case "rate_limit": { const seconds = error.retryAfterMs ? Math.max(1, Math.ceil(error.retryAfterMs / 1_000)) @@ -239,7 +239,9 @@ export function getProviderErrorAttributes( ...(error.retryAfterMs !== undefined ? { "app.ai.provider_error.retry_after_ms": error.retryAfterMs } : {}), - ...(error.provider ? { "gen_ai.provider.name": error.provider } : {}), + ...(error.provider + ? { "app.ai.provider_error.resolved_provider": error.provider } + : {}), ...(error.modelId ? { "gen_ai.request.model": error.modelId } : {}), }; } diff --git a/packages/junior/tests/unit/services/provider-retry.test.ts b/packages/junior/tests/unit/services/provider-retry.test.ts index 90e95f10d..14f279a71 100644 --- a/packages/junior/tests/unit/services/provider-retry.test.ts +++ b/packages/junior/tests/unit/services/provider-retry.test.ts @@ -4,6 +4,8 @@ import { fauxAssistantMessage } from "@earendil-works/pi-ai/providers/faux"; import type { PiMessage } from "@/chat/pi/messages"; import { createProviderError, + getProviderErrorAttributes, + getProviderErrorUserMessage, isProviderRetryError, isRetryableProviderMessage, ProviderError, @@ -122,6 +124,8 @@ describe("provider retry helpers", () => { it("retries common unstructured server failures", () => { for (const message of [ "Internal Server Error", + "Error: 502 Bad Gateway", + "Gateway Timeout", "upstream request failed", "An unexpected error occurred", ]) { @@ -133,6 +137,27 @@ describe("provider retry helpers", () => { } }); + it("keeps the gateway provider identity separate from the resolved provider", () => { + const attributes = getProviderErrorAttributes( + createProviderError(XAI_SERVICE_UNAVAILABLE), + ); + + expect(attributes).toMatchObject({ + "app.ai.provider_error.resolved_provider": "xai", + "gen_ai.request.model": "xai/grok-4.5", + }); + expect(attributes).not.toHaveProperty("gen_ai.provider.name"); + }); + + it("does not claim a retry happened in terminal capacity copy", () => { + const message = getProviderErrorUserMessage( + createProviderError("Provider capacity exceeded"), + ); + + expect(message).toContain("at capacity"); + expect(message).not.toContain("retried"); + }); + it("does not retry explicit credential failures", () => { for (const message of [ "Missing AI gateway credentials (AI_GATEWAY_API_KEY or VERCEL_OIDC_TOKEN)", From 774bbca61fb1ffdabf905ef07bc5a40deaca1def Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 18:00:43 +0000 Subject: [PATCH 08/10] test(provider): Preserve timeout classification Co-Authored-By: immutable dcramer Co-Authored-By: Nick --- packages/junior/tests/unit/services/provider-retry.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/junior/tests/unit/services/provider-retry.test.ts b/packages/junior/tests/unit/services/provider-retry.test.ts index 14f279a71..d93f1f8c8 100644 --- a/packages/junior/tests/unit/services/provider-retry.test.ts +++ b/packages/junior/tests/unit/services/provider-retry.test.ts @@ -125,7 +125,6 @@ describe("provider retry helpers", () => { for (const message of [ "Internal Server Error", "Error: 502 Bad Gateway", - "Gateway Timeout", "upstream request failed", "An unexpected error occurred", ]) { @@ -135,6 +134,11 @@ describe("provider retry helpers", () => { fallbackEligible: true, }); } + expect(createProviderError("Gateway Timeout")).toMatchObject({ + kind: "timeout", + retryable: true, + fallbackEligible: true, + }); }); it("keeps the gateway provider identity separate from the resolved provider", () => { From 404cf26461b4f69ea27a60f2370d6b975d846576 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 18:30:34 +0000 Subject: [PATCH 09/10] ref(provider): Remove unused resolved provider field Co-Authored-By: OpenAI GPT-5.4 Co-Authored-By: immutable dcramer Co-Authored-By: Nick --- .../junior/src/chat/services/provider-error.ts | 7 ------- .../tests/unit/services/provider-retry.test.ts | 14 -------------- .../unit/services/turn-failure-response.test.ts | 2 +- 3 files changed, 1 insertion(+), 22 deletions(-) diff --git a/packages/junior/src/chat/services/provider-error.ts b/packages/junior/src/chat/services/provider-error.ts index 05d0a8c0f..5f0d74fa2 100644 --- a/packages/junior/src/chat/services/provider-error.ts +++ b/packages/junior/src/chat/services/provider-error.ts @@ -13,7 +13,6 @@ export type ProviderErrorKind = interface ProviderErrorContext { modelId?: string; - provider?: string; } interface ProviderErrorDetails extends ProviderErrorContext { @@ -29,7 +28,6 @@ export class ProviderError extends Error { readonly code = "provider_error"; readonly kind: ProviderErrorKind; readonly status?: number; - readonly provider?: string; readonly modelId?: string; readonly retryable: boolean; readonly retryAfterMs?: number; @@ -42,7 +40,6 @@ export class ProviderError extends Error { this.name = "ProviderError"; this.kind = details.kind; this.status = details.status; - this.provider = details.provider; this.modelId = details.modelId; this.retryable = details.retryable; this.retryAfterMs = details.retryAfterMs; @@ -165,7 +162,6 @@ export function createProviderError( { kind, status, - provider: context.provider ?? extractField(message, "resolvedProvider"), modelId: context.modelId ?? extractField(message, "originalModelId"), retryable: Boolean(message) && @@ -239,9 +235,6 @@ export function getProviderErrorAttributes( ...(error.retryAfterMs !== undefined ? { "app.ai.provider_error.retry_after_ms": error.retryAfterMs } : {}), - ...(error.provider - ? { "app.ai.provider_error.resolved_provider": error.provider } - : {}), ...(error.modelId ? { "gen_ai.request.model": error.modelId } : {}), }; } diff --git a/packages/junior/tests/unit/services/provider-retry.test.ts b/packages/junior/tests/unit/services/provider-retry.test.ts index d93f1f8c8..da16e30c4 100644 --- a/packages/junior/tests/unit/services/provider-retry.test.ts +++ b/packages/junior/tests/unit/services/provider-retry.test.ts @@ -4,7 +4,6 @@ import { fauxAssistantMessage } from "@earendil-works/pi-ai/providers/faux"; import type { PiMessage } from "@/chat/pi/messages"; import { createProviderError, - getProviderErrorAttributes, getProviderErrorUserMessage, isProviderRetryError, isRetryableProviderMessage, @@ -75,7 +74,6 @@ describe("provider retry helpers", () => { expect(providerError).toMatchObject({ kind: "server", status: 503, - provider: "xai", modelId: "xai/grok-4.5", retryable: true, fallbackEligible: true, @@ -141,18 +139,6 @@ describe("provider retry helpers", () => { }); }); - it("keeps the gateway provider identity separate from the resolved provider", () => { - const attributes = getProviderErrorAttributes( - createProviderError(XAI_SERVICE_UNAVAILABLE), - ); - - expect(attributes).toMatchObject({ - "app.ai.provider_error.resolved_provider": "xai", - "gen_ai.request.model": "xai/grok-4.5", - }); - expect(attributes).not.toHaveProperty("gen_ai.provider.name"); - }); - it("does not claim a retry happened in terminal capacity copy", () => { const message = getProviderErrorUserMessage( createProviderError("Provider capacity exceeded"), diff --git a/packages/junior/tests/unit/services/turn-failure-response.test.ts b/packages/junior/tests/unit/services/turn-failure-response.test.ts index 8d36bb35b..6543a2955 100644 --- a/packages/junior/tests/unit/services/turn-failure-response.test.ts +++ b/packages/junior/tests/unit/services/turn-failure-response.test.ts @@ -48,7 +48,7 @@ describe("finalizeFailedTurnReply", () => { const logException = vi.fn().mockReturnValue("evt_503"); const providerError = createProviderError( '503 {"error":{"message":"Service temporarily unavailable"}}', - { modelId: "xai/grok-4.5", provider: "xai" }, + { modelId: "xai/grok-4.5" }, ); const finalized = finalizeFailedTurnReply({ From 22698f0983d34d5767e3f0800758d231d2ab35a0 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 18:36:31 +0000 Subject: [PATCH 10/10] fix(provider): Read AI SDK retry headers Co-Authored-By: OpenAI GPT-5.4 --- packages/junior/src/chat/services/provider-error.ts | 8 ++++++-- .../junior/tests/unit/services/provider-retry.test.ts | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/junior/src/chat/services/provider-error.ts b/packages/junior/src/chat/services/provider-error.ts index 5f0d74fa2..40caa2392 100644 --- a/packages/junior/src/chat/services/provider-error.ts +++ b/packages/junior/src/chat/services/provider-error.ts @@ -50,6 +50,7 @@ export class ProviderError extends Error { type ErrorShape = Error & { status?: unknown; statusCode?: unknown; + responseHeaders?: unknown; response?: { headers?: unknown }; }; @@ -80,13 +81,16 @@ function extractRetryAfterMs( ): number | undefined { const headers = error instanceof Error - ? (error as ErrorShape).response?.headers + ? ((error as ErrorShape).responseHeaders ?? + (error as ErrorShape).response?.headers) : undefined; const retryAfter = headers instanceof Headers ? headers.get("retry-after") : headers && typeof headers === "object" - ? (headers as Record)["retry-after"] + ? Object.entries(headers as Record).find( + ([name]) => name.toLowerCase() === "retry-after", + )?.[1] : undefined; const raw = typeof retryAfter === "string" || typeof retryAfter === "number" diff --git a/packages/junior/tests/unit/services/provider-retry.test.ts b/packages/junior/tests/unit/services/provider-retry.test.ts index da16e30c4..8f171706c 100644 --- a/packages/junior/tests/unit/services/provider-retry.test.ts +++ b/packages/junior/tests/unit/services/provider-retry.test.ts @@ -90,9 +90,9 @@ describe("provider retry helpers", () => { }); it("normalizes rate-limit hints and honors them when planning retries", () => { - const error = Object.assign(new Error("429 too many requests"), { - status: 429, - response: { headers: { "retry-after": "30" } }, + const error = Object.assign(new Error("rate limited"), { + statusCode: 429, + responseHeaders: { "Retry-After": "30" }, }); expect(createProviderError(error)).toMatchObject({