diff --git a/CHANGELOG.md b/CHANGELOG.md index fda4778..10f64e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ Release tags use the form `vX.Y.Z` and match `package.json`. GitHub Releases car ## [Unreleased] +## [0.5.11] - 2026-08-12 + +### Fixed + +- Connection-level failures now retry twice at ten-second intervals, preserve their underlying network cause in logs, advance named-route fallback after exhaustion, and never create misleading OAuth account locks. + ## [0.5.10] - 2026-07-29 ### Fixed @@ -108,7 +114,8 @@ Release tags use the form `vX.Y.Z` and match `package.json`. GitHub Releases car See [GitHub Releases](https://github.com/gitcommit90/rerouted/releases) for artifact digests and notes prior to the Keep a Changelog narrative. Notable themes in late 0.4.x included signed/notarized distribution, in-app updates, named routes, OAuth account pools, OpenAI chat completions and Responses routing, and launch hardening. -[Unreleased]: https://github.com/gitcommit90/rerouted/compare/v0.5.10...HEAD +[Unreleased]: https://github.com/gitcommit90/rerouted/compare/v0.5.11...HEAD +[0.5.11]: https://github.com/gitcommit90/rerouted/compare/v0.5.10...v0.5.11 [0.5.10]: https://github.com/gitcommit90/rerouted/releases/tag/v0.5.10 [0.5.9]: https://github.com/gitcommit90/rerouted/releases/tag/v0.5.9 [0.5.8]: https://github.com/gitcommit90/rerouted/releases/tag/v0.5.8 diff --git a/package-lock.json b/package-lock.json index 253ef2e..db42db2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gitcommit90/rerouted", - "version": "0.5.10", + "version": "0.5.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@gitcommit90/rerouted", - "version": "0.5.10", + "version": "0.5.11", "license": "MIT", "bin": { "rerouted": "src/cli/index.js" diff --git a/package.json b/package.json index 2f51024..9e548d9 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@gitcommit90/rerouted", "productName": "ReRouted", - "version": "0.5.10", + "version": "0.5.11", "description": "A local AI router for connected accounts, models, named routes, and automatic fallback.", "author": "gitcommit90", "license": "MIT", diff --git a/src/lib/router.js b/src/lib/router.js index 08941ed..53175ca 100644 --- a/src/lib/router.js +++ b/src/lib/router.js @@ -24,6 +24,8 @@ const COOLDOWN_MS = { auth: 2 * 60_000, transient: 30_000, }; +const TRANSPORT_RETRY_DELAY_MS = 10_000; +const TRANSPORT_RETRY_ATTEMPTS = 2; const PREOUTPUT_INSPECTION_BYTES = 64 * 1024; const CLAUDE_CODE_CANONICAL_ROUTES = new Map([ ["claude-fable-5", "fable"], @@ -746,6 +748,66 @@ function isAbortError(err) { return /aborted|timeout|TimeoutError/i.test(msg); } +function transportErrorMessage(error) { + const parts = []; + const seen = new Set(); + let current = error; + for (let depth = 0; current && depth < 4; depth += 1) { + if (seen.has(current)) break; + seen.add(current); + const code = typeof current.code === "string" ? current.code.trim() : ""; + const message = String(current.message || current).trim(); + const detail = code && !message.includes(code) ? `${code}: ${message}` : message; + if (detail && !parts.includes(detail)) parts.push(detail); + current = current.cause; + } + return parts.join("; ") || "Upstream connection failed"; +} + +function isTransportError(error) { + const codes = new Set([ + "EAI_AGAIN", + "ECONNABORTED", + "ECONNREFUSED", + "ECONNRESET", + "EHOSTUNREACH", + "ENETDOWN", + "ENETUNREACH", + "ENOTFOUND", + "EPIPE", + "ETIMEDOUT", + "UND_ERR_CONNECT_TIMEOUT", + "UND_ERR_HEADERS_TIMEOUT", + "UND_ERR_SOCKET", + ]); + const seen = new Set(); + let current = error; + for (let depth = 0; current && depth < 4; depth += 1) { + if (seen.has(current)) break; + seen.add(current); + if (codes.has(current.code)) return true; + if (/\bfetch failed\b/i.test(String(current.message || ""))) return true; + current = current.cause; + } + return false; +} + +function waitForTransportRetry(ms, signal) { + if (signal?.aborted) return Promise.resolve(false); + if (!ms || ms <= 0) return Promise.resolve(true); + return new Promise((resolve) => { + let timer; + const finish = (ready) => { + if (timer) clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + resolve(ready); + }; + const onAbort = () => finish(false); + timer = setTimeout(() => finish(true), ms); + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + function memberSignal(outer, timeoutMs) { const ms = timeoutMs ?? REQUEST_TIMEOUT_MS; const ctrl = new AbortController(); @@ -790,7 +852,16 @@ function memberSignal(outer, timeoutMs) { /** * @param {{ store, fetchImpl?, requestLog?, timeoutMs?, usage?, logger? }} opts */ -function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, logger = appLogger } = {}) { +function createRouter({ + store, + fetchImpl = fetch, + requestLog, + timeoutMs, + usage, + logger = appLogger, + transportRetryDelayMs = TRANSPORT_RETRY_DELAY_MS, + transportRetryAttempts = TRANSPORT_RETRY_ATTEMPTS, +} = {}) { const rrState = new Map(); const log = requestLog || createRequestLog(); let totalRequests = 0; @@ -1158,17 +1229,29 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, return { ok: false, status: 408, - error: e.message || "Upstream request timeout", - cooldownEligible: true, - failureKind: "transient", - defaultCooldownMs: COOLDOWN_MS.transient, + error: transportErrorMessage(e) || "Upstream request timeout", + cooldownEligible: false, + failureKind: "transport", + defaultCooldownMs: 0, + transportError: false, }; } - const error = e.message || String(e); + const error = transportErrorMessage(e); const classification = classifyFailure(502, error); + if (classification.kind !== "request" && isTransportError(e)) { + return { + ok: false, + status: 502, + error, + cooldownEligible: false, + failureKind: "transport", + defaultCooldownMs: 0, + transportError: true, + }; + } return { ok: false, - status: classification.kind === "request" ? 400 : 502, + status: 400, error, cooldownEligible: classification.eligible, failureKind: classification.kind, @@ -1290,19 +1373,55 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, } } let result; - try { - result = await tryMember(attemptMember, body, stream, signal); - } catch (e) { - const status = isAbortError(e) ? 408 : 500; - const classification = classifyFailure(status, e.message); - result = { - ok: false, - status, - error: e.message || String(e), - cooldownEligible: classification.eligible, - failureKind: classification.kind, - defaultCooldownMs: classification.defaultCooldownMs, - }; + let transportRetry = 0; + while (true) { + try { + result = await tryMember(attemptMember, body, stream, signal); + } catch (e) { + const status = isAbortError(e) ? 408 : 500; + const error = transportErrorMessage(e); + const classification = classifyFailure(status, error); + const transportError = isTransportError(e); + result = { + ok: false, + status, + error, + cooldownEligible: transportError ? false : classification.eligible, + failureKind: transportError ? "transport" : classification.kind, + defaultCooldownMs: transportError ? 0 : classification.defaultCooldownMs, + transportError, + }; + } + if (!result.transportError || result.canceled || transportRetry >= transportRetryAttempts) { + break; + } + transportRetry += 1; + logger.warn("router transport retry scheduled", { + event: "transport_retry_scheduled", + routeModel: modelId, + upstreamModel: member.upstreamModel, + providerId: provider.id, + providerType: canonicalProviderType(provider.type), + providerName: provider.name, + accountAlias: provider.accountAlias || null, + status: result.status, + error: result.error, + retryAttempt: transportRetry, + maxRetries: transportRetryAttempts, + retryDelayMs: transportRetryDelayMs, + }); + if (!(await waitForTransportRetry(transportRetryDelayMs, signal))) { + result = { + ok: false, + status: 499, + error: "Client disconnected", + cooldownEligible: false, + canceled: true, + failureKind: "canceled", + defaultCooldownMs: 0, + }; + break; + } } if (result.ok) { @@ -1359,7 +1478,14 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, }; } - const savedLock = persistModelLock(provider, member.upstreamModel, result); + const transportCause = result.transportError ? result.error : null; + const publicError = result.transportError + ? `Upstream connection failed after ${transportRetryAttempts} retries` + : result.error; + const savedLock = persistModelLock(provider, member.upstreamModel, { + ...result, + error: publicError, + }); const failed = { providerId: provider.id, providerType: canonicalProviderType(provider.type), @@ -1367,7 +1493,7 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, accountAlias: provider.accountAlias || null, model: member.upstreamModel, status: result.status, - error: result.error, + error: publicError, failureKind: result.failureKind, retryable: true, lockedUntil: savedLock?.until || null, @@ -1378,6 +1504,7 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, event: oauthAttempt ? "account_failure" : "route_member_failure", routeModel: modelId, ...failed, + ...(transportCause ? { transportCause } : {}), }); fallbackFrom = failed; } diff --git a/tests/router-fallback.test.js b/tests/router-fallback.test.js index d735dad..d960021 100644 --- a/tests/router-fallback.test.js +++ b/tests/router-fallback.test.js @@ -320,6 +320,96 @@ describe("Claude Code canonical named routes", () => { }); describe("same-provider OAuth account fallback", () => { + it("retries transport failures without locking the account", async () => { + const store = createStore(tmpConfig()); + store.seed({ providers: [chatgptAccount("prov_a", "token-a", 100)] }); + const logger = captureLogger(); + let calls = 0; + const router = createRouter({ + store, + logger, + transportRetryDelayMs: 0, + fetchImpl: async () => { + calls += 1; + if (calls < 3) { + const cause = new Error("getaddrinfo EAI_AGAIN chatgpt.com"); + cause.code = "EAI_AGAIN"; + throw new TypeError("fetch failed", { cause }); + } + return responsesSuccessResponse("recovered"); + }, + }); + + const result = await router.chatCompletions({ + body: { + model: "chatgpt/gpt-5.4", + messages: [{ role: "user", content: "hello" }], + stream: false, + }, + }); + + assert.equal(result.ok, true, JSON.stringify(result.error)); + assert.equal(calls, 3); + assert.deepEqual(store.load().providers[0].modelLocks, {}); + const retries = logger.entries.filter((entry) => entry.meta?.event === "transport_retry_scheduled"); + assert.equal(retries.length, 2); + assert.match(retries[0].meta.error, /EAI_AGAIN/); + }); + + it("advances a route after transport retries are exhausted without locking the account", async () => { + const store = createStore(tmpConfig()); + store.seed({ + providers: [ + chatgptAccount("prov_chatgpt", "chatgpt-token", 100), + claudeAccount("prov_claude", "claude-token", 200), + ], + combos: [ + { + id: "transport-fallback", + name: "transport-fallback", + strategy: "fallback", + members: [ + { providerId: "prov_chatgpt", model: "gpt-5.4" }, + { providerId: "prov_claude", model: "claude-fable-5" }, + ], + }, + ], + }); + const calls = []; + const logger = captureLogger(); + const router = createRouter({ + store, + logger, + transportRetryDelayMs: 0, + fetchImpl: async (_url, options) => { + const token = authToken(options); + calls.push(token); + if (token === "chatgpt-token") { + const cause = new Error("socket disconnected before TLS handshake"); + cause.code = "ECONNRESET"; + throw new TypeError("fetch failed", { cause }); + } + return claudeSuccessResponse("fallback worked"); + }, + }); + + const result = await router.chatCompletions({ + body: { + model: "transport-fallback", + messages: [{ role: "user", content: "hello" }], + stream: false, + }, + }); + + assert.equal(result.ok, true, JSON.stringify(result.error)); + assert.equal(result.openAiJson.choices[0].message.content, "fallback worked"); + assert.deepEqual(calls, ["chatgpt-token", "chatgpt-token", "chatgpt-token", "claude-token"]); + assert.deepEqual(store.load().providers.find((provider) => provider.id === "prov_chatgpt").modelLocks, {}); + const failure = logger.entries.find((entry) => entry.meta?.event === "account_failure"); + assert.equal(failure.meta.error, "Upstream connection failed after 2 retries"); + assert.match(failure.meta.transportCause, /ECONNRESET/); + }); + it("does not turn context failures into cooldowns or a terminal 429", async () => { const store = createStore(tmpConfig()); const quotaLock = {