diff --git a/.env.example b/.env.example index a5e20f75..457392a8 100644 --- a/.env.example +++ b/.env.example @@ -33,11 +33,32 @@ QINIU_PRIVATE_SPACE=false # 键名前缀必须是 AI_，由 framework/config/provider.py 的 env_prefix 决定。 AI_BASE_URL=https://api.qnaigc.com/v1 AI_API_KEY=your-ai-api-key -# 各能力分开配型号：同时在用不同模型，共用一个字段会换一个连带换全部。 -# 取值即默认值——不写这几行时跑的就是它们。 -AI_CHAT_MODEL=gpt-4o-mini +AI_MODEL=your-chat-model +# 各能力分开配型号：三条能力同时在用不同模型，共用一个字段会换一个连带换全部。 +# 取值即默认值——不写这两行时跑的就是它们。 +AI_CHAT_FALLBACKS= AI_IMAGE_MODEL=gemini-2.5-flash-image AI_VIDEO_MODEL=kling-v2-5-turbo +AI_IMAGE_FALLBACKS= +AI_VIDEO_FALLBACKS=kling-v2-6 +AI_IMAGE_UNIT_COST= +AI_VIDEO_UNIT_COST_PER_SECOND= +AI_PRICE_VERSION=2026-08-16 +# Gateway 路由试运行：primary 留空时复用 AI_BASE_URL / AI_API_KEY； +# fallback 三项都填才启用，用来验证 525 / SSL / 断连等 base_url 级故障切换。 +# 同入口多个 key 用逗号分隔：429 换下一个 key，522/525 跳过该入口剩余 key。 +AI_ROUTE_PRIMARY_NAME=qnaigc-primary +AI_ROUTE_PRIMARY_BASE_URL= +AI_ROUTE_PRIMARY_API_KEY= +AI_ROUTE_PRIMARY_API_KEYS= +AI_ROUTE_FALLBACK_NAME= +AI_ROUTE_FALLBACK_BASE_URL= +AI_ROUTE_FALLBACK_API_KEY= +AI_ROUTE_FALLBACK_API_KEYS= +# 示例：AI_ROUTE_FALLBACK_NAME=qnaigc-backup +# 示例：AI_ROUTE_FALLBACK_BASE_URL=https://backup.example.com/v1 +# 示例：AI_ROUTE_FALLBACK_API_KEY=your-backup-ai-api-key +AI_GATEWAY_LEDGER_ENABLED=true # 判官(看图问答)。必须填一个**能读图的聊天模型**,不是出图模型:它要回 JSON 不是回图。 AI_JUDGE_MODEL=gemini-2.5-flash diff --git a/backend/packages/app/src/windup_app/bootstrap/app.py b/backend/packages/app/src/windup_app/bootstrap/app.py index 7a706aa3..1238350e 100644 --- a/backend/packages/app/src/windup_app/bootstrap/app.py +++ b/backend/packages/app/src/windup_app/bootstrap/app.py @@ -13,6 +13,7 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from windup_framework.db import Base, engine +from windup_framework.gateway.models import AIGatewayAttempt, AIGatewayAttemptDetail # noqa: F401 # 模型导入：触发 Base.metadata 注册，确保 create_all 能发现所有表 from windup_ai_engine.impl.character_namer import LangChainCharacterNamer diff --git a/backend/packages/app/src/windup_app/server/orchestrator/executor.py b/backend/packages/app/src/windup_app/server/orchestrator/executor.py index eb84c142..90d344a7 100644 --- a/backend/packages/app/src/windup_app/server/orchestrator/executor.py +++ b/backend/packages/app/src/windup_app/server/orchestrator/executor.py @@ -26,6 +26,9 @@ from windup_ai_engine.slicing.quality import subject_blobs from windup_common.directions import direction_prompt from windup_common.models import ActionSpec, ActionType as EngineActionType, CharacterCard +from windup_framework.gateway import bind_call_context +from windup_framework.gateway.registry import ModelRegistry +from windup_framework.gateway.types import Scene from windup_framework.config.quality_gate import settings as gate_settings from windup_app.server.orchestrator import billing, quality_gate, task_repo @@ -159,27 +162,17 @@ def step(self, stage: str, i: int, total: int, note: str = "") -> None: logger.info("[gen] %s %s/%s %s", stage, i, total, note) -# 白名单而不是放开任意模型名:每个模型的入参形状不同(image_list / input_reference / -# Fal 队列 + `Authorization: Key`)。列进来却没适配它的协议,等于"看起来能选、点了必然 -# 产生一个用不了的付费任务"。只列 SufyVideoProvider 真能建单的。Refs #239。 -ALLOWED_VIDEO_MODELS: dict[str, str] = { - "kling-v2-5-turbo": "默认。稳,本地首帧即可", - "kling-v2-6": "有 motion-control", -} - - def _resolve_video_model(name: str | None) -> str | None: """校验并返回视频模型名;``None`` 表示用部署默认值。 + 只允许是 CHARACTER_ACTION 链上的一员,含义是「这次从它开始试」。 非法取值在入口炸,不等到付费调用才失败。 """ if name is None: return None - if name not in ALLOWED_VIDEO_MODELS: - raise ValueError( - f"视频模型 {name!r} 不在本期开放列表内。可选:" - + "；".join(f"{k}({v})" for k, v in ALLOWED_VIDEO_MODELS.items()) - ) + chain = ModelRegistry.from_settings().chain(Scene.CHARACTER_ACTION) + if name not in chain: + raise ValueError(f"视频模型 {name!r} 不在本期开放列表内。可选:" + "；".join(chain)) return name @@ -223,17 +216,14 @@ def __init__( | None = None, session_factory: Callable[[], Session] | None = None, ) -> None: - self._generator = generator # None → 懒加载真实装配 - # 按视频模型名分桶的 generator 缓存(模型是 provider 的构造参数,不能事后换) - # 三渲二的渲染方向数属于项目约束；同一视频模型在 4 向和 8 向项目中 - # 不是同一个缓存实例，否则先请求的项目会把后续项目的方向数锁死。 - # directions=1 保留旧的 model/None 键，避免已有注入测试和调用方失效； - # 多方向项目用二元组分桶，防止同一模型的方向配置互相污染。 - self._by_model: dict[ - str | None | tuple[str | None, int], CharacterGeneratorPort - ] = {} - # 抠图 / 图生图 provider 与视频模型无关,所有模型桶共用一份:每个抠图实例都会 - # 各自惰性加载一份 ONNX 会话,按桶各建等于把同一个模型在进程里装多次。 + self._generator = generator # None → 懒加载真实装配(一套共享 Gateway) + # 选哪个 kling 是 Gateway 读 start_from_model 的事,不分模型桶。 + # 三渲二的渲染方向数仍属项目约束:4 向和 8 向的相机表不同,必须分桶, + # 否则先请求的项目会把后续项目的方向数锁死。directions=1 用整数 1 + # 做键,已有 `_get_generator()` 调用仍走同一份。 + self._by_model: dict[int, CharacterGeneratorPort] = {} + # 抠图 / 图生图与视频 Gateway 无关方向分桶,所有桶共用一份:每个抠图实例 + # 都会惰性加载一份 ONNX 会话,按桶各建等于把同一个模型在进程里装多次。 self._matte: MatteProvider | None = None self._image: ImageProvider | None = None # 判官同样与视频模型无关,故不分桶。缺省 None 时**不建**实例:建了就意味着每个 @@ -261,14 +251,21 @@ def run_action_task( 先从 ``project`` 取全局约束(朝向/画风/尺寸/方向)再调 ai_engine。``session`` 缺省时自开一个(后台场景);测试可传入自己的 session。 """ + request_id = f"act-{task_id}" own = session is None session = session or self._make_session() + reset = None try: task_repo.update_status(session, task_id, TaskStatus.RUNNING) if own: session.commit() cons = (self._fetch_constraints or _load_constraints)(session, project_id) + reset = bind_call_context( + request_id=request_id, + task_id=str(task_id), + start_from_model=_resolve_video_model(input.video_model), + ) result = self._produce_action(input, cons) task_repo.update_result(session, task_id, _ACTION_RESULT, result) _settle_credit(session, task_id, success=True) @@ -301,6 +298,8 @@ def run_action_task( if own: session.commit() finally: + if reset is not None: + reset() if own: session.close() @@ -369,6 +368,8 @@ def _produce_action( # **不静默回退。** 拿到了 model_3d_url 却下载不下来 / 渲不出来,就报错,不改走 # i2v —— 两条路线的画风、成本、多朝向能力都不同,悄悄换一条等于让调用方拿着 # 错误的前提做后续决定,而帧数、时长、成色全都正常,没有任何一道会红。 + # + # 选哪个 kling 不在这里传:run_action_task 已经 bind_call_context(start_from_model)。 model_url = (input.model_3d_url or "").strip() # 三渲二那支不取母版,而出口的判官闸口要拿它当参照 —— 不先置 None 的话那支会 # 撞 UnboundLocalError,而它只在有 3D 资产的造型上触发。 @@ -433,49 +434,43 @@ def _get_generator( video_model: str | None = None, directions: int = 1, ) -> CharacterGeneratorPort: - """懒装配 CharacterGenerator,按模型名分桶。 + """懒装配 CharacterGenerator(ImageGateway + VideoGateway + matte)。 - 视频 provider 的模型是构造参数,不分桶的话第一个请求指定的模型会被后续所有请求 - 沿用,而调用方以为自己指定了。 + 选哪个 kling 不在装配时定,由 bind_call_context 的 start_from_model 交给 Gateway。 + ``video_model`` 仍接入口传入,但不参与分桶;分桶只为三渲二的方向数。 """ + del video_model if self._generator is not None: return self._generator # 命中缓存的快路径不进锁,否则每个请求都要在这里排一次队。只有装配新桶才上锁, # 锁内重查一次:两个线程同时错过同一个桶时,后进来的那个要看见前一个的成果。 - cache_key: str | None | tuple[str | None, int] - cache_key = video_model if directions == 1 else (video_model, directions) - cached = self._by_model.get(cache_key) + cached = self._by_model.get(directions) if cached is not None: return cached with self._assembly_lock: - cached = self._by_model.get(cache_key) + cached = self._by_model.get(directions) if cached is None: - cached = self._assemble(video_model, directions) - self._by_model[cache_key] = cached + cached = self._assemble(directions) + self._by_model[directions] = cached return cached - def _assemble( - self, video_model: str | None, directions: int - ) -> CharacterGeneratorPort: - """装一个模型桶。**调用方须持有 ``self._assembly_lock``**(会写共用 provider)。""" + def _assemble(self, directions: int) -> CharacterGeneratorPort: + """装一个方向桶。**调用方须持有 ``self._assembly_lock``**(会写共用 provider)。""" from windup_ai_engine.impl import CharacterGenerator from windup_ai_engine.strategy.concrete import ( PerFrameStrategy, VideoFrameStrategy, ) from windup_common.models import GenRoute - from windup_framework.providers import ( - OnnxU2NetMatteProvider, - SufyImageProvider, - SufyVideoProvider, - ) + from windup_framework.gateway import build_image_gateway, build_video_gateway + from windup_framework.gateway.image import _CIRCUIT + from windup_framework.providers import OnnxU2NetMatteProvider if self._matte is None: self._matte = OnnxU2NetMatteProvider() if self._image is None: - self._image = SufyImageProvider() - # 只有它随模型变 —— 模型是构造参数,换模型必须换实例。 - video = SufyVideoProvider(model=video_model) + self._image = build_image_gateway(circuit=_CIRCUIT) + video = build_video_gateway(circuit=_CIRCUIT) # 装配表必须与 GenRoute 对齐。下面那条断言让漏装在装配时暴露,而不是等到某个 # 动作第一次被请求时才炸——注入 generator 的测试走不到这条装配路径,漏了会测试 # 全绿而真实调用全崩。 @@ -574,7 +569,7 @@ class ImageTaskExecutor: def __init__( self, *, - image=None, # None → 懒加载 SufyImageProvider + image=None, # None → 懒加载 ImageGateway matte: MatteProvider | None = None, # None → 懒加载 OnnxU2NetMatteProvider upload: Callable[[bytes], str] | None = None, # None → 真实对象存储上传 fetch_ref: Callable[[str], bytes] @@ -595,13 +590,19 @@ def run_image_task( *, session: Session | None = None, ) -> None: + request_id = f"img-{task_id}" own = session is None session = session or self._make_session() + reset = None try: task_repo.update_status(session, task_id, TaskStatus.RUNNING) if own: session.commit() cons = _load_constraints(session, project_id) # 角色图也受项目约束 + reset = bind_call_context( + request_id=request_id, + task_id=str(task_id), + ) urls, quality = self._produce_image(input, cons) task_repo.update_result( session, @@ -627,6 +628,8 @@ def run_image_task( if own: session.commit() finally: + if reset is not None: + reset() if own: session.close() @@ -711,9 +714,9 @@ def _produce_image( def _get_image(self): if self._image is None: - from windup_framework.providers import SufyImageProvider + from windup_framework.gateway import build_image_gateway - self._image = SufyImageProvider() + self._image = build_image_gateway() return self._image def _get_matte(self): diff --git a/backend/packages/app/src/windup_app/server/orchestrator/model.py b/backend/packages/app/src/windup_app/server/orchestrator/model.py index 491b3eb7..25082871 100644 --- a/backend/packages/app/src/windup_app/server/orchestrator/model.py +++ b/backend/packages/app/src/windup_app/server/orchestrator/model.py @@ -78,9 +78,9 @@ class CharacterActionInput: # 这个动作是否循环播放。``None`` 原样往下传,由编排层兜成一次性:本层替调用方填默认值 # 的话,"没给"和"明确给了 False"从这里起就再也分不开了。 loop: bool | None = None - # 视频模型。``None`` = 用部署配置的默认值(kling-v2-5-turbo)。 - # 取值域见 executor.ALLOWED_VIDEO_MODELS —— 只开放两个,因为每个模型的入参形状不同 - # (image_list / input_reference / Fal 队列),全开等于把三套协议适配塞进一个改动。 + # 视频模型。``None`` = 用部署配置的默认值。取值域为 + # ``ModelRegistry.chain(CHARACTER_ACTION)``(部署默认 + fallbacks);不在链上 → 入口 + # 报错,不到付费调用才失败。选中的型号表示这次从它开始试,由 Gateway 读 start_from_model。 video_model: str | None = None # ── 三渲二(#192)──────────────────────────────────────────────────── # diff --git a/backend/packages/app/src/windup_app/web/api/generation.py b/backend/packages/app/src/windup_app/web/api/generation.py index b4591d13..04339291 100644 --- a/backend/packages/app/src/windup_app/web/api/generation.py +++ b/backend/packages/app/src/windup_app/web/api/generation.py @@ -198,8 +198,8 @@ class CharacterActionGenerateRequest(BaseModel): # 不对称:一次性动作被当成循环会让末帧接回首帧抽搐、产物不可用,反之只是不无缝闭环、 # 仍可用。而且猜错是静默的,帧数/时长/成色全部正常、没有任何一道会红。 loop: bool | None = None - # 视频模型。None = 用部署默认(kling-v2-5-turbo)。取值域见 - # orchestrator.executor.ALLOWED_VIDEO_MODELS;非法值在入口就报错,不到付费调用才失败。 + # 视频模型。None = 用部署默认。取值域见 ModelRegistry.chain(CHARACTER_ACTION); + # 非法值在入口就报错,不到付费调用才失败。选中的型号表示这次从它开始试。 video_model: str | None = None # 这次动作属于哪个造型。给了才可能走三渲二 —— 3D 资产挂在造型一级(#121)。 # 不给则照旧走 i2v(向后兼容:前端接上之前所有调用都是这样)。 diff --git a/backend/packages/common/src/windup_common/enums/model.py b/backend/packages/common/src/windup_common/enums/model.py index 0a032bed..816ae8eb 100644 --- a/backend/packages/common/src/windup_common/enums/model.py +++ b/backend/packages/common/src/windup_common/enums/model.py @@ -15,6 +15,9 @@ class ModelErrorType(str, Enum): NETWORK = "network" # 网络错误(连接失败 / DNS),可重试 AUTH = "auth" # 鉴权失败(密钥错 / 失效),不可重试 INVALID_RESPONSE = "invalid_response" # 返回格式错误(如该出图却返回纯文本 / 空) + UNREACHED = "unreached" # 521/522/523/525,请求大概率未到上游 + MAYBE_BILLED = "maybe_billed" # 520/524/其它可能已计费 5xx + UPSTREAM_FAILED = "upstream_failed" # 视频 job failed/cancelled UNKNOWN = "unknown" # 未知错误 @property @@ -24,4 +27,5 @@ def retryable(self) -> bool: ModelErrorType.RATE_LIMIT, ModelErrorType.TIMEOUT, ModelErrorType.NETWORK, + ModelErrorType.UNREACHED, } diff --git a/backend/packages/framework/src/windup_framework/config/provider.py b/backend/packages/framework/src/windup_framework/config/provider.py index 98c0c7d4..beb654ae 100644 --- a/backend/packages/framework/src/windup_framework/config/provider.py +++ b/backend/packages/framework/src/windup_framework/config/provider.py @@ -1,5 +1,6 @@ """AI Provider 配置。""" +from pydantic import field_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -40,9 +41,53 @@ class AIProviderSettings(BaseSettings): # ``GET /models`` 去核对。 judge_model: str = "gemini-2.5-flash" + chat_fallbacks: str = "" + image_fallbacks: str = "" + video_fallbacks: str = "" + image_unit_cost: float | None = None + video_unit_cost_per_second: float | None = None + price_version: str = "2026-08-16" + + # ── Gateway route spike: base_url / key route candidates ──────────────── + # 第一版仍以 env 管理。primary 留空时复用上面的 AI_BASE_URL / AI_API_KEY; + # fallback 三个字段都填才表示启用一个备用入口。 + # *_API_KEYS 是同入口额外 key（逗号分隔）：429 换 key，UNREACHED 跳过剩余 key。 + route_primary_name: str = "primary" + route_primary_base_url: str = "" + route_primary_api_key: str = "" + route_primary_api_keys: str = "" + route_fallback_name: str = "" + route_fallback_base_url: str = "" + route_fallback_api_key: str = "" + route_fallback_api_keys: str = "" + gateway_ledger_enabled: bool = True + + @field_validator("image_unit_cost", "video_unit_cost_per_second", mode="before") + @classmethod + def _empty_cost_is_none(cls, v): + if v == "" or v is None: + return None + return v + @property def normalized_base_url(self) -> str: return self.base_url.rstrip("/") + @property + def effective_route_primary_base_url(self) -> str: + return (self.route_primary_base_url or self.base_url).rstrip("/") + + @property + def effective_route_primary_api_key(self) -> str: + return self.route_primary_api_key or self.api_key + + @property + def route_fallback_enabled(self) -> bool: + return all(( + self.route_fallback_name.strip(), + self.route_fallback_base_url.strip(), + self.route_fallback_api_key.strip(), + )) + settings = AIProviderSettings() diff --git a/backend/packages/framework/src/windup_framework/gateway/__init__.py b/backend/packages/framework/src/windup_framework/gateway/__init__.py new file mode 100644 index 00000000..da0396e9 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/__init__.py @@ -0,0 +1,17 @@ +from windup_framework.gateway.chat import ChatGateway, build_chat_gateway +from windup_framework.gateway.context import bind_call_context +from windup_framework.gateway.image import ImageGateway, build_image_gateway +from windup_framework.gateway.models import AIGatewayAttempt, AIGatewayAttemptDetail +from windup_framework.gateway.video import VideoGateway, build_video_gateway + +__all__ = [ + "AIGatewayAttempt", + "AIGatewayAttemptDetail", + "ChatGateway", + "ImageGateway", + "VideoGateway", + "bind_call_context", + "build_chat_gateway", + "build_image_gateway", + "build_video_gateway", +] diff --git a/backend/packages/framework/src/windup_framework/gateway/chat.py b/backend/packages/framework/src/windup_framework/gateway/chat.py new file mode 100644 index 00000000..be961be7 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/chat.py @@ -0,0 +1,445 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import time +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any + +from langchain_openai import ChatOpenAI + +from windup_common.enums.model import ModelErrorType +from windup_framework.config.provider import AIProviderSettings, settings as default_settings +from windup_framework.gateway.circuit import CircuitBreaker +from windup_framework.gateway.context import current_call_context +from windup_framework.gateway.policy import decide +from windup_framework.gateway.routes import ( + GatewayRoute, + config_for_route, + key_circuit_id, + lookup_adapter, + routes_from_settings, +) +from windup_framework.gateway.trace import AttemptDetail, AttemptTrace, emit +from windup_framework.gateway.types import Family, NextStep, Scene + +_CIRCUIT = CircuitBreaker() +_DEFAULT_RETRY_AFTER_S = 2.0 +_SLEEP_CAP_S = 30.0 + + +@dataclass(frozen=True) +class ChatAdapterResult: + ok: bool + value: Any = None + error_type: ModelErrorType | None = None + http_status: int | None = None + edge_fingerprint: str = "" + retry_after_s: float | None = None + provider_usage: object | None = None + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _parse_fallbacks(raw: str) -> tuple[str, ...]: + return tuple(part.strip() for part in raw.split(",") if part.strip()) + + +def _hash_messages(messages: Any) -> str: + payload = json.dumps(messages, ensure_ascii=False, default=str, sort_keys=True) + return hashlib.sha256(payload.encode()).hexdigest() + + +def _error_type_from_exception(exc: Exception) -> tuple[ModelErrorType, int | None, str]: + from windup_framework.gateway.classify import classify_exception + + return classify_exception(exc) + + +class LangChainChatAdapter: + """Protocol adapter: Gateway policy around LangChain's ChatOpenAI client.""" + + def __init__( + self, + config: AIProviderSettings, + *, + tools: list[Any] | None = None, + **client_kwargs: Any, + ) -> None: + self._cfg = config + self._client_kwargs = client_kwargs + self._tools = tools + + def bind_tools(self, tools: list[Any]) -> LangChainChatAdapter: + return LangChainChatAdapter( + self._cfg, tools=list(tools), **self._client_kwargs + ) + + def _client(self, model: str) -> Any: + client = ChatOpenAI( + model=model, + api_key=self._cfg.api_key or None, + base_url=self._cfg.normalized_base_url, + timeout=self._cfg.timeout, + # Gateway owns retry/circuit accounting; hidden SDK retries blur attempts. + max_retries=0, + **self._client_kwargs, + ) + if self._tools: + client = client.bind_tools(self._tools) + return client + + def invoke(self, messages: Any, *, model: str, **kwargs: Any) -> ChatAdapterResult: + try: + return ChatAdapterResult(ok=True, value=self._client(model).invoke(messages, **kwargs)) + except Exception as exc: + error_type, status, edge = _error_type_from_exception(exc) + return ChatAdapterResult( + ok=False, + error_type=error_type, + http_status=status, + edge_fingerprint=edge, + ) + + async def ainvoke(self, messages: Any, *, model: str, **kwargs: Any) -> ChatAdapterResult: + try: + value = await self._client(model).ainvoke(messages, **kwargs) + return ChatAdapterResult(ok=True, value=value) + except Exception as exc: + error_type, status, edge = _error_type_from_exception(exc) + return ChatAdapterResult( + ok=False, + error_type=error_type, + http_status=status, + edge_fingerprint=edge, + ) + + async def astream(self, messages: Any, *, model: str, **kwargs: Any): + try: + async for chunk in self._client(model).astream(messages, **kwargs): + yield ChatAdapterResult(ok=True, value=chunk) + except Exception as exc: + error_type, status, edge = _error_type_from_exception(exc) + yield ChatAdapterResult( + ok=False, + error_type=error_type, + http_status=status, + edge_fingerprint=edge, + ) + + +class ChatGateway: + def __init__(self, adapter, circuit, settings, route_adapters=None) -> None: + self._adapter = adapter + self._circuit = circuit + self._settings = settings + self._routes = routes_from_settings(settings, route_group=Scene.CHAT.value) + self._route_adapters = dict(route_adapters or {}) + + def _adapter_for(self, route: GatewayRoute): + return lookup_adapter(self._route_adapters, route, self._adapter) + + @property + def model_name(self) -> str: + return (self._settings.chat_model or self._settings.model).strip() + + def _models(self) -> tuple[str, ...]: + primary = self.model_name + if not primary: + raise RuntimeError("chat gateway requires AI_CHAT_MODEL") + return (primary, *_parse_fallbacks(self._settings.chat_fallbacks)) + + def bind_tools(self, tools: list[Any]) -> ChatGateway: + """LangChain 兼容：``/ai/chat`` 在流式转发前会绑工具定义。""" + bound_default = self._adapter + bind = getattr(self._adapter, "bind_tools", None) + if callable(bind): + bound_default = bind(tools) + bound_routes = {} + for route_id, adapter in self._route_adapters.items(): + route_bind = getattr(adapter, "bind_tools", None) + bound_routes[route_id] = route_bind(tools) if callable(route_bind) else adapter + return ChatGateway( + adapter=bound_default, + circuit=self._circuit, + settings=self._settings, + route_adapters=bound_routes, + ) + + def invoke(self, messages: Any, **kwargs: Any) -> Any: + return asyncio.run(self.ainvoke(messages, **kwargs)) + + async def _adapter_result(self, adapter, messages: Any, model: str, kwargs: dict) -> ChatAdapterResult: + fn = getattr(adapter, "ainvoke", None) + if callable(fn): + return await fn(messages, model=model, **kwargs) + return await asyncio.to_thread(adapter.invoke, messages, model=model, **kwargs) + + async def ainvoke(self, messages: Any, **kwargs: Any) -> Any: + ctx = current_call_context() + request_id = ctx.request_id or str(uuid.uuid4()) + started = time.monotonic() + input_hash = _hash_messages(messages) + models = self._models() + fallback_used = False + fallback_reason: str | None = None + route_reason_override: str | None = None + last_http_status: int | None = None + + def total_ms() -> int: + return int((time.monotonic() - started) * 1000) + + def fail(http_status: int | None) -> None: + raise RuntimeError( + f"chat gateway failed request_id={request_id} http_status={http_status}" + ) + + if self._circuit.is_open("aggregator"): + self._emit( + AttemptTrace( + request_id=request_id, + scene=Scene.CHAT, + model=models[0], + family=Family.CHAT_COMPLETIONS.value, + route=self._routes[0], + attempt_index=0, + retry_count=0, + route_reason="skip_circuit_open", + outcome="failed", + circuit_scope="aggregator", + total_latency_ms=total_ms(), + detail=AttemptDetail(input_hash=input_hash), + ) + ) + fail(None) + + for route_index, route in enumerate(self._routes): + if self._circuit.is_open("base_url:" + route.base_url_id): + if route_index + 1 < len(self._routes): + fallback_used = True + route_reason_override = "base_url_unreached" + continue + fail(last_http_status) + if self._circuit.is_open(key_circuit_id(route)): + if route_index + 1 < len(self._routes): + fallback_used = True + route_reason_override = "key_rate_limit" + continue + fail(last_http_status) + + adapter = self._adapter_for(route) + switch_to_next_route = False + for model_index, model in enumerate(models): + if model_index == 0: + route_reason = route_reason_override or "primary" + elif fallback_reason == "429": + route_reason = "fallback_after_429" + else: + route_reason = "fallback_after_upstream_fail" + + retry_count = 0 + while True: + attempt_t0 = time.monotonic() + started_at = _utc_now() + result = await self._adapter_result(adapter, messages, model, kwargs) + ended_at = _utc_now() + attempt_latency_ms = int((time.monotonic() - attempt_t0) * 1000) + last_http_status = result.http_status + retry_after_ms = ( + int(result.retry_after_s * 1000) + if result.retry_after_s is not None + else None + ) + if result.ok: + self._emit( + AttemptTrace( + request_id=request_id, + scene=Scene.CHAT, + model=model, + family=Family.CHAT_COMPLETIONS.value, + route=route, + attempt_index=model_index, + retry_count=retry_count, + route_reason=route_reason, + outcome="fallback_success" if fallback_used else "success", + http_status=result.http_status, + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + total_latency_ms=total_ms(), + maybe_billed=True, + detail=AttemptDetail( + input_hash=input_hash, + output_bytes=len(str(result.value).encode()), + retry_after_ms=retry_after_ms, + edge_fingerprint=result.edge_fingerprint or None, + provider_usage=result.provider_usage, + ), + ) + ) + return result.value + + error_type = result.error_type or ModelErrorType.UNKNOWN + step = decide( + error_type=error_type, + retry_count=retry_count, + has_job_id=False, + ) + has_next_route = route_index + 1 < len(self._routes) + circuit_scope = None + if step is NextStep.OPEN_AGGREGATOR: + if has_next_route: + self._circuit.open("base_url:" + route.base_url_id) + circuit_scope = "base_url" + else: + self._circuit.open("aggregator") + circuit_scope = "aggregator" + elif step is NextStep.FALLBACK_KEY: + self._circuit.open(key_circuit_id(route)) + circuit_scope = "key" + elif step is NextStep.FALLBACK: + self._circuit.open("model:" + model) + circuit_scope = "model" + + self._emit( + AttemptTrace( + request_id=request_id, + scene=Scene.CHAT, + model=model, + family=Family.CHAT_COMPLETIONS.value, + route=route, + attempt_index=model_index, + retry_count=retry_count, + route_reason=route_reason, + outcome="failed", + circuit_scope=circuit_scope, + error_type=error_type.value, + http_status=result.http_status, + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + total_latency_ms=total_ms(), + maybe_billed=False, + detail=AttemptDetail( + input_hash=input_hash, + retry_after_ms=retry_after_ms, + edge_fingerprint=result.edge_fingerprint or None, + provider_usage=result.provider_usage, + ), + ) + ) + if step is NextStep.OPEN_AGGREGATOR and has_next_route: + fallback_used = True + route_reason_override = "base_url_unreached" + switch_to_next_route = True + break + if step is NextStep.FALLBACK_KEY: + if has_next_route: + fallback_used = True + route_reason_override = "key_rate_limit" + switch_to_next_route = True + break + fail(last_http_status) + if step is NextStep.RETRY_SAME: + if error_type is ModelErrorType.RATE_LIMIT: + wait = ( + result.retry_after_s + if result.retry_after_s is not None + else _DEFAULT_RETRY_AFTER_S + ) + time.sleep(min(wait, _SLEEP_CAP_S)) + retry_count += 1 + continue + if step is NextStep.FALLBACK: + fallback_used = True + fallback_reason = ( + "429" if error_type is ModelErrorType.RATE_LIMIT else "upstream" + ) + break + fail(last_http_status) + if switch_to_next_route: + break + if switch_to_next_route: + continue + route_reason_override = None + + fail(last_http_status) + + async def astream(self, messages: Any, **kwargs: Any): + """LangChain 兼容流式出口。开流前仍跳过已熔断路由；一旦吐出 chunk 不再换路。""" + ctx = current_call_context() + request_id = ctx.request_id or str(uuid.uuid4()) + models = self._models() + if self._circuit.is_open("aggregator"): + raise RuntimeError( + f"chat gateway failed request_id={request_id} http_status=None" + ) + last_http_status: int | None = None + for route_index, route in enumerate(self._routes): + if self._circuit.is_open("base_url:" + route.base_url_id): + continue + if self._circuit.is_open(key_circuit_id(route)): + continue + adapter = self._adapter_for(route) + stream = getattr(adapter, "astream", None) + if not callable(stream): + yield self.invoke(messages, **kwargs) + return + yielded = False + async for result in stream(messages, model=models[0], **kwargs): + if not isinstance(result, ChatAdapterResult): + yielded = True + yield result + continue + last_http_status = result.http_status + if result.ok: + yielded = True + yield result.value + continue + if yielded: + raise RuntimeError( + f"chat gateway failed request_id={request_id} " + f"http_status={result.http_status}" + ) + break + else: + return + if route_index + 1 >= len(self._routes): + break + raise RuntimeError( + f"chat gateway failed request_id={request_id} http_status={last_http_status}" + ) + + def _emit(self, trace: AttemptTrace) -> None: + if not trace.started_at: + trace.started_at = _utc_now() + if not trace.ended_at: + trace.ended_at = _utc_now() + if trace.price_version is None: + trace.price_version = self._settings.price_version + emit(trace) + + +def build_chat_gateway(config=None, *, adapter=None, circuit=None, **client_kwargs: Any) -> ChatGateway: + cfg: AIProviderSettings = config or default_settings + route_adapters = None + if adapter is None: + routes = routes_from_settings(cfg, route_group=Scene.CHAT.value) + route_adapters = { + route.route_id: LangChainChatAdapter(config_for_route(cfg, route), **client_kwargs) + for route in routes + } + adapter = route_adapters[routes[0].route_id] + return ChatGateway( + adapter=adapter, + circuit=circuit if circuit is not None else _CIRCUIT, + settings=cfg, + route_adapters=route_adapters, + ) diff --git a/backend/packages/framework/src/windup_framework/gateway/circuit.py b/backend/packages/framework/src/windup_framework/gateway/circuit.py new file mode 100644 index 00000000..29680cb1 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/circuit.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import threading +import time +from collections.abc import Callable + + +class CircuitBreaker: + def __init__( + self, + *, + cooldown_s: float = 60, + monotonic: Callable[[], float] | None = None, + ) -> None: + self._cooldown_s = cooldown_s + self._monotonic = monotonic or time.monotonic + self._open_until: dict[str, float] = {} + self._lock = threading.Lock() + + def is_open(self, key: str) -> bool: + with self._lock: + until = self._open_until.get(key) + if until is None: + return False + if self._monotonic() >= until: + self._open_until.pop(key, None) + return False + return True + + def open(self, key: str) -> None: + with self._lock: + self._open_until[key] = self._monotonic() + self._cooldown_s diff --git a/backend/packages/framework/src/windup_framework/gateway/classify.py b/backend/packages/framework/src/windup_framework/gateway/classify.py new file mode 100644 index 00000000..b86309ea --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/classify.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +import math + +import httpx + +from windup_common.enums.model import ModelErrorType + +_MAX_RETRY_WAIT = 30.0 + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def retry_after_seconds(value: str) -> float | None: + try: + delay = float(value) + except ValueError: + try: + retry_at = parsedate_to_datetime(value) + except (TypeError, ValueError, OverflowError): + return None + if retry_at.tzinfo is None: + retry_at = retry_at.replace(tzinfo=timezone.utc) + delay = (retry_at.astimezone(timezone.utc) - _utc_now()).total_seconds() + if not math.isfinite(delay): + return None + return min(max(delay, 0.0), _MAX_RETRY_WAIT) + + +def classify_http(status: int) -> ModelErrorType: + if status == 429: + return ModelErrorType.RATE_LIMIT + if status in (401, 403): + return ModelErrorType.AUTH + if status in (521, 522, 523, 525): + return ModelErrorType.UNREACHED + if status in (400, 404): + return ModelErrorType.UNKNOWN + if status >= 500: + return ModelErrorType.MAYBE_BILLED + return ModelErrorType.UNKNOWN + + +def classify_exception(exc: BaseException) -> tuple[ModelErrorType, int | None, str]: + """把没有 HTTP 状态行的传输失败收成策略输入。 + + 对端拆连接、连不上、写出失败:都还没拿到响应,按 UNREACHED(可同路重试)。 + 读超时另算 TIMEOUT:请求可能已经离开本机,不能当成 52x。 + """ + status = getattr(exc, "status_code", None) + response = getattr(exc, "response", None) + if status is None and response is not None: + status = getattr(response, "status_code", None) + if isinstance(status, int): + return classify_http(status), status, str(exc)[:200] + if isinstance( + exc, + ( + httpx.RemoteProtocolError, + httpx.LocalProtocolError, + httpx.ConnectError, + httpx.WriteError, + httpx.NetworkError, + ), + ): + return ModelErrorType.UNREACHED, None, str(exc)[:200] + if isinstance(exc, (httpx.ReadTimeout, httpx.TimeoutException, TimeoutError)): + return ModelErrorType.TIMEOUT, None, str(exc)[:200] + return ModelErrorType.UNKNOWN, None, str(exc)[:200] diff --git a/backend/packages/framework/src/windup_framework/gateway/context.py b/backend/packages/framework/src/windup_framework/gateway/context.py new file mode 100644 index 00000000..ea9c02b1 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/context.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from contextvars import ContextVar, Token +from dataclasses import dataclass +from typing import Callable + + +@dataclass(frozen=True) +class CallContext: + request_id: str | None = None + task_id: str | None = None + user_id: str | None = None + start_from_model: str | None = None + + +_call_context: ContextVar[CallContext] = ContextVar("windup_gateway_call_context", default=CallContext()) + + +def current_call_context() -> CallContext: + return _call_context.get() + + +def bind_call_context( + *, + request_id: str | None = None, + task_id: str | None = None, + user_id: str | None = None, + start_from_model: str | None = None, +) -> Callable[[], None]: + token: Token[CallContext] = _call_context.set( + CallContext( + request_id=request_id, + task_id=task_id, + user_id=user_id, + start_from_model=start_from_model, + ) + ) + + def reset() -> None: + _call_context.reset(token) + + return reset diff --git a/backend/packages/framework/src/windup_framework/gateway/image.py b/backend/packages/framework/src/windup_framework/gateway/image.py new file mode 100644 index 00000000..3ab36e85 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/image.py @@ -0,0 +1,334 @@ +from __future__ import annotations + +import time +import uuid +from datetime import datetime, timezone + +from windup_common.enums.model import ModelErrorType +from windup_framework.config.provider import AIProviderSettings, settings as default_settings +from windup_framework.gateway.circuit import CircuitBreaker +from windup_framework.gateway.context import current_call_context +from windup_framework.gateway.policy import decide +from windup_framework.gateway.registry import ModelRegistry, RegistryError +from windup_framework.gateway.routes import ( + GatewayRoute, + config_for_route, + key_circuit_id, + lookup_adapter, + routes_from_settings, +) +from windup_framework.gateway.trace import ( + AttemptDetail, + AttemptTrace, + emit, + estimate_cost, + hash_bytes, + hash_image_input, +) +from windup_framework.gateway.types import NextStep, Scene + +_CIRCUIT = CircuitBreaker() +_DEFAULT_RETRY_AFTER_S = 2.0 +_SLEEP_CAP_S = 30.0 + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +class ImageGateway: + def __init__(self, registry, adapter, circuit, settings, route_adapters=None) -> None: + self._registry = registry + self._adapter = adapter + self._circuit = circuit + self._settings = settings + self._routes = routes_from_settings(settings, route_group=Scene.CHARACTER_IMAGE.value) + self._route_adapters = dict(route_adapters or {}) + + def _adapter_for(self, route: GatewayRoute): + return lookup_adapter(self._route_adapters, route, self._adapter) + + def gen_image(self, prompt: str, refs: list[bytes]) -> bytes: + ctx = current_call_context() + request_id = ctx.request_id or str(uuid.uuid4()) + started = time.monotonic() + input_hash = hash_image_input(prompt, refs) + last_http_status: int | None = None + fallback_used = False + fallback_reason: str | None = None + route_reason_override: str | None = None + routes = self._routes + + chain = list(self._registry.chain(Scene.CHARACTER_IMAGE)) + if ctx.start_from_model and ctx.start_from_model in chain: + start_i = chain.index(ctx.start_from_model) + models = chain[start_i:] + else: + start_i = 0 + models = chain + + def total_ms() -> int: + return int((time.monotonic() - started) * 1000) + + def fail(http_status: int | None) -> None: + raise RuntimeError( + f"image gateway failed request_id={request_id} http_status={http_status}" + ) + + if self._circuit.is_open("aggregator"): + model = models[0] if models else "" + route = routes[0] + self._emit( + AttemptTrace( + request_id=request_id, + scene=Scene.CHARACTER_IMAGE, + model=model, + route=route, + attempt_index=start_i, + retry_count=0, + route_reason="skip_circuit_open", + outcome="failed", + circuit_scope="aggregator", + total_latency_ms=total_ms(), + detail=AttemptDetail(input_hash=input_hash), + ) + ) + fail(None) + + for route_index, route in enumerate(routes): + if self._circuit.is_open("base_url:" + route.base_url_id): + if route_index + 1 < len(routes): + fallback_used = True + route_reason_override = "base_url_unreached" + continue + fail(last_http_status) + if self._circuit.is_open(key_circuit_id(route)): + if route_index + 1 < len(routes): + fallback_used = True + route_reason_override = "key_rate_limit" + continue + fail(last_http_status) + + adapter = self._adapter_for(route) + switch_to_next_route = False + for i, model in enumerate(models): + attempt_index = start_i + i + if self._circuit.is_open("model:" + model): + fallback_used = True + fallback_reason = "skip" + self._emit( + AttemptTrace( + request_id=request_id, + scene=Scene.CHARACTER_IMAGE, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=0, + route_reason="skip_circuit_open", + outcome="failed", + circuit_scope="model", + fallback_used=fallback_used, + total_latency_ms=total_ms(), + detail=AttemptDetail(input_hash=input_hash), + ) + ) + continue + + if i == 0: + route_reason = route_reason_override or ( + "start_from_caller" + if ctx.start_from_model and ctx.start_from_model in chain + else "primary" + ) + elif fallback_reason == "429": + route_reason = "fallback_after_429" + elif fallback_reason == "skip": + route_reason = "skip_circuit_open" + else: + route_reason = "fallback_after_upstream_fail" + + retry_count = 0 + resend_spent = 0 + while True: + attempt_t0 = time.monotonic() + started_at = _utc_now() + result = adapter.submit_image(prompt, refs, model) + ended_at = _utc_now() + attempt_latency_ms = int((time.monotonic() - attempt_t0) * 1000) + last_http_status = result.http_status + billed = result.ok or result.maybe_billed + cost = estimate_cost( + Scene.CHARACTER_IMAGE, + billed=billed, + seconds=0, + image_unit_cost=self._settings.image_unit_cost, + video_unit_cost_per_second=self._settings.video_unit_cost_per_second, + ) + retry_after_ms = ( + int(result.retry_after_s * 1000) + if result.retry_after_s is not None + else None + ) + if result.ok: + self._emit( + AttemptTrace( + request_id=request_id, + scene=Scene.CHARACTER_IMAGE, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + outcome="fallback_success" if fallback_used else "success", + http_status=result.http_status, + job_id=result.job_id, + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + total_latency_ms=total_ms(), + maybe_billed=True, + cost=cost, + detail=AttemptDetail( + input_hash=input_hash, + output_hash=hash_bytes(result.body), + output_bytes=len(result.body), + expected_bytes=result.expected_bytes, + retry_after_ms=retry_after_ms, + resend_spent=resend_spent, + job_status=result.job_status, + edge_fingerprint=result.edge_fingerprint or None, + provider_usage=result.provider_usage, + ), + ) + ) + return result.body + + error_type = result.error_type or ModelErrorType.UNKNOWN + step = decide( + error_type=error_type, + retry_count=retry_count, + has_job_id=bool(result.job_id), + ) + circuit_scope = None + has_next_route = route_index + 1 < len(routes) + if step is NextStep.OPEN_AGGREGATOR: + if has_next_route: + self._circuit.open("base_url:" + route.base_url_id) + circuit_scope = "base_url" + else: + self._circuit.open("aggregator") + circuit_scope = "aggregator" + elif step is NextStep.FALLBACK_KEY: + self._circuit.open(key_circuit_id(route)) + circuit_scope = "key" + elif step is NextStep.FALLBACK: + self._circuit.open("model:" + model) + circuit_scope = "model" + + self._emit( + AttemptTrace( + request_id=request_id, + scene=Scene.CHARACTER_IMAGE, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + outcome="failed", + circuit_scope=circuit_scope, + error_type=error_type.value, + http_status=result.http_status, + job_id=result.job_id, + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + total_latency_ms=total_ms(), + maybe_billed=result.maybe_billed, + cost=cost, + detail=AttemptDetail( + input_hash=input_hash, + output_bytes=result.output_bytes or None, + expected_bytes=result.expected_bytes, + retry_after_ms=retry_after_ms, + resend_spent=resend_spent, + job_status=result.job_status, + edge_fingerprint=result.edge_fingerprint or None, + provider_usage=result.provider_usage, + ), + ) + ) + if step is NextStep.OPEN_AGGREGATOR and has_next_route: + fallback_used = True + route_reason_override = "base_url_unreached" + switch_to_next_route = True + break + if step is NextStep.FALLBACK_KEY: + if has_next_route: + fallback_used = True + route_reason_override = "key_rate_limit" + switch_to_next_route = True + break + fail(last_http_status) + if step is NextStep.RETRY_SAME: + if error_type is ModelErrorType.RATE_LIMIT: + wait = ( + result.retry_after_s + if result.retry_after_s is not None + else _DEFAULT_RETRY_AFTER_S + ) + time.sleep(min(wait, _SLEEP_CAP_S)) + retry_count += 1 + if error_type is ModelErrorType.UNREACHED: + resend_spent = 1 + continue + if step is NextStep.FALLBACK: + fallback_used = True + fallback_reason = ( + "429" if error_type is ModelErrorType.RATE_LIMIT else "upstream" + ) + break + fail(last_http_status) + if switch_to_next_route: + break + if switch_to_next_route: + continue + route_reason_override = None + + fail(last_http_status) + + def _emit(self, trace: AttemptTrace) -> None: + if not trace.family and trace.model: + try: + trace.family = self._registry.family_of(trace.model).value + except RegistryError: + pass + if not trace.started_at: + trace.started_at = _utc_now() + if not trace.ended_at: + trace.ended_at = _utc_now() + if trace.price_version is None: + trace.price_version = self._settings.price_version + emit(trace) + + +def build_image_gateway(config=None, *, adapter=None, circuit=None) -> ImageGateway: + cfg: AIProviderSettings = config or default_settings + route_adapters = None + if adapter is None: + from windup_framework.providers.sufy import SufyImageProvider + + routes = routes_from_settings(cfg, route_group=Scene.CHARACTER_IMAGE.value) + route_adapters = { + route.route_id: SufyImageProvider(config=config_for_route(cfg, route)) + for route in routes + } + adapter = route_adapters[routes[0].route_id] + return ImageGateway( + ModelRegistry.from_settings(cfg), + adapter, + circuit if circuit is not None else _CIRCUIT, + cfg, + route_adapters=route_adapters, + ) diff --git a/backend/packages/framework/src/windup_framework/gateway/ledger.py b/backend/packages/framework/src/windup_framework/gateway/ledger.py new file mode 100644 index 00000000..e0f0f468 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/ledger.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import logging +import uuid +from datetime import datetime, timezone +from decimal import Decimal +from typing import Any + +from windup_framework.db import SessionLocal +from windup_framework.gateway.models import AIGatewayAttempt, AIGatewayAttemptDetail +from windup_framework.gateway.routes import route_layer_for +from windup_framework.gateway.trace import AttemptDetail, AttemptTrace + +logger = logging.getLogger("windup.gateway.ledger") + + +def _uuid(value: str | None) -> uuid.UUID: + return uuid.UUID(value) if value else uuid.uuid4() + + +def _int_or_none(value: str | None) -> int | None: + if value is None or value == "": + return None + return int(value) + + +def _dt_or_now(value: str | None) -> datetime: + if not value: + return datetime.now(timezone.utc) + return datetime.fromisoformat(value) + + +def _cost_or_none(value: float | None) -> Decimal | None: + if value is None: + return None + return Decimal(str(value)) + + +def _ledger_outcome(value: str | None) -> str: + if value == "fallback_success": + return "success" + if value in {"success", "accepted", "failed"}: + return value + return "failed" + + +def _phase_for(trace: AttemptTrace) -> str: + if trace.scene.value == "chat": + return "chat_sync" + if trace.scene.value == "character_image": + return "image_sync" + return "submit" + + +def _json_or_none(value: Any) -> Any: + if value is None: + return None + if isinstance(value, dict | list | str | int | float | bool): + return value + return {"value": str(value)} + + +def persist_attempt(trace: AttemptTrace, *, session_factory=SessionLocal) -> None: + """Persist one gateway attempt without letting ledger failures affect generation.""" + + attempt_uuid = _uuid(trace.attempt_id) + route = trace.route + try: + with session_factory() as session: + session.add( + AIGatewayAttempt( + request_id=trace.request_id, + attempt_id=attempt_uuid, + task_id=_int_or_none(trace.task_id), + user_id=_int_or_none(trace.user_id), + project_id=None, + scene=trace.scene.value, + attempt_index=trace.attempt_index or 0, + retry_count=trace.retry_count, + route_id=route.route_id, + route_group=route.route_group, + candidate_index=route.candidate_index, + provider_name=route.provider_name, + base_url_id=route.base_url_id, + base_url_host=route.host or "", + api_key_id=route.api_key_id, + model=trace.model, + family=trace.family or "", + route_reason=trace.route_reason or "primary", + route_layer=route_layer_for(trace.route_reason), + circuit_scope=trace.circuit_scope, + phase=_phase_for(trace), + outcome=_ledger_outcome(trace.outcome), + job_id=trace.job_id, + error_type=trace.error_type, + http_status=trace.http_status, + maybe_billed=bool(trace.maybe_billed), + estimated_cost=_cost_or_none(trace.cost), + cost_currency="USD" if trace.cost is not None else None, + price_version=trace.price_version, + started_at=_dt_or_now(trace.started_at), + ended_at=_dt_or_now(trace.ended_at), + attempt_latency_ms=trace.attempt_latency_ms, + ) + ) + session.commit() + except Exception: + logger.exception("Gateway hot ledger write failed request_id=%s", trace.request_id) + return + + detail = trace.detail or AttemptDetail() + try: + with session_factory() as session: + session.add( + AIGatewayAttemptDetail( + attempt_id=attempt_uuid, + request_id=trace.request_id, + task_id=_int_or_none(trace.task_id), + job_status=detail.job_status, + edge_fingerprint=detail.edge_fingerprint, + error_message=None, + provider_request_id=None, + provider_usage=_json_or_none(detail.provider_usage), + input_hash=detail.input_hash, + output_hash=detail.output_hash, + output_bytes=detail.output_bytes, + expected_bytes=detail.expected_bytes, + retry_after_ms=detail.retry_after_ms, + submit_ms=detail.submit_ms, + poll_ms=detail.poll_ms, + download_ms=detail.download_ms, + poll_count=detail.poll_count, + extra=None, + ) + ) + session.commit() + except Exception: + logger.exception("Gateway detail ledger write failed request_id=%s", trace.request_id) diff --git a/backend/packages/framework/src/windup_framework/gateway/models.py b/backend/packages/framework/src/windup_framework/gateway/models.py new file mode 100644 index 00000000..1b22588a --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/models.py @@ -0,0 +1,205 @@ +"""Gateway attempt ledger ORM models. + +The hot/cold split keeps route health and cost attribution queries on a compact +table, while larger troubleshooting payloads live in the detail table. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from decimal import Decimal +from uuid import UUID + +from sqlalchemy import ( + BigInteger, + Boolean, + CheckConstraint, + DateTime, + Index, + Integer, + JSON, + Numeric, + String, + Text, + Uuid, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from windup_framework.db import Base + + +_BIGINT = BigInteger().with_variant(Integer, "sqlite") +_JSONB = JSON().with_variant(JSONB, "postgresql") + + +class AIGatewayAttempt(Base): + """Hot ledger row for one gateway attempt. + + One row represents one actual attempt against a model/key/base_url candidate. + It intentionally stores only compact routing, outcome, and cost fields. + """ + + __tablename__ = "windup_ai_gateway_attempt" + __table_args__ = ( + CheckConstraint( + "scene IN ('chat', 'character_image', 'character_action')", + name="ck_gateway_attempt_scene", + ), + CheckConstraint( + "phase IN ('chat_sync', 'image_sync', 'submit', 'follow', 'download')", + name="ck_gateway_attempt_phase", + ), + CheckConstraint( + "route_layer IN ('none', 'model', 'key', 'base_url')", + name="ck_gateway_attempt_route_layer", + ), + CheckConstraint( + "outcome IN ('success', 'accepted', 'failed')", + name="ck_gateway_attempt_outcome", + ), + CheckConstraint( + "http_status IS NULL OR (http_status >= 100 AND http_status <= 599)", + name="ck_gateway_attempt_http_status", + ), + CheckConstraint( + "estimated_cost IS NULL OR estimated_cost >= 0", + name="ck_gateway_attempt_cost_non_negative", + ), + CheckConstraint( + "attempt_index >= 0 " + "AND retry_count >= 0 " + "AND candidate_index >= 0 " + "AND (attempt_latency_ms IS NULL OR attempt_latency_ms >= 0)", + name="ck_gateway_attempt_non_negative_counts", + ), + Index("ix_gateway_attempt_request", "request_id", "attempt_index"), + Index("ix_gateway_attempt_task", "task_id", "scene", "created_at"), + Index( + "ix_gateway_attempt_provider_error", + "provider_name", + "base_url_id", + "error_type", + "created_at", + ), + Index( + "ix_gateway_attempt_key_error", + "provider_name", + "base_url_id", + "api_key_id", + "error_type", + "created_at", + ), + Index("ix_gateway_attempt_model_error", "model", "error_type", "created_at"), + Index( + "ix_gateway_attempt_route_health", + "route_group", + "route_id", + "outcome", + "created_at", + ), + Index("ix_gateway_attempt_maybe_billed", "maybe_billed", "outcome", "created_at"), + Index("ix_gateway_attempt_job", "job_id"), + ) + + id: Mapped[int] = mapped_column(_BIGINT, primary_key=True, autoincrement=True) + request_id: Mapped[str] = mapped_column(String(96), nullable=False) + attempt_id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), unique=True, nullable=False) + + task_id: Mapped[int | None] = mapped_column(_BIGINT, nullable=True) + user_id: Mapped[int | None] = mapped_column(_BIGINT, nullable=True) + project_id: Mapped[int | None] = mapped_column(_BIGINT, nullable=True) + scene: Mapped[str] = mapped_column(Text, nullable=False) + + attempt_index: Mapped[int] = mapped_column(Integer, nullable=False) + retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + route_id: Mapped[str] = mapped_column(Text, nullable=False) + route_group: Mapped[str] = mapped_column(Text, nullable=False) + candidate_index: Mapped[int] = mapped_column(Integer, nullable=False) + + provider_name: Mapped[str] = mapped_column(Text, nullable=False) + base_url_id: Mapped[str] = mapped_column(Text, nullable=False) + base_url_host: Mapped[str] = mapped_column(Text, nullable=False) + api_key_id: Mapped[str | None] = mapped_column(Text, nullable=True) + + model: Mapped[str] = mapped_column(Text, nullable=False) + family: Mapped[str] = mapped_column(Text, nullable=False) + + route_reason: Mapped[str] = mapped_column(Text, nullable=False) + route_layer: Mapped[str] = mapped_column(Text, nullable=False, default="none") + circuit_scope: Mapped[str | None] = mapped_column(Text, nullable=True) + phase: Mapped[str] = mapped_column(Text, nullable=False) + outcome: Mapped[str] = mapped_column(Text, nullable=False) + + job_id: Mapped[str | None] = mapped_column(Text, nullable=True) + + error_type: Mapped[str | None] = mapped_column(Text, nullable=True) + http_status: Mapped[int | None] = mapped_column(Integer, nullable=True) + + maybe_billed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + estimated_cost: Mapped[Decimal | None] = mapped_column(Numeric(14, 6), nullable=True) + cost_currency: Mapped[str | None] = mapped_column(String(8), nullable=True) + price_version: Mapped[str | None] = mapped_column(Text, nullable=True) + + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + attempt_latency_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + +class AIGatewayAttemptDetail(Base): + """Cold troubleshooting row for one gateway attempt.""" + + __tablename__ = "windup_ai_gateway_attempt_detail" + __table_args__ = ( + CheckConstraint( + "(output_bytes IS NULL OR output_bytes >= 0) " + "AND (expected_bytes IS NULL OR expected_bytes >= 0) " + "AND (retry_after_ms IS NULL OR retry_after_ms >= 0) " + "AND (submit_ms IS NULL OR submit_ms >= 0) " + "AND (poll_ms IS NULL OR poll_ms >= 0) " + "AND (download_ms IS NULL OR download_ms >= 0) " + "AND (poll_count IS NULL OR poll_count >= 0)", + name="ck_gateway_attempt_detail_non_negative_counts", + ), + Index("ix_gateway_attempt_detail_request", "request_id"), + Index("ix_gateway_attempt_detail_task", "task_id", "created_at"), + Index("ix_gateway_attempt_detail_job_status", "job_status", "created_at"), + ) + + id: Mapped[int] = mapped_column(_BIGINT, primary_key=True, autoincrement=True) + attempt_id: Mapped[UUID] = mapped_column(Uuid(as_uuid=True), unique=True, nullable=False) + request_id: Mapped[str] = mapped_column(String(96), nullable=False) + task_id: Mapped[int | None] = mapped_column(_BIGINT, nullable=True) + + job_status: Mapped[str | None] = mapped_column(Text, nullable=True) + edge_fingerprint: Mapped[str | None] = mapped_column(Text, nullable=True) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) + provider_request_id: Mapped[str | None] = mapped_column(Text, nullable=True) + provider_usage: Mapped[dict | None] = mapped_column(_JSONB, nullable=True) + + input_hash: Mapped[str | None] = mapped_column(String(64), nullable=True) + output_hash: Mapped[str | None] = mapped_column(String(64), nullable=True) + output_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) + expected_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) + + retry_after_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) + submit_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) + poll_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) + download_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) + poll_count: Mapped[int | None] = mapped_column(Integer, nullable=True) + + extra: Mapped[dict | None] = mapped_column(_JSONB, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) diff --git a/backend/packages/framework/src/windup_framework/gateway/policy.py b/backend/packages/framework/src/windup_framework/gateway/policy.py new file mode 100644 index 00000000..181b4ea7 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/policy.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from windup_common.enums.model import ModelErrorType +from windup_framework.gateway.types import NextStep + + +def decide( + *, + error_type: ModelErrorType, + retry_count: int, + has_job_id: bool, +) -> NextStep: + if error_type in (ModelErrorType.MAYBE_BILLED, ModelErrorType.AUTH): + return NextStep.FAIL + if has_job_id and error_type is ModelErrorType.TIMEOUT: + return NextStep.FAIL + if has_job_id and error_type is ModelErrorType.UPSTREAM_FAILED: + return NextStep.FALLBACK + if error_type is ModelErrorType.UNREACHED and retry_count == 0: + return NextStep.RETRY_SAME + if error_type is ModelErrorType.UNREACHED and has_job_id: + return NextStep.FAIL + if error_type is ModelErrorType.UNREACHED: + return NextStep.OPEN_AGGREGATOR + if error_type is ModelErrorType.RATE_LIMIT and retry_count < 2: + return NextStep.RETRY_SAME + if error_type is ModelErrorType.RATE_LIMIT: + return NextStep.FALLBACK_KEY + if error_type is ModelErrorType.INVALID_RESPONSE and retry_count < 2: + return NextStep.RETRY_SAME + if error_type is ModelErrorType.INVALID_RESPONSE: + return NextStep.FALLBACK + return NextStep.FAIL diff --git a/backend/packages/framework/src/windup_framework/gateway/registry.py b/backend/packages/framework/src/windup_framework/gateway/registry.py new file mode 100644 index 00000000..77d3ee3d --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/registry.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from windup_framework.config.provider import AIProviderSettings, settings as default_settings +from windup_framework.gateway.types import Family, Scene + +FAMILIES: dict[str, Family] = { + "gemini-2.5-flash-image": Family.IMAGE_CHAT_DATA_URI, + "gemini-2.5-flash-image-alt": Family.IMAGE_CHAT_DATA_URI, # test double; not a production default + "kling-v2-5-turbo": Family.VIDEO_INPUT_REFERENCE, + "kling-v2-6": Family.VIDEO_INPUT_REFERENCE, + "kling-video-o1": Family.VIDEO_IMAGE_LIST, # 登记但不允许进 chain +} + + +class RegistryError(ValueError): + pass + + +def _parse_fallbacks(raw: str) -> tuple[str, ...]: + return tuple(part.strip() for part in raw.split(",") if part.strip()) + + +class ModelRegistry: + def __init__(self, chains: dict[Scene, tuple[str, ...]]) -> None: + self._chains = chains + + @classmethod + def from_settings(cls, cfg: AIProviderSettings | None = None) -> ModelRegistry: + cfg = default_settings if cfg is None else cfg + chains = { + Scene.CHARACTER_IMAGE: (cfg.image_model, *_parse_fallbacks(cfg.image_fallbacks)), + Scene.CHARACTER_ACTION: (cfg.video_model, *_parse_fallbacks(cfg.video_fallbacks)), + } + for scene, models in chains.items(): + cls._validate_chain(scene, models) + return cls(chains) + + @staticmethod + def _validate_chain(scene: Scene, models: tuple[str, ...]) -> None: + families: list[Family] = [] + for model in models: + if model not in FAMILIES: + raise RegistryError(f"未登记型号: {model}") + family = FAMILIES[model] + if scene is Scene.CHARACTER_ACTION and family is Family.VIDEO_IMAGE_LIST: + raise RegistryError( + f"family {family.value} 不允许出现在 {scene.value} 链上: {model}" + ) + families.append(family) + if len(set(families)) > 1: + raise RegistryError( + f"scene {scene.value} 链上 family 不一致: {models}" + ) + + def chain(self, scene: Scene) -> tuple[str, ...]: + return self._chains[scene] + + def family_of(self, model: str) -> Family: + if model not in FAMILIES: + raise RegistryError(f"未登记型号: {model}") + return FAMILIES[model] + + def contains(self, scene: Scene, model: str) -> bool: + return model in self._chains[scene] diff --git a/backend/packages/framework/src/windup_framework/gateway/routes.py b/backend/packages/framework/src/windup_framework/gateway/routes.py new file mode 100644 index 00000000..89db9aae --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/routes.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +from dataclasses import dataclass +from urllib.parse import urlparse + +from windup_framework.config.provider import AIProviderSettings + + +@dataclass(frozen=True) +class GatewayRoute: + route_id: str + route_group: str + candidate_index: int + provider_name: str + base_url_id: str + base_url: str + api_key_id: str | None + api_key: str + + @property + def host(self) -> str | None: + return urlparse(self.base_url).hostname + + +def _parse_csv(raw: str) -> tuple[str, ...]: + return tuple(part.strip() for part in raw.split(",") if part.strip()) + + +def _unique_keys(first: str, extra: str) -> tuple[str, ...]: + keys: list[str] = [] + for key in (first.strip(), *_parse_csv(extra)): + if key and key not in keys: + keys.append(key) + return tuple(keys) or ("",) + + +def _expand_url( + *, + route_group: str, + provider_name: str, + base_url_id: str, + base_url: str, + first_key: str, + extra_keys: str, + start_index: int, +) -> list[GatewayRoute]: + routes: list[GatewayRoute] = [] + for i, api_key in enumerate(_unique_keys(first_key, extra_keys)): + api_key_id = f"{base_url_id}.key{i}" + routes.append( + GatewayRoute( + route_id=api_key_id, + route_group=route_group, + candidate_index=start_index + i, + provider_name=provider_name, + base_url_id=base_url_id, + base_url=base_url, + api_key_id=api_key_id, + api_key=api_key, + ) + ) + return routes + + +def routes_from_settings(cfg: AIProviderSettings, *, route_group: str) -> tuple[GatewayRoute, ...]: + primary_name = cfg.route_primary_name.strip() or "primary" + routes = _expand_url( + route_group=route_group, + provider_name=cfg.provider, + base_url_id=primary_name, + base_url=cfg.effective_route_primary_base_url, + first_key=cfg.effective_route_primary_api_key, + extra_keys=cfg.route_primary_api_keys, + start_index=0, + ) + if cfg.route_fallback_enabled: + fallback_name = cfg.route_fallback_name.strip() + routes.extend( + _expand_url( + route_group=route_group, + provider_name=cfg.provider, + base_url_id=fallback_name, + base_url=cfg.route_fallback_base_url.rstrip("/"), + first_key=cfg.route_fallback_api_key, + extra_keys=cfg.route_fallback_api_keys, + start_index=len(routes), + ) + ) + return tuple(routes) + + +def config_for_route(cfg: AIProviderSettings, route: GatewayRoute) -> AIProviderSettings: + return cfg.model_copy(update={"base_url": route.base_url, "api_key": route.api_key}) + + +def lookup_adapter(route_adapters: dict, route: GatewayRoute, default): + return ( + route_adapters.get(route.route_id) + or route_adapters.get(route.api_key_id) + or route_adapters.get(route.base_url_id) + or default + ) + + +def key_circuit_id(route: GatewayRoute) -> str: + return f"key:{route.base_url_id}:{route.api_key_id}" + + +def route_layer_for(reason: str) -> str: + if reason == "base_url_unreached": + return "base_url" + if reason == "key_rate_limit": + return "key" + if reason in { + "fallback_after_429", + "fallback_after_upstream_fail", + "skip_circuit_open", + "start_from_caller", + }: + return "model" + return "none" diff --git a/backend/packages/framework/src/windup_framework/gateway/trace.py b/backend/packages/framework/src/windup_framework/gateway/trace.py new file mode 100644 index 00000000..b8149d33 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/trace.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import hashlib +import json +import logging +import uuid +from dataclasses import dataclass, fields +from enum import Enum + +from windup_framework.config.provider import settings as provider_settings +from windup_framework.gateway.context import current_call_context +from windup_framework.gateway.routes import GatewayRoute, route_layer_for +from windup_framework.gateway.types import Scene + +logger = logging.getLogger("windup.gateway") + + +@dataclass +class AttemptDetail: + input_hash: str | None = None + output_hash: str | None = None + output_bytes: int | None = None + expected_bytes: int | None = None + retry_after_ms: int | None = None + submit_ms: int | None = None + poll_ms: int | None = None + download_ms: int | None = None + poll_count: int | None = None + resend_spent: int | None = None + job_status: str | None = None + edge_fingerprint: str | None = None + provider_usage: object | None = None + + +@dataclass +class AttemptTrace: + request_id: str + scene: Scene + model: str + route: GatewayRoute + attempt_index: int + retry_count: int + route_reason: str + outcome: str + attempt_id: str | None = None + task_id: str | None = None + user_id: str | None = None + family: str | None = None + circuit_scope: str | None = None + error_type: str | None = None + http_status: int | None = None + job_id: str | None = None + fallback_used: bool = False + started_at: str | None = None + ended_at: str | None = None + attempt_latency_ms: int | None = None + total_latency_ms: int | None = None + maybe_billed: bool | None = None + cost: float | None = None + price_version: str | None = None + detail: AttemptDetail | None = None + + def as_dict(self) -> dict[str, object]: + out: dict[str, object] = {} + for f in fields(self): + if f.name in {"route", "detail"}: + continue + value = getattr(self, f.name) + if isinstance(value, Enum): + value = value.value + out[f.name] = value + out["route_id"] = self.route.route_id + out["route_group"] = self.route.route_group + out["candidate_index"] = self.route.candidate_index + out["provider_name"] = self.route.provider_name + out["base_url_id"] = self.route.base_url_id + out["base_url_host"] = self.route.host + out["api_key_id"] = self.route.api_key_id + out["route_layer"] = route_layer_for(self.route_reason) + detail = self.detail or AttemptDetail() + for f in fields(detail): + out[f.name] = getattr(detail, f.name) + return out + + +def estimate_cost( + scene: Scene, + *, + billed: bool, + seconds: int, + image_unit_cost: float | None = None, + video_unit_cost_per_second: float | None = None, +) -> float | None: + if not billed: + return None + if scene == Scene.CHARACTER_IMAGE: + return image_unit_cost + if scene == Scene.CHARACTER_ACTION: + if video_unit_cost_per_second is None: + return None + return video_unit_cost_per_second * seconds + return None + + +def hash_image_input(prompt: str, refs: list[bytes]) -> str: + payload = prompt.encode() + b"\0".join(refs) + return hashlib.sha256(payload).hexdigest() + + +def hash_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def emit(trace: AttemptTrace) -> None: + ctx = current_call_context() + if not trace.attempt_id: + trace.attempt_id = str(uuid.uuid4()) + if trace.task_id is None: + trace.task_id = ctx.task_id + if trace.user_id is None: + trace.user_id = ctx.user_id + if trace.price_version is None: + trace.price_version = provider_settings.price_version + logger.info("%s", json.dumps(trace.as_dict(), ensure_ascii=False, default=str)) + if provider_settings.gateway_ledger_enabled: + from windup_framework.gateway.ledger import persist_attempt + + persist_attempt(trace) diff --git a/backend/packages/framework/src/windup_framework/gateway/types.py b/backend/packages/framework/src/windup_framework/gateway/types.py new file mode 100644 index 00000000..2566b51b --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/types.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +from windup_common.enums.model import ModelErrorType + + +class Scene(str, Enum): + CHAT = "chat" + CHARACTER_IMAGE = "character_image" + CHARACTER_ACTION = "character_action" + + +class Family(str, Enum): + CHAT_COMPLETIONS = "chat.completions" + IMAGE_CHAT_DATA_URI = "image.chat_data_uri" + VIDEO_INPUT_REFERENCE = "video.input_reference" + VIDEO_IMAGE_LIST = "video.image_list" + + +class NextStep(str, Enum): + RETRY_SAME = "retry_same" + FALLBACK = "fallback" + FALLBACK_KEY = "fallback_key" + FAIL = "fail" + OPEN_AGGREGATOR = "open_aggregator" + + +@dataclass(frozen=True) +class AdapterResult: + ok: bool + body: bytes = b"" + job_id: str | None = None + error_type: ModelErrorType | None = None + http_status: int | None = None + maybe_billed: bool = False + edge_fingerprint: str = "" + output_bytes: int = 0 + expected_bytes: int | None = None + provider_usage: object | None = None + job_status: str | None = None + retry_after_s: float | None = None + poll_ms: int | None = None + download_ms: int | None = None + poll_count: int | None = None diff --git a/backend/packages/framework/src/windup_framework/gateway/video.py b/backend/packages/framework/src/windup_framework/gateway/video.py new file mode 100644 index 00000000..643b26e1 --- /dev/null +++ b/backend/packages/framework/src/windup_framework/gateway/video.py @@ -0,0 +1,419 @@ +from __future__ import annotations + +import time +import uuid +from dataclasses import replace +from datetime import datetime, timezone + +from windup_common.enums.model import ModelErrorType +from windup_framework.config.provider import AIProviderSettings, settings as default_settings +from windup_framework.gateway.context import current_call_context +from windup_framework.gateway.image import _CIRCUIT +from windup_framework.gateway.policy import decide +from windup_framework.gateway.registry import ModelRegistry, RegistryError +from windup_framework.gateway.routes import ( + GatewayRoute, + config_for_route, + key_circuit_id, + lookup_adapter, + routes_from_settings, +) +from windup_framework.gateway.trace import ( + AttemptDetail, + AttemptTrace, + emit, + estimate_cost, + hash_bytes, + hash_image_input, +) +from windup_framework.gateway.types import NextStep, Scene + +_DEFAULT_RETRY_AFTER_S = 2.0 +_SLEEP_CAP_S = 30.0 + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +class VideoGateway: + def __init__(self, registry, adapter, circuit, settings, route_adapters=None) -> None: + self._registry = registry + self._adapter = adapter + self._circuit = circuit + self._settings = settings + self._routes = routes_from_settings(settings, route_group=Scene.CHARACTER_ACTION.value) + self._route_adapters = dict(route_adapters or {}) + + def _adapter_for(self, route: GatewayRoute): + return lookup_adapter(self._route_adapters, route, self._adapter) + + def i2v( + self, + first_frame: bytes, + prompt: str, + seconds: int = 5, + size: str = "1280x720", + ) -> bytes: + ctx = current_call_context() + request_id = ctx.request_id or str(uuid.uuid4()) + started = time.monotonic() + input_hash = hash_image_input(prompt, [first_frame]) + last_http_status: int | None = None + last_error: ModelErrorType | None = None + fallback_used = False + fallback_reason: str | None = None + route_reason_override: str | None = None + routes = self._routes + + chain = list(self._registry.chain(Scene.CHARACTER_ACTION)) + if ctx.start_from_model and ctx.start_from_model in chain: + start_i = chain.index(ctx.start_from_model) + models = chain[start_i:] + else: + start_i = 0 + models = chain + + def total_ms() -> int: + return int((time.monotonic() - started) * 1000) + + def fail(http_status: int | None) -> None: + err = last_error.value if last_error is not None else None + raise RuntimeError( + f"video gateway failed request_id={request_id} " + f"http_status={http_status} error_type={err}" + ) + + if self._circuit.is_open("aggregator"): + model = models[0] if models else "" + route = routes[0] + self._emit( + AttemptTrace( + request_id=request_id, + scene=Scene.CHARACTER_ACTION, + model=model, + route=route, + attempt_index=start_i, + retry_count=0, + route_reason="skip_circuit_open", + outcome="failed", + circuit_scope="aggregator", + total_latency_ms=total_ms(), + detail=AttemptDetail(input_hash=input_hash), + ) + ) + fail(None) + + for route_index, route in enumerate(routes): + if self._circuit.is_open("base_url:" + route.base_url_id): + if route_index + 1 < len(routes): + fallback_used = True + route_reason_override = "base_url_unreached" + continue + fail(last_http_status) + if self._circuit.is_open(key_circuit_id(route)): + if route_index + 1 < len(routes): + fallback_used = True + route_reason_override = "key_rate_limit" + continue + fail(last_http_status) + + adapter = self._adapter_for(route) + switch_to_next_route = False + for i, model in enumerate(models): + attempt_index = start_i + i + if self._circuit.is_open("model:" + model): + fallback_used = True + fallback_reason = "skip" + self._emit( + AttemptTrace( + request_id=request_id, + scene=Scene.CHARACTER_ACTION, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=0, + route_reason="skip_circuit_open", + outcome="failed", + circuit_scope="model", + fallback_used=fallback_used, + total_latency_ms=total_ms(), + detail=AttemptDetail(input_hash=input_hash), + ) + ) + continue + + if i == 0: + route_reason = route_reason_override or ( + "start_from_caller" + if ctx.start_from_model and ctx.start_from_model in chain + else "primary" + ) + elif fallback_reason == "429": + route_reason = "fallback_after_429" + elif fallback_reason == "skip": + route_reason = "skip_circuit_open" + else: + route_reason = "fallback_after_upstream_fail" + + retry_count = 0 + resend_spent = 0 + bound_job_id: str | None = None + while True: + attempt_t0 = time.monotonic() + started_at = _utc_now() + submit_ms: int | None = None + if bound_job_id is None: + submit_t0 = time.monotonic() + result = adapter.submit_video( + first_frame, prompt, seconds, size, model + ) + submit_ms = int((time.monotonic() - submit_t0) * 1000) + if result.ok and result.job_id: + bound_job_id = result.job_id + result = adapter.follow_job(bound_job_id) + elif result.ok: + result = replace( + result, + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + body=b"", + ) + else: + result = adapter.follow_job(bound_job_id) + + ended_at = _utc_now() + attempt_latency_ms = int((time.monotonic() - attempt_t0) * 1000) + last_http_status = result.http_status + if result.ok: + self._emit_result( + request_id=request_id, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + result=result, + input_hash=input_hash, + total_latency_ms=total_ms(), + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + resend_spent=resend_spent, + seconds=seconds, + outcome="fallback_success" if fallback_used else "success", + submit_ms=submit_ms, + ) + return result.body + + error_type = result.error_type or ModelErrorType.UNKNOWN + last_error = error_type + has_job_id = bool(result.job_id or bound_job_id) + step = decide( + error_type=error_type, + retry_count=retry_count, + has_job_id=has_job_id, + ) + circuit_scope = None + has_next_route = route_index + 1 < len(routes) + if step is NextStep.OPEN_AGGREGATOR: + if has_next_route: + self._circuit.open("base_url:" + route.base_url_id) + circuit_scope = "base_url" + else: + self._circuit.open("aggregator") + circuit_scope = "aggregator" + elif step is NextStep.FALLBACK_KEY: + self._circuit.open(key_circuit_id(route)) + circuit_scope = "key" + elif step is NextStep.FALLBACK: + self._circuit.open("model:" + model) + circuit_scope = "model" + + self._emit_result( + request_id=request_id, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + result=result, + input_hash=input_hash, + total_latency_ms=total_ms(), + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + resend_spent=resend_spent, + seconds=seconds, + outcome="failed", + circuit_scope=circuit_scope, + error_type=error_type, + submit_ms=submit_ms, + ) + if ( + step is NextStep.OPEN_AGGREGATOR + and has_next_route + and bound_job_id is None + ): + fallback_used = True + route_reason_override = "base_url_unreached" + switch_to_next_route = True + break + if step is NextStep.FALLBACK_KEY: + if bound_job_id is not None: + fail(last_http_status) + if has_next_route: + fallback_used = True + route_reason_override = "key_rate_limit" + switch_to_next_route = True + break + fail(last_http_status) + if step is NextStep.RETRY_SAME: + if error_type is ModelErrorType.RATE_LIMIT: + wait = ( + result.retry_after_s + if result.retry_after_s is not None + else _DEFAULT_RETRY_AFTER_S + ) + time.sleep(min(wait, _SLEEP_CAP_S)) + retry_count += 1 + if error_type is ModelErrorType.UNREACHED: + resend_spent = 1 + continue + if step is NextStep.FALLBACK: + if ( + bound_job_id is not None + and error_type is not ModelErrorType.UPSTREAM_FAILED + ): + fail(last_http_status) + if ( + bound_job_id is None + and error_type is not ModelErrorType.RATE_LIMIT + ): + fail(last_http_status) + fallback_used = True + fallback_reason = ( + "429" if error_type is ModelErrorType.RATE_LIMIT else "upstream" + ) + bound_job_id = None + break + fail(last_http_status) + if switch_to_next_route: + break + if switch_to_next_route: + continue + route_reason_override = None + + fail(last_http_status) + + def _emit_result( + self, + *, + request_id: str, + model: str, + route: GatewayRoute, + attempt_index: int, + retry_count: int, + route_reason: str, + result, + input_hash: str, + total_latency_ms: int, + fallback_used: bool, + started_at: str, + ended_at: str, + attempt_latency_ms: int, + resend_spent: int, + seconds: int, + outcome: str, + circuit_scope: str | None = None, + error_type: ModelErrorType | None = None, + submit_ms: int | None = None, + ) -> None: + billed = result.ok or result.maybe_billed + cost = estimate_cost( + Scene.CHARACTER_ACTION, + billed=billed, + seconds=seconds, + image_unit_cost=self._settings.image_unit_cost, + video_unit_cost_per_second=self._settings.video_unit_cost_per_second, + ) + retry_after_ms = ( + int(result.retry_after_s * 1000) + if result.retry_after_s is not None + else None + ) + self._emit( + AttemptTrace( + request_id=request_id, + scene=Scene.CHARACTER_ACTION, + model=model, + route=route, + attempt_index=attempt_index, + retry_count=retry_count, + route_reason=route_reason, + outcome=outcome, + circuit_scope=circuit_scope, + error_type=error_type.value if error_type is not None else None, + http_status=result.http_status, + job_id=result.job_id, + fallback_used=fallback_used, + started_at=started_at, + ended_at=ended_at, + attempt_latency_ms=attempt_latency_ms, + total_latency_ms=total_latency_ms, + maybe_billed=True if result.ok else result.maybe_billed, + cost=cost, + detail=AttemptDetail( + input_hash=input_hash, + output_hash=hash_bytes(result.body) if result.ok else None, + output_bytes=len(result.body) if result.ok else (result.output_bytes or None), + expected_bytes=result.expected_bytes, + retry_after_ms=retry_after_ms, + submit_ms=submit_ms, + poll_ms=result.poll_ms, + download_ms=result.download_ms, + poll_count=result.poll_count, + resend_spent=resend_spent, + job_status=result.job_status, + edge_fingerprint=result.edge_fingerprint or None, + provider_usage=result.provider_usage, + ), + ) + ) + + def _emit(self, trace: AttemptTrace) -> None: + if not trace.family and trace.model: + try: + trace.family = self._registry.family_of(trace.model).value + except RegistryError: + pass + if not trace.started_at: + trace.started_at = _utc_now() + if not trace.ended_at: + trace.ended_at = _utc_now() + if trace.price_version is None: + trace.price_version = self._settings.price_version + emit(trace) + + +def build_video_gateway(config=None, *, adapter=None, circuit=None) -> VideoGateway: + cfg: AIProviderSettings = config or default_settings + route_adapters = None + if adapter is None: + from windup_framework.providers.sufy import SufyVideoProvider + + routes = routes_from_settings(cfg, route_group=Scene.CHARACTER_ACTION.value) + route_adapters = { + route.route_id: SufyVideoProvider(config=config_for_route(cfg, route)) + for route in routes + } + adapter = route_adapters[routes[0].route_id] + return VideoGateway( + ModelRegistry.from_settings(cfg), + adapter, + circuit if circuit is not None else _CIRCUIT, + cfg, + route_adapters=route_adapters, + ) diff --git a/backend/packages/framework/src/windup_framework/providers/__init__.py b/backend/packages/framework/src/windup_framework/providers/__init__.py index 6408c8cf..e505701c 100644 --- a/backend/packages/framework/src/windup_framework/providers/__init__.py +++ b/backend/packages/framework/src/windup_framework/providers/__init__.py @@ -1,6 +1,7 @@ """按模型能力划分的 AI Provider:官方客户端工厂 + 能力接口 + SUFY 实现。""" from windup_framework.config.provider import AIProviderSettings +from windup_framework.gateway import bind_call_context, build_image_gateway, build_video_gateway from windup_framework.providers.chat import create_chat_model from windup_framework.providers.image import create_image_client from windup_framework.providers.judge import JudgeResponseError, SufyJudgeProvider @@ -30,6 +31,10 @@ # FAL 队列面的 i2v(现役接口形态);首帧要公网 URL,故与 uploader 成对出现 "SufyImageProvider", "OnnxU2NetMatteProvider", + # Gateway 工厂(executor 从 windup_framework.gateway 取;此处再导出方便装配) + "bind_call_context", + "build_image_gateway", + "build_video_gateway", # 判官:出参是结构化读数而不是 bytes,故不在 interfaces 的三个 Protocol 之列 "SufyJudgeProvider", "JudgeResponseError", diff --git a/backend/packages/framework/src/windup_framework/providers/chat.py b/backend/packages/framework/src/windup_framework/providers/chat.py index bd9bc4e3..13669900 100644 --- a/backend/packages/framework/src/windup_framework/providers/chat.py +++ b/backend/packages/framework/src/windup_framework/providers/chat.py @@ -2,34 +2,21 @@ from typing import Any -from langchain_openai import ChatOpenAI - from windup_framework.config.provider import AIProviderSettings, settings +from windup_framework.gateway.chat import ChatGateway, build_chat_gateway def create_chat_model( config: AIProviderSettings = settings, **kwargs: Any, -) -> ChatOpenAI: - """创建 LangChain 官方 ``ChatOpenAI`` 实例。 - - 这里仅统一 Windup 配置到 LangChain 官方客户端的映射，不重新实现 - ``BaseChatModel``、消息转换、工具调用或结构化输出。 +) -> ChatGateway: + """创建带 Gateway 策略的 Chat 模型。 - 空 ``AI_API_KEY`` 或空型号直接拒绝，避免 langchain-openai 1.4 抛 - ``OpenAIError`` 或留下 ``ChatOpenAI(model="")``。 + 协议适配仍由 LangChain 官方 ``ChatOpenAI`` 完成；Gateway 只负责 + route / retry / circuit / trace。 """ - model = (config.chat_model or config.model or "").strip() - api_key = (config.api_key or "").strip() - if not api_key: - raise ValueError("AI_API_KEY 未配置") - if not model: - raise ValueError("AI_CHAT_MODEL / AI_MODEL 未配置") - return ChatOpenAI( - model=model, - api_key=api_key, - base_url=config.normalized_base_url, - timeout=config.timeout, - max_retries=config.max_retries, - **kwargs, - ) + if not config.api_key.strip(): + raise ValueError("AI_API_KEY is required") + if not (config.chat_model or config.model).strip(): + raise ValueError("AI_CHAT_MODEL is required") + return build_chat_gateway(config=config, **kwargs) diff --git a/backend/packages/framework/src/windup_framework/providers/sufy.py b/backend/packages/framework/src/windup_framework/providers/sufy.py index bf1d463b..72aebf01 100644 --- a/backend/packages/framework/src/windup_framework/providers/sufy.py +++ b/backend/packages/framework/src/windup_framework/providers/sufy.py @@ -23,18 +23,23 @@ from __future__ import annotations import base64 -from datetime import datetime, timezone -from email.utils import parsedate_to_datetime import io import json import logging -import math import re import time +from dataclasses import replace import httpx +from windup_common.enums.model import ModelErrorType from windup_framework.config.provider import AIProviderSettings, settings +from windup_framework.gateway.classify import ( + classify_exception, + classify_http, + retry_after_seconds as _retry_after_seconds, +) +from windup_framework.gateway.types import AdapterResult from .interfaces import ImageProvider, VideoProvider @@ -73,6 +78,43 @@ def _first_frame_datauri(frame: bytes, size: str) -> str: return "data:image/jpeg;base64," + base64.b64encode(_fit_first_frame(frame, size)).decode() +def _video_http_error(resp: httpx.Response, *, job_id: str | None = None) -> AdapterResult: + error_type = classify_http(resp.status_code) + retry_after_header = resp.headers.get("Retry-After") + retry_after_s = ( + _retry_after_seconds(retry_after_header) if retry_after_header else None + ) + return AdapterResult( + ok=False, + error_type=error_type, + http_status=resp.status_code, + maybe_billed=job_id is not None or error_type is ModelErrorType.MAYBE_BILLED, + edge_fingerprint=_edge_fingerprint(resp), + retry_after_s=retry_after_s, + job_id=job_id, + ) + + +def _transport_result(exc: BaseException) -> AdapterResult: + """POST 还没拿到状态行:收成 AdapterResult,让 Gateway 按 UNREACHED 决定是否重发。""" + error_type, status, edge = classify_exception(exc) + return AdapterResult( + ok=False, + error_type=error_type, + http_status=status, + maybe_billed=error_type is ModelErrorType.MAYBE_BILLED, + edge_fingerprint=edge, + ) + + +def _poll_get(client: httpx.Client, job_id: str) -> httpx.Response: + """轮询 GET;522/525(及同档未达上游码)该次再试 1 次,不新开单。""" + resp = client.get(f"/videos/{job_id}") + if resp.status_code in (521, 522, 523, 525): + resp = client.get(f"/videos/{job_id}") + return resp + + class SufyVideoProvider(VideoProvider): """kling i2v(默认 v2-5-turbo)。首帧 + 动作 prompt → mp4 bytes。""" @@ -105,26 +147,79 @@ def _client(self) -> httpx.Client: timeout=self._cfg.timeout, ) - def i2v( - self, first_frame: bytes, prompt: str, seconds: int = 5, size: str = "1280x720" - ) -> bytes: + def submit_video( + self, + first_frame: bytes, + prompt: str, + seconds: int, + size: str, + model: str, + ) -> AdapterResult: + """一次 POST 建单。成功: ok=True, job_id, body=b"", maybe_billed=True。""" body: dict = { - "model": self._model, + "model": model, "prompt": prompt, "size": size, "seconds": str(seconds), "mode": self._mode, } - if self._model in _IMAGE_LIST_MODELS: + if model in _IMAGE_LIST_MODELS: b64 = _first_frame_datauri(first_frame, size).split(",", 1)[1] body["image_list"] = [{"image": b64}] else: body["input_reference"] = _first_frame_datauri(first_frame, size) with self._client() as client: - job = client.post("/videos", json=body).raise_for_status().json() - jid = job.get("id") + try: + resp = client.post("/videos", json=body) + except httpx.TransportError as exc: + return _transport_result(exc) + + if 200 <= resp.status_code < 300: + try: + payload = resp.json() + except ValueError: + return AdapterResult( + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + http_status=resp.status_code, + edge_fingerprint="响应不是 JSON", + ) + jid = payload.get("id") + if not jid: + return AdapterResult( + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + http_status=resp.status_code, + edge_fingerprint="响应没有 job id", + ) + return AdapterResult( + ok=True, + job_id=str(jid), + body=b"", + maybe_billed=True, + http_status=resp.status_code, + ) + return _video_http_error(resp) + + def follow_job(self, job_id: str) -> AdapterResult: + """轮询已建单据 + 下载。poll GET 522/525 该次再试 1 次,不新开单。""" + poll_t0 = time.monotonic() + poll_count = 0 + + def with_poll( + result: AdapterResult, *, download_ms: int | None = None + ) -> AdapterResult: + return replace( + result, + poll_ms=int((time.monotonic() - poll_t0) * 1000), + poll_count=poll_count, + download_ms=download_ms, + ) + + with self._client() as client: url = None + last_status: str | None = None # 先短后长,而不是每次都睡满 ``poll_interval``。此前第一次查询也要等满一个 # 间隔:60 秒的间隔下,一段 20 秒就绪的视频要到第 60 秒才被发现,纯白等。 # 退避到上限后与原来一致,所以对慢任务不增加网关压力。 @@ -139,17 +234,104 @@ def i2v( break time.sleep(wait) wait = min(wait * 2, self._poll) - st = client.get(f"/videos/{jid}").raise_for_status().json() - status = st.get("status") - if status == "completed": + resp = _poll_get(client, job_id) + poll_count += 1 + if not (200 <= resp.status_code < 300): + return with_poll(_video_http_error(resp, job_id=job_id)) + try: + st = resp.json() + except ValueError: + return with_poll( + AdapterResult( + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + http_status=resp.status_code, + job_id=job_id, + maybe_billed=True, + edge_fingerprint="轮询响应不是 JSON", + ) + ) + last_status = st.get("status") + if last_status == "completed": vids = (st.get("task_result") or {}).get("videos") or [] url = vids[0].get("url") if vids else None break - if status in ("failed", "cancelled"): - raise RuntimeError(f"i2v 失败: {status} — {st.get('error')}") + if last_status in ("failed", "cancelled"): + return with_poll( + AdapterResult( + ok=False, + error_type=ModelErrorType.UPSTREAM_FAILED, + job_id=job_id, + maybe_billed=True, + job_status=last_status, + edge_fingerprint=str(st.get("error") or ""), + ) + ) + poll_ms = int((time.monotonic() - poll_t0) * 1000) if not url: - raise RuntimeError("i2v 未取得视频 URL(超时或失败)") - return _download(client, url) + return replace( + AdapterResult( + ok=False, + error_type=ModelErrorType.TIMEOUT, + job_id=job_id, + maybe_billed=True, + job_status=last_status or "timeout", + ), + poll_ms=poll_ms, + poll_count=poll_count, + ) + try: + download_t0 = time.monotonic() + body = _download(client, url) + download_ms = int((time.monotonic() - download_t0) * 1000) + except RuntimeError as exc: + return replace( + AdapterResult( + ok=False, + error_type=ModelErrorType.MAYBE_BILLED, + job_id=job_id, + maybe_billed=True, + job_status="completed", + edge_fingerprint=str(exc), + ), + poll_ms=poll_ms, + poll_count=poll_count, + ) + return replace( + AdapterResult( + ok=True, + body=body, + job_id=job_id, + maybe_billed=True, + job_status="completed", + ), + poll_ms=poll_ms, + poll_count=poll_count, + download_ms=download_ms, + ) + + def i2v( + self, first_frame: bytes, prompt: str, seconds: int = 5, size: str = "1280x720" + ) -> bytes: + submitted = self.submit_video(first_frame, prompt, seconds, size, self._model) + if not submitted.ok or not submitted.job_id: + raise RuntimeError( + f"i2v 建单失败(HTTP {submitted.http_status} {submitted.error_type}): " + f"{submitted.edge_fingerprint}" + ) + followed = self.follow_job(submitted.job_id) + if followed.ok: + return followed.body + if followed.error_type is ModelErrorType.TIMEOUT: + raise RuntimeError("i2v 未取得视频 URL(超时或失败)") + if followed.error_type is ModelErrorType.UPSTREAM_FAILED: + raise RuntimeError( + f"i2v 失败: {followed.job_status} — {followed.edge_fingerprint}" + ) + raise RuntimeError( + f"i2v 失败(HTTP {followed.http_status} {followed.error_type}): " + f"{followed.edge_fingerprint}" + ) class IncompleteDownloadError(RuntimeError): @@ -282,34 +464,19 @@ def _download(client: httpx.Client, url: str, tries: int = 3) -> bytes: DEFAULT_IMAGE_MODEL = "gemini-2.5-flash-image" -# "调用成功但没返回有效图"的重试次数。与 _download 的网络重试是两码事:那个治连接断, -# 这个治模型返回了一条不含图的正常响应(实测偶发)。也是为什么下面要判 base64 长度 —— -# 返回里可能带一个几十字节的占位串,当图存下去就是一个打不开的文件。 -_IMAGE_TRIES = 3 +# "调用成功但没返回有效图"的下限。返回里可能带一个几十字节的占位串,当图存下去就是一个打不开的文件。 _MIN_IMAGE_BYTES = 5000 _CONNECT_RETRIES = 3 _MAX_RETRY_WAIT = 30.0 _IMAGE_TIMEOUT_MULTIPLIER = 1.5 -# 429 是被限流拒收、必然没计费,所以按次数放开重试。它与 _IMAGE_TRIES 会叠乘,单次 -# gen_image 的最坏情况因此是:_IMAGE_TRIES × _POST_TRIES = 9 次请求,退避最多睡 -# 6 × _MAX_RETRY_WAIT = 180 秒,加上每次请求自身 timeout × _IMAGE_TIMEOUT_MULTIPLIER。 +# 判官 ``_post`` 自带的 429 / 52x 重试。出图不走这里：Gateway 一次一枪。 _POST_TRIES = 3 # 521 源站拒绝连接、523 源站不可达都止步于 TCP 层;522 按 Cloudflare 自己的定义含两种 # 情形 —— 握手没收到 SYN+ACK,以及连接已建立但源站未及时确认请求,后者请求已经写到源站。 # 所以"重发不会重复计费"是大概率而非保证,重发次数因此要受 _UNREACHED_RESENDS 约束。 -# -# 判据只看码、不看响应头:``AI_BASE_URL`` 后面挂的是哪家网关不可知,靠 ``cf-ray`` + -# ``server: cloudflare`` 认 Cloudflare 会把真实链路上的 52x 全判否(实测网关自报 -# ``server: APISIX``),整条重试等于不存在。 -# -# 520 与 524 不在此列:连接已建立、请求可能正在源站处理中(524 就是"源站 100 秒没答完"), -# 重发一次就是为同一张图付两次钱。 _CLOUDFLARE_UNREACHED_STATUS = frozenset({521, 522, 523}) - -# 一次 gen_image 内允许把 52x 重发几次。只按码判就无法排除"网关转发给上游之后才回 52x", -# 与其赌它不存在,不如把最坏情况封成一个小常数:最多多付两张图,且不随上面两层循环叠乘。 _UNREACHED_RESENDS = 2 _DIAGNOSTIC_HEADERS = ("server", "cf-ray", "via", "x-served-by", "retry-after") @@ -330,32 +497,6 @@ def take(self) -> bool: return True -def _edge_fingerprint(response: httpx.Response) -> str: - """52x 出自链路上哪一跳,只能从这几个头看 —— 不记下来,线上就只剩一个状态码可复盘。""" - seen = {k: response.headers.get(k) for k in _DIAGNOSTIC_HEADERS} - return " ".join(f"{k}={v}" for k, v in seen.items() if v) or "无可辨识的边缘响应头" - - -def _utc_now() -> datetime: - return datetime.now(timezone.utc) - - -def _retry_after_seconds(value: str) -> float | None: - try: - delay = float(value) - except ValueError: - try: - retry_at = parsedate_to_datetime(value) - except (TypeError, ValueError, OverflowError): - return None - if retry_at.tzinfo is None: - retry_at = retry_at.replace(tzinfo=timezone.utc) - delay = (retry_at.astimezone(timezone.utc) - _utc_now()).total_seconds() - if not math.isfinite(delay): - return None - return min(max(delay, 0.0), _MAX_RETRY_WAIT) - - def _retry_exhausted_message(status: int, tries: int, fingerprint: str) -> str: """这条文本常常是线上唯一留下的失败记录,少一样就得靠猜是限流、还是哪一跳断的。""" if status == 429: @@ -369,17 +510,52 @@ def _retry_exhausted_message(status: int, tries: int, fingerprint: str) -> str: ) +def _edge_fingerprint(response: httpx.Response) -> str: + """52x 出自链路上哪一跳,只能从这几个头看 —— 不记下来,线上就只剩一个状态码可复盘。""" + seen = {k: response.headers.get(k) for k in _DIAGNOSTIC_HEADERS} + return " ".join(f"{k}={v}" for k, v in seen.items() if v) or "无可辨识的边缘响应头" + + # 从响应里捞 data URI。模型把图放在 message.content 里,而不同网关的包裹层级不一样 # (有的 content 是字符串、有的是 parts 数组),故对整个响应 JSON 做一次正则, # 不去猜层级 —— 猜错的代价是"调用成功、费用已产生、但我们说没图"。 _DATA_URI = re.compile(r"data:image/[^;]+;base64,([A-Za-z0-9+/=]{100,})") +def _image_result_from_2xx(resp: httpx.Response) -> AdapterResult: + try: + payload = resp.json() + except ValueError: + return AdapterResult( + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + http_status=resp.status_code, + edge_fingerprint="响应不是 JSON", + ) + found = _DATA_URI.search(json.dumps(payload)) + if not found: + return AdapterResult( + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + http_status=resp.status_code, + edge_fingerprint="响应里没有 data URI", + ) + data = base64.b64decode(found.group(1)) + if len(data) < _MIN_IMAGE_BYTES: + return AdapterResult( + ok=False, + error_type=ModelErrorType.INVALID_RESPONSE, + http_status=resp.status_code, + edge_fingerprint=f"图只有 {len(data)} 字节(下限 {_MIN_IMAGE_BYTES})", + ) + return AdapterResult(ok=True, body=data, http_status=resp.status_code) + + class ChatCompletionsFace: - """网关 ``/chat/completions`` 面的共用管道:建 client、发请求、判哪些失败可以重发。 + """网关 ``/chat/completions`` 面的共用管道:建 client、发请求。 - 出图与判官共用一份 —— 同一网关同一把 key,限流与 52x 的语义一样;各写一份的话, - 改一次重试判据要记得改两处,漏掉的那处的代价是重复计费。 + 判官走 ``_post``(自带 429 / 52x 重试);出图走 ``submit_image`` 一次一枪, + 重试由 Gateway 做。client / 指纹 / 超时倍数仍共用,免得两处配成两套。 """ # 出图比一次问答慢得多,所以超时按能力放大;判官用基准超时。 @@ -416,7 +592,6 @@ def _post(self, client: httpx.Client, body: dict, resends: _ResendBudget) -> dic raise RuntimeError(_retry_exhausted_message(code, resends.spent, edge)) retryable = code == 429 or code in _CLOUDFLARE_UNREACHED_STATUS if not retryable: - # 5xx 一律留指纹:要不要人工重发,取决于失败落在链路的哪一跳。 if code >= 500: logger.warning( "图像服务返回 %d,不重发(无法排除请求已到达上游并计费);%s", @@ -427,7 +602,6 @@ def _post(self, client: httpx.Client, body: dict, resends: _ResendBudget) -> dic raise RuntimeError(_retry_exhausted_message(code, _POST_TRIES, edge)) delay = _retry_after_seconds(resp.headers.get("Retry-After", "")) if delay is None: - # 上限同样兜住指数退避:上游挂掉时不该把一个图像任务堵成长时间阻塞。 delay = min(float(2**attempt), _MAX_RETRY_WAIT) logger.warning( "模型服务返回 %d，第 %d/%d 次请求，%.2f 秒后重试;%s", @@ -447,6 +621,48 @@ def _post(self, client: httpx.Client, body: dict, resends: _ResendBudget) -> dic ) return resp.raise_for_status().json() + def submit_image(self, prompt: str, refs: list[bytes], model: str) -> AdapterResult: + """提示词 + 参考图 → 一次 POST → AdapterResult。重试由 Gateway 做。""" + content: list[dict] = [{"type": "text", "text": prompt}] + for raw in refs: + b64 = base64.b64encode(raw).decode() + content.append({ + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{b64}"}, + }) + body = {"model": model, "messages": [{"role": "user", "content": content}]} + with self._client() as client: + try: + resp = client.post(self._cfg.chat_completions_path, json=body) + except httpx.TransportError as exc: + return _transport_result(exc) + + if 200 <= resp.status_code < 300: + return _image_result_from_2xx(resp) + + error_type = classify_http(resp.status_code) + if resp.status_code in (400, 404): + edge = ( + f"网关 {self._cfg.normalized_base_url} 拒绝了模型 {model!r}" + f"(HTTP {resp.status_code})。先确认该网关的目录里有它:" + f"GET {self._cfg.normalized_base_url}/models —— 不同网关目录不同," + f"同一把 key 也是。原始响应:{resp.text[:200]}" + ) + else: + edge = _edge_fingerprint(resp) + retry_after_header = resp.headers.get("Retry-After") + retry_after_s = ( + _retry_after_seconds(retry_after_header) if retry_after_header else None + ) + return AdapterResult( + ok=False, + error_type=error_type, + http_status=resp.status_code, + maybe_billed=error_type is ModelErrorType.MAYBE_BILLED, + edge_fingerprint=edge, + retry_after_s=retry_after_s, + ) + class SufyImageProvider(ChatCompletionsFace, ImageProvider): """文生图 / 图生图 provider(OpenAI 兼容的 ``/chat/completions`` 面)。 @@ -475,28 +691,11 @@ def gen_image(self, prompt: str, refs: list[bytes]) -> bytes: 为什么不返回空 bytes 兜底:上游 ``ImageTaskExecutor`` 会把返回值直接上传对象存储 并写进任务结果,一个 0 字节的"成功"会变成用户看到的一张裂图。 """ - content: list[dict] = [{"type": "text", "text": prompt}] - for raw in refs: - b64 = base64.b64encode(raw).decode() - content.append({ - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{b64}"}, - }) - body = {"model": self._model, "messages": [{"role": "user", "content": content}]} - - last = "" - # 预算建在循环外:同一张图的多次尝试共用一份"可能已计费"的额度。 - resends = _ResendBudget() - with self._client() as client: - for attempt in range(1, _IMAGE_TRIES + 1): - payload = self._post(client, body, resends) - found = _DATA_URI.search(json.dumps(payload)) - if found: - data = base64.b64decode(found.group(1)) - if len(data) >= _MIN_IMAGE_BYTES: - return data - last = f"图只有 {len(data)} 字节(下限 {_MIN_IMAGE_BYTES})" - else: - last = "响应里没有 data URI" - logger.warning("文生图第 %d/%d 次没拿到有效图:%s", attempt, _IMAGE_TRIES, last) - raise RuntimeError(f"文生图 {_IMAGE_TRIES} 次均未取得有效图:{last}") + r = self.submit_image(prompt, refs, self._model) + if r.ok: + return r.body + if r.error_type is ModelErrorType.INVALID_RESPONSE: + raise RuntimeError(f"文生图未取得有效图:{r.edge_fingerprint}") + raise RuntimeError( + f"文生图失败(HTTP {r.http_status} {r.error_type}): {r.edge_fingerprint}" + ) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index c75e3220..0084e651 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -12,6 +12,7 @@ # setdefault 不覆盖已有的环境变量(本地 .env 或 CI secrets 优先生效)。 os.environ.setdefault("JWT_SECRET", "test-secret-key-for-ci-only-32chars") os.environ.setdefault("POSTGRES_PASSWORD", "testpassword123") +os.environ.setdefault("AI_GATEWAY_LEDGER_ENABLED", "false") import pytest from fastapi.testclient import TestClient diff --git a/backend/tests/test_custom_action.py b/backend/tests/test_custom_action.py index b682d291..131414aa 100644 --- a/backend/tests/test_custom_action.py +++ b/backend/tests/test_custom_action.py @@ -228,19 +228,12 @@ def test_illegal_facing_raises_instead_of_falling_back(): def test_only_the_opened_models_are_accepted(): - from windup_app.server.orchestrator.executor import ( - ALLOWED_VIDEO_MODELS, - _resolve_video_model, - ) + from windup_framework.config.provider import AIProviderSettings + from windup_framework.gateway.registry import ModelRegistry + from windup_framework.gateway.types import Scene - # veo3.1 不在表里:它走 Fal 队列协议(Authorization: Key + 公网图 URL),而 - # SufyVideoProvider 走 OpenAI 风格 /videos + Bearer + base64。列进去 = 看起来能选、 - # 点了必然产生一个用不了的付费任务。 - assert set(ALLOWED_VIDEO_MODELS) == {"kling-v2-5-turbo", "kling-v2-6"} - assert "veo3.1" not in ALLOWED_VIDEO_MODELS - for name in ALLOWED_VIDEO_MODELS: - assert _resolve_video_model(name) == name - assert _resolve_video_model(None) is None, "None = 用部署默认值" + r = ModelRegistry.from_settings(AIProviderSettings(video_fallbacks="kling-v2-6")) + assert set(r.chain(Scene.CHARACTER_ACTION)) == {"kling-v2-5-turbo", "kling-v2-6"} def test_unknown_model_fails_at_entry_not_at_the_paid_call(): @@ -252,23 +245,21 @@ def test_unknown_model_fails_at_entry_not_at_the_paid_call(): assert "kling-v2-5-turbo" in str(e.value), "报错要带上可选值,否则调用方无从改" -def test_generator_is_bucketed_by_video_model(): - """按模型分桶,否则第一个请求指定 veo3.1 之后所有请求都沿用它。""" +def test_start_from_model_reuses_one_generator(): from windup_app.server.orchestrator.executor import ActionTaskExecutor ex = ActionTaskExecutor() - a = ex._get_generator("kling-v2-6") - b = ex._get_generator("veo3.1") - assert a is not b, "两个模型拿到了同一个 generator" - assert ex._get_generator("kling-v2-6") is a, "同一模型该复用" + assert ex._get_generator() is ex._get_generator() def test_concurrent_first_requests_build_one_shared_provider_set(monkeypatch): - """并发首请求只装一份共用 provider。 + """并发首请求只装一份共用 Gateway / matte。 执行器是进程级单例、每个请求起一个线程,check-and-insert 不加锁时每个线程都会各装 一套;而每个抠图实例会各自惰性加载一份 ONNX 会话,重复的代价落在内存与加载耗时上。 + 选哪个 kling 是 Gateway 的事,不同 video_model 仍共用同一个 generator。 """ + import windup_framework.gateway as gateway from windup_framework import providers from windup_app.server.orchestrator.executor import ActionTaskExecutor @@ -285,8 +276,8 @@ def _factory(*_args, **_kwargs): return _factory monkeypatch.setattr(providers, "OnnxU2NetMatteProvider", _counting("matte")) - monkeypatch.setattr(providers, "SufyImageProvider", _counting("image")) - monkeypatch.setattr(providers, "SufyVideoProvider", _counting("video")) + monkeypatch.setattr(gateway, "build_image_gateway", _counting("image")) + monkeypatch.setattr(gateway, "build_video_gateway", _counting("video")) ex = ActionTaskExecutor() models = ["kling-v2-5-turbo", "kling-v2-6"] * 3 @@ -295,7 +286,7 @@ def _factory(*_args, **_kwargs): def _ask(i: int) -> None: start.wait(timeout=5) - gen = ex._get_generator(models[i]) + gen = ex._get_generator() with tally: got[i] = gen @@ -307,10 +298,11 @@ def _ask(i: int) -> None: assert not any(t.is_alive() for t in threads), "有线程没跑完,装配路径可能卡在锁上" assert built.count("matte") == 1, f"抠图 provider 装了 {built.count('matte')} 次,该只装一次" - assert built.count("image") == 1, f"图生图 provider 装了 {built.count('image')} 次" - assert built.count("video") == 2, "视频 provider 随模型变,两个模型该各一份" - for i, model in enumerate(models): - assert got[i] is ex._by_model[model], "同一模型的并发请求该拿到同一个 generator" + assert built.count("image") == 1, f"图生图 Gateway 装了 {built.count('image')} 次" + assert built.count("video") == 1, f"视频 Gateway 装了 {built.count('video')} 次,该只装一次" + gens = {got[i] for i in range(len(models))} + assert len(gens) == 1, "不同 video_model 的并发请求该拿到同一个 generator" + assert next(iter(gens)) is ex._get_generator() # ── ⑥ 骨架不得夹带姿态前提(游泳/潜水/飞行都不着地不直立)───────────────────── diff --git a/backend/tests/test_gateway_chat.py b/backend/tests/test_gateway_chat.py new file mode 100644 index 00000000..91cbc470 --- /dev/null +++ b/backend/tests/test_gateway_chat.py @@ -0,0 +1,393 @@ +import asyncio +import json +import logging + +import httpx +import pytest + +from windup_common.enums.model import ModelErrorType +from windup_framework.config.provider import AIProviderSettings +from windup_framework.gateway.chat import ( + ChatAdapterResult, + ChatGateway, + LangChainChatAdapter, +) +from windup_framework.gateway.circuit import CircuitBreaker +from windup_framework.gateway.routes import key_circuit_id, routes_from_settings +from windup_framework.gateway.types import Scene +from windup_framework.providers.chat import create_chat_model + +UNREACHED = ChatAdapterResult(ok=False, error_type=ModelErrorType.UNREACHED, http_status=522) +OK = ChatAdapterResult(ok=True, value="pong") + + +class FakeChatAdapter: + def __init__(self, by_model: dict[str, list[ChatAdapterResult]]): + self.by_model = {k: list(v) for k, v in by_model.items()} + self.calls: list[str] = [] + + def invoke(self, messages, *, model: str, **kwargs): + self.calls.append(model) + q = self.by_model[model] + return q.pop(0) if q else ChatAdapterResult(ok=False, error_type=ModelErrorType.UNKNOWN) + + +def test_chat_gateway_switches_base_url_route_after_unreached(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + primary = FakeChatAdapter({"gpt-4o-mini": [UNREACHED, UNREACHED]}) + backup = FakeChatAdapter({"gpt-4o-mini": [OK]}) + cfg = AIProviderSettings( + model="gpt-4o-mini", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="primary-key", + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="backup-key", + ) + gw = ChatGateway( + adapter=primary, + circuit=CircuitBreaker(), + settings=cfg, + route_adapters={"primary": primary, "backup": backup}, + ) + + assert gw.invoke([{"role": "user", "content": "ping"}]) == "pong" + assert primary.calls == ["gpt-4o-mini", "gpt-4o-mini"] + assert backup.calls == ["gpt-4o-mini"] + + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + assert success, caplog.text + line = success[-1] + assert line["scene"] == "chat" + assert line["family"] == "chat.completions" + assert line["route_reason"] == "base_url_unreached" + assert line["route_layer"] == "base_url" + assert line["base_url_id"] == "backup" + + +def test_chat_gateway_switches_key_after_429(monkeypatch, caplog): + monkeypatch.setattr("windup_framework.gateway.chat.time.sleep", lambda _: None) + caplog.set_level(logging.INFO, logger="windup.gateway") + rate = ChatAdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) + key_a = FakeChatAdapter({"gpt-4o-mini": [rate, rate, rate]}) + key_b = FakeChatAdapter({"gpt-4o-mini": [OK]}) + cfg = AIProviderSettings( + model="gpt-4o-mini", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="key-a", + route_primary_api_keys="key-b", + ) + gw = ChatGateway( + adapter=key_a, + circuit=CircuitBreaker(), + settings=cfg, + route_adapters={"primary.key0": key_a, "primary.key1": key_b}, + ) + + assert gw.invoke([{"role": "user", "content": "ping"}]) == "pong" + assert key_a.calls == ["gpt-4o-mini"] * 3 + assert key_b.calls == ["gpt-4o-mini"] + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + line = success[-1] + assert line["route_reason"] == "key_rate_limit" + assert line["route_layer"] == "key" + + +def test_chat_gateway_ainvoke_returns_adapter_value(): + """``/ai/chat`` 走 ainvoke；没有这个方法时 AttributeError 会被翻成 502。""" + adapter = FakeChatAdapter({"gpt-4o-mini": [OK]}) + gw = ChatGateway( + adapter=adapter, + circuit=CircuitBreaker(), + settings=_primary_cfg(), + route_adapters={"primary": adapter}, + ) + assert asyncio.run(gw.ainvoke([{"role": "user", "content": "ping"}])) == "pong" + assert adapter.calls == ["gpt-4o-mini"] + + +def test_create_chat_model_returns_gateway_without_hand_rolling_protocol(): + cfg = AIProviderSettings(api_key="test-key", model="gpt-4o-mini") + chat = create_chat_model(config=cfg) + + assert hasattr(chat, "invoke") + assert hasattr(chat, "ainvoke") + assert hasattr(chat, "astream") + assert hasattr(chat, "bind_tools") + assert chat.__class__.__name__ == "ChatGateway" + + +def _primary_cfg(**kwargs) -> AIProviderSettings: + base = dict( + chat_model="gpt-4o-mini", + api_key="k", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="primary-key", + ) + base.update(kwargs) + return AIProviderSettings(**base) + + +def test_chat_requires_chat_model(): + gw = ChatGateway( + adapter=FakeChatAdapter({}), + circuit=CircuitBreaker(), + settings=_primary_cfg(chat_model="", model=""), + ) + with pytest.raises(RuntimeError, match="AI_CHAT_MODEL"): + gw.invoke([{"role": "user", "content": "ping"}]) + + +def test_chat_skips_when_aggregator_circuit_is_open(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + adapter = FakeChatAdapter({"gpt-4o-mini": [OK]}) + circuit = CircuitBreaker() + circuit.open("aggregator") + gw = ChatGateway(adapter=adapter, circuit=circuit, settings=_primary_cfg()) + with pytest.raises(RuntimeError, match="chat gateway failed"): + gw.invoke([{"role": "user", "content": "ping"}]) + assert adapter.calls == [] + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + assert records[-1]["route_reason"] == "skip_circuit_open" + assert records[-1]["circuit_scope"] == "aggregator" + + +def test_chat_skips_open_base_url_to_backup(): + primary = FakeChatAdapter({"gpt-4o-mini": [OK]}) + backup = FakeChatAdapter({"gpt-4o-mini": [OK]}) + cfg = _primary_cfg( + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="backup-key", + ) + circuit = CircuitBreaker() + circuit.open("base_url:primary") + gw = ChatGateway( + adapter=primary, + circuit=circuit, + settings=cfg, + route_adapters={"primary": primary, "backup": backup}, + ) + assert gw.invoke([{"role": "user", "content": "ping"}]) == "pong" + assert primary.calls == [] + assert backup.calls == ["gpt-4o-mini"] + + +def test_chat_fails_when_last_base_url_circuit_is_open(): + adapter = FakeChatAdapter({"gpt-4o-mini": [OK]}) + circuit = CircuitBreaker() + circuit.open("base_url:primary") + gw = ChatGateway(adapter=adapter, circuit=circuit, settings=_primary_cfg()) + with pytest.raises(RuntimeError, match="chat gateway failed"): + gw.invoke([{"role": "user", "content": "ping"}]) + assert adapter.calls == [] + + +def test_chat_skips_open_key_circuit_to_next_key(): + key_a = FakeChatAdapter({"gpt-4o-mini": [OK]}) + key_b = FakeChatAdapter({"gpt-4o-mini": [OK]}) + cfg = _primary_cfg(route_primary_api_keys="key-b") + routes = routes_from_settings(cfg, route_group=Scene.CHAT.value) + circuit = CircuitBreaker() + circuit.open(key_circuit_id(routes[0])) + gw = ChatGateway( + adapter=key_a, + circuit=circuit, + settings=cfg, + route_adapters={"primary.key0": key_a, "primary.key1": key_b}, + ) + assert gw.invoke([{"role": "user", "content": "ping"}]) == "pong" + assert key_a.calls == [] + assert key_b.calls == ["gpt-4o-mini"] + + +def test_chat_falls_back_to_next_model_after_invalid_response(): + bad = ChatAdapterResult(ok=False, error_type=ModelErrorType.INVALID_RESPONSE) + adapter = FakeChatAdapter({ + "gpt-4o-mini": [bad, bad, bad], + "gpt-4o-mini-alt": [OK], + }) + gw = ChatGateway( + adapter=adapter, + circuit=CircuitBreaker(), + settings=_primary_cfg(chat_fallbacks="gpt-4o-mini-alt"), + ) + assert gw.invoke([{"role": "user", "content": "ping"}]) == "pong" + assert adapter.calls == ["gpt-4o-mini"] * 3 + ["gpt-4o-mini-alt"] + + +def test_chat_auth_error_fails_without_fallback(): + auth = ChatAdapterResult(ok=False, error_type=ModelErrorType.AUTH, http_status=401) + adapter = FakeChatAdapter({ + "gpt-4o-mini": [auth], + "gpt-4o-mini-alt": [OK], + }) + gw = ChatGateway( + adapter=adapter, + circuit=CircuitBreaker(), + settings=_primary_cfg(chat_fallbacks="gpt-4o-mini-alt"), + ) + with pytest.raises(RuntimeError, match="http_status=401"): + gw.invoke([{"role": "user", "content": "ping"}]) + assert "gpt-4o-mini-alt" not in adapter.calls + + +def test_langchain_adapter_returns_ok(monkeypatch): + class _Ok: + def __init__(self, **kwargs): + pass + + def invoke(self, messages, **kwargs): + return "hi" + + monkeypatch.setattr("windup_framework.gateway.chat.ChatOpenAI", _Ok) + r = LangChainChatAdapter(_primary_cfg()).invoke([], model="gpt-4o-mini") + assert r.ok and r.value == "hi" + + +def test_langchain_adapter_maps_status_code(monkeypatch): + class _Boom: + def __init__(self, **kwargs): + pass + + def invoke(self, messages, **kwargs): + err = Exception("quota") + err.status_code = 429 + raise err + + monkeypatch.setattr("windup_framework.gateway.chat.ChatOpenAI", _Boom) + r = LangChainChatAdapter(_primary_cfg()).invoke([], model="gpt-4o-mini") + assert not r.ok + assert r.error_type is ModelErrorType.RATE_LIMIT + assert r.http_status == 429 + + +def test_langchain_adapter_maps_response_status(monkeypatch): + class _Boom: + def __init__(self, **kwargs): + pass + + def invoke(self, messages, **kwargs): + err = Exception("denied") + err.response = httpx.Response(401, request=httpx.Request("POST", "https://x")) + raise err + + monkeypatch.setattr("windup_framework.gateway.chat.ChatOpenAI", _Boom) + r = LangChainChatAdapter(_primary_cfg()).invoke([], model="gpt-4o-mini") + assert r.error_type is ModelErrorType.AUTH + assert r.http_status == 401 + + +def test_langchain_adapter_maps_connect_and_timeout(monkeypatch): + req = httpx.Request("POST", "https://x") + + class _Connect: + def __init__(self, **kwargs): + pass + + def invoke(self, messages, **kwargs): + raise httpx.ConnectError("down", request=req) + + monkeypatch.setattr("windup_framework.gateway.chat.ChatOpenAI", _Connect) + r = LangChainChatAdapter(_primary_cfg()).invoke([], model="gpt-4o-mini") + assert r.error_type is ModelErrorType.UNREACHED + + class _Disconnect: + def __init__(self, **kwargs): + pass + + def invoke(self, messages, **kwargs): + raise httpx.RemoteProtocolError("Server disconnected without sending a response") + + monkeypatch.setattr("windup_framework.gateway.chat.ChatOpenAI", _Disconnect) + r = LangChainChatAdapter(_primary_cfg()).invoke([], model="gpt-4o-mini") + assert r.error_type is ModelErrorType.UNREACHED + assert r.http_status is None + + class _Timeout: + def __init__(self, **kwargs): + pass + + def invoke(self, messages, **kwargs): + raise httpx.ReadTimeout("slow", request=req) + + monkeypatch.setattr("windup_framework.gateway.chat.ChatOpenAI", _Timeout) + r = LangChainChatAdapter(_primary_cfg()).invoke([], model="gpt-4o-mini") + assert r.error_type is ModelErrorType.TIMEOUT + + +def test_langchain_adapter_unknown_exception_stays_unknown(monkeypatch): + class _Boom: + def __init__(self, **kwargs): + pass + + def invoke(self, messages, **kwargs): + raise ValueError("weird") + + monkeypatch.setattr("windup_framework.gateway.chat.ChatOpenAI", _Boom) + r = LangChainChatAdapter(_primary_cfg()).invoke([], model="gpt-4o-mini") + assert r.error_type is ModelErrorType.UNKNOWN + + +class _StreamAdapter: + def __init__(self, chunks): + self.chunks = list(chunks) + self.tools = None + self.calls: list[str] = [] + + def bind_tools(self, tools): + bound = _StreamAdapter(self.chunks) + bound.tools = tools + return bound + + async def astream(self, messages, *, model: str, **kwargs): + self.calls.append(model) + for chunk in self.chunks: + if isinstance(chunk, ChatAdapterResult): + yield chunk + else: + yield ChatAdapterResult(ok=True, value=chunk) + + +def test_chat_gateway_bind_tools_keeps_model_name(): + adapter = _StreamAdapter(["hi"]) + gw = ChatGateway( + adapter=adapter, + circuit=CircuitBreaker(), + settings=_primary_cfg(), + route_adapters={"primary": adapter}, + ) + bound = gw.bind_tools([{"type": "function", "function": {"name": "lookup"}}]) + assert bound.model_name == "gpt-4o-mini" + assert bound._adapter.tools[0]["function"]["name"] == "lookup" + + +def test_chat_gateway_astream_yields_chunks_and_skips_open_route(): + down = _StreamAdapter([UNREACHED]) + up = _StreamAdapter(["你", "好"]) + cfg = _primary_cfg( + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="backup-key", + ) + circuit = CircuitBreaker() + circuit.open("base_url:primary") + gw = ChatGateway( + adapter=down, + circuit=circuit, + settings=cfg, + route_adapters={"primary": down, "backup": up}, + ) + + async def _collect(): + return [chunk async for chunk in gw.astream([{"role": "user", "content": "ping"}])] + + chunks = asyncio.run(_collect()) + assert chunks == ["你", "好"] + assert down.calls == [] + assert up.calls == ["gpt-4o-mini"] diff --git a/backend/tests/test_gateway_classify.py b/backend/tests/test_gateway_classify.py new file mode 100644 index 00000000..797e73bf --- /dev/null +++ b/backend/tests/test_gateway_classify.py @@ -0,0 +1,51 @@ +from windup_common.enums.model import ModelErrorType +from windup_framework.gateway.classify import classify_http, retry_after_seconds + + +def test_522_is_unreached(): + assert classify_http(522) is ModelErrorType.UNREACHED + assert classify_http(525) is ModelErrorType.UNREACHED + assert classify_http(521) is ModelErrorType.UNREACHED + assert classify_http(523) is ModelErrorType.UNREACHED + + +def test_remote_protocol_error_is_unreached(): + """对端未回任何 HTTP 状态行:与 52x 一样按未达源站,不当已计费。""" + import httpx + + from windup_framework.gateway.classify import classify_exception + + err = httpx.RemoteProtocolError("Server disconnected without sending a response") + error_type, status, edge = classify_exception(err) + assert error_type is ModelErrorType.UNREACHED + assert status is None + assert "disconnected" in edge + + +def test_520_and_524_are_maybe_billed(): + assert classify_http(520) is ModelErrorType.MAYBE_BILLED + assert classify_http(524) is ModelErrorType.MAYBE_BILLED + assert classify_http(500) is ModelErrorType.MAYBE_BILLED + + +def test_429_is_rate_limit(): + assert classify_http(429) is ModelErrorType.RATE_LIMIT + + +def test_401_is_auth(): + assert classify_http(401) is ModelErrorType.AUTH + assert classify_http(403) is ModelErrorType.AUTH + + +def test_unreached_is_retryable_maybe_billed_is_not(): + assert ModelErrorType.UNREACHED.retryable + assert ModelErrorType.RATE_LIMIT.retryable + assert not ModelErrorType.MAYBE_BILLED.retryable + assert not ModelErrorType.UPSTREAM_FAILED.retryable + + +def test_retry_after_seconds_number_and_cap(): + assert retry_after_seconds("2") == 2.0 + assert retry_after_seconds("300") == 30.0 + assert retry_after_seconds("invalid") is None + assert retry_after_seconds("NaN") is None diff --git a/backend/tests/test_gateway_executor.py b/backend/tests/test_gateway_executor.py new file mode 100644 index 00000000..504c4071 --- /dev/null +++ b/backend/tests/test_gateway_executor.py @@ -0,0 +1,154 @@ +"""Executor 经 Gateway 装配,失败时仍绑定 request_id 供日志/trace,界面文案走脱敏出口。""" +from __future__ import annotations + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from windup_framework.db.base import Base +from windup_framework.gateway.context import current_call_context +from windup_app.server.orchestrator.executor import ( + ActionTaskExecutor, + ImageTaskExecutor, + _resolve_video_model, +) +from windup_app.server.orchestrator.model import ( + ActionType, + CharacterActionInput, + CharacterImageInput, + TaskStatus, +) +from windup_app.server.orchestrator.service import AiGenerationService +from windup_app.server.project.model import Project # noqa: F401 — 注册表 +from windup_app.server.quota.model import CreditAccount, CreditTransaction # noqa: F401 — 注册表 + +from conftest import seed_credit_account + + +@pytest.fixture +def session_factory(): + engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine) + with factory() as session: + seed_credit_account(session, 1) + session.commit() + return factory + + +def test_none_video_model_means_deploy_default(): + assert _resolve_video_model(None) is None + + +def test_unknown_model_error_lists_chain_members(): + with pytest.raises(ValueError) as e: + _resolve_video_model("sora-2") + msg = str(e.value) + assert "sora-2" in msg + chain_hint = "kling-v2-5-turbo" + assert chain_hint in msg, "报错要带上链上型号,否则调用方无从改" + + +def test_action_task_failure_includes_request_id(session_factory): + seen: dict[str, str | None] = {} + + class _BoomGen: + def generate(self, *args, **kwargs): + ctx = current_call_context() + seen["request_id"] = ctx.request_id + seen["task_id"] = ctx.task_id + seen["start_from_model"] = ctx.start_from_model + raise RuntimeError("gateway boom") + + service = AiGenerationService() + executor = ActionTaskExecutor( + generator=_BoomGen(), + fetch_master=lambda _input: b"png", + session_factory=session_factory, + ) + action_input = CharacterActionInput( + character_id=1, action_type=ActionType.WALK, num_frames=4, + ) + with session_factory() as s: + task = service.generate_character_action(s, user_id=1, input=action_input) + s.commit() + task_id = task.id + + executor.run_action_task(task_id, action_input) + + with session_factory() as s: + done = service.get_task(s, project_id=1, task_id=task_id) + assert done.status is TaskStatus.FAILED + assert done.error_message + assert "request_id" not in (done.error_message or "") + assert seen["request_id"] == f"act-{task_id}" + assert seen["task_id"] == str(task_id) + assert seen["start_from_model"] is None + + +def test_action_task_binds_start_from_model(session_factory): + seen: dict[str, str | None] = {} + + class _BoomGen: + def generate(self, *args, **kwargs): + seen["start_from_model"] = current_call_context().start_from_model + raise RuntimeError("boom") + + service = AiGenerationService() + executor = ActionTaskExecutor( + generator=_BoomGen(), + fetch_master=lambda _input: b"png", + session_factory=session_factory, + ) + action_input = CharacterActionInput( + character_id=1, action_type=ActionType.WALK, num_frames=4, + video_model="kling-v2-5-turbo", + ) + with session_factory() as s: + task = service.generate_character_action(s, user_id=1, input=action_input) + s.commit() + task_id = task.id + + executor.run_action_task(task_id, action_input) + + with session_factory() as s: + done = service.get_task(s, project_id=1, task_id=task_id) + assert done.status is TaskStatus.FAILED + assert seen["start_from_model"] == "kling-v2-5-turbo" + assert "request_id" not in (done.error_message or "") + + +def test_image_task_failure_includes_request_id(session_factory): + seen: dict[str, str | None] = {} + + class _BoomImage: + def gen_image(self, prompt, refs): + seen["request_id"] = current_call_context().request_id + seen["task_id"] = current_call_context().task_id + raise RuntimeError("image boom") + + service = AiGenerationService() + executor = ImageTaskExecutor( + image=_BoomImage(), + session_factory=session_factory, + ) + image_input = CharacterImageInput(prompt="knight") + with session_factory() as s: + task = service.generate_character_image(s, user_id=1, input=image_input) + s.commit() + task_id = task.id + + executor.run_image_task(task_id, image_input) + + with session_factory() as s: + done = service.get_task(s, project_id=1, task_id=task_id) + assert done.status is TaskStatus.FAILED + assert done.error_message + assert "request_id" not in (done.error_message or "") + assert seen["request_id"] == f"img-{task_id}" + assert seen["task_id"] == str(task_id) diff --git a/backend/tests/test_gateway_image.py b/backend/tests/test_gateway_image.py new file mode 100644 index 00000000..e4448361 --- /dev/null +++ b/backend/tests/test_gateway_image.py @@ -0,0 +1,239 @@ +import json +import logging + +import pytest +from windup_common.enums.model import ModelErrorType +from windup_framework.config.provider import AIProviderSettings +from windup_framework.gateway.circuit import CircuitBreaker +from windup_framework.gateway.image import ImageGateway +from windup_framework.gateway.registry import ModelRegistry +from windup_framework.gateway.types import AdapterResult + +UNREACHED = AdapterResult(ok=False, error_type=ModelErrorType.UNREACHED, http_status=522) +BILLED = AdapterResult(ok=False, error_type=ModelErrorType.MAYBE_BILLED, http_status=520) +PNG = AdapterResult(ok=True, body=b"\x89PNG\r\n" + b"x" * 5000) + + +class FakeImageAdapter: + def __init__(self, by_model: dict[str, list[AdapterResult]]): + self.by_model = {k: list(v) for k, v in by_model.items()} + self.calls: list[str] = [] + + def submit_image(self, prompt, refs, model): + self.calls.append(model) + q = self.by_model[model] + return q.pop(0) if q else AdapterResult(ok=False, error_type=ModelErrorType.UNKNOWN) + + +def _make_gw(adapter, **kw): + circuit = kw.pop("circuit", None) + cfg = AIProviderSettings( + image_model="gemini-2.5-flash-image", + image_fallbacks=kw.pop("image_fallbacks", ""), + **kw, + ) + registry = ModelRegistry.from_settings(cfg) + return ImageGateway(registry, adapter, circuit or CircuitBreaker(), cfg) + + +def test_522_retries_same_model_once_and_does_not_fallback(): + ad = FakeImageAdapter({ + "gemini-2.5-flash-image": [UNREACHED, UNREACHED], + "gemini-2.5-flash-image-alt": [PNG], + }) + gw = _make_gw(ad, image_fallbacks="gemini-2.5-flash-image-alt") + with pytest.raises(RuntimeError, match="522"): + gw.gen_image("p", []) + assert ad.calls == ["gemini-2.5-flash-image", "gemini-2.5-flash-image"] + + +def test_522_switches_base_url_route_before_model_fallback(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + primary = FakeImageAdapter({ + "gemini-2.5-flash-image": [UNREACHED, UNREACHED], + "gemini-2.5-flash-image-alt": [PNG], + }) + backup = FakeImageAdapter({"gemini-2.5-flash-image": [PNG]}) + cfg = AIProviderSettings( + image_model="gemini-2.5-flash-image", + image_fallbacks="gemini-2.5-flash-image-alt", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="primary-key", + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="backup-key", + ) + gw = ImageGateway( + ModelRegistry.from_settings(cfg), + primary, + CircuitBreaker(), + cfg, + route_adapters={"primary": primary, "backup": backup}, + ) + + assert gw.gen_image("p", []).startswith(b"\x89PNG") + assert primary.calls == ["gemini-2.5-flash-image", "gemini-2.5-flash-image"] + assert backup.calls == ["gemini-2.5-flash-image"] + assert "gemini-2.5-flash-image-alt" not in primary.calls + + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + assert success, caplog.text + line = success[-1] + assert line["route_reason"] == "base_url_unreached" + assert line["route_layer"] == "base_url" + assert line["base_url_id"] == "backup" + + +def test_aggregator_circuit_skips_fallback_model(): + ad = FakeImageAdapter({ + "gemini-2.5-flash-image": [UNREACHED, UNREACHED], + "gemini-2.5-flash-image-alt": [PNG], + }) + br = CircuitBreaker(cooldown_s=60) + gw = _make_gw(ad, image_fallbacks="gemini-2.5-flash-image-alt", circuit=br) + with pytest.raises(RuntimeError): + gw.gen_image("p", []) + assert "gemini-2.5-flash-image-alt" not in ad.calls + assert br.is_open("aggregator") + + +def test_429_does_not_switch_model_when_only_one_key(monkeypatch): + monkeypatch.setattr("windup_framework.gateway.image.time.sleep", lambda _: None) + rate = AdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) + ad = FakeImageAdapter({ + "gemini-2.5-flash-image": [rate, rate, rate], + "gemini-2.5-flash-image-alt": [PNG], + }) + gw = _make_gw(ad, image_fallbacks="gemini-2.5-flash-image-alt") + with pytest.raises(RuntimeError, match="429"): + gw.gen_image("p", []) + assert ad.calls == ["gemini-2.5-flash-image"] * 3 + assert "gemini-2.5-flash-image-alt" not in ad.calls + + +def test_429_switches_key_on_same_base_url_before_model(monkeypatch, caplog): + monkeypatch.setattr("windup_framework.gateway.image.time.sleep", lambda _: None) + caplog.set_level(logging.INFO, logger="windup.gateway") + rate = AdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) + key_a = FakeImageAdapter({ + "gemini-2.5-flash-image": [rate, rate, rate], + "gemini-2.5-flash-image-alt": [PNG], + }) + key_b = FakeImageAdapter({"gemini-2.5-flash-image": [PNG]}) + cfg = AIProviderSettings( + image_model="gemini-2.5-flash-image", + image_fallbacks="gemini-2.5-flash-image-alt", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="key-a", + route_primary_api_keys="key-b", + ) + gw = ImageGateway( + ModelRegistry.from_settings(cfg), + key_a, + CircuitBreaker(), + cfg, + route_adapters={"primary.key0": key_a, "primary.key1": key_b}, + ) + + assert gw.gen_image("p", []).startswith(b"\x89PNG") + assert key_a.calls == ["gemini-2.5-flash-image"] * 3 + assert key_b.calls == ["gemini-2.5-flash-image"] + assert "gemini-2.5-flash-image-alt" not in key_a.calls + + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + assert success, caplog.text + line = success[-1] + assert line["route_reason"] == "key_rate_limit" + assert line["route_layer"] == "key" + assert line["base_url_id"] == "primary" + assert line["api_key_id"].endswith("key1") + + +def test_522_skips_remaining_keys_on_same_url(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + key_a = FakeImageAdapter({ + "gemini-2.5-flash-image": [UNREACHED, UNREACHED], + "gemini-2.5-flash-image-alt": [PNG], + }) + key_b = FakeImageAdapter({"gemini-2.5-flash-image": [PNG]}) + backup = FakeImageAdapter({"gemini-2.5-flash-image": [PNG]}) + cfg = AIProviderSettings( + image_model="gemini-2.5-flash-image", + image_fallbacks="gemini-2.5-flash-image-alt", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="key-a", + route_primary_api_keys="key-b", + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="key-c", + ) + gw = ImageGateway( + ModelRegistry.from_settings(cfg), + key_a, + CircuitBreaker(), + cfg, + route_adapters={"primary.key0": key_a, "primary.key1": key_b, "backup.key0": backup}, + ) + + assert gw.gen_image("p", []).startswith(b"\x89PNG") + assert key_a.calls == ["gemini-2.5-flash-image", "gemini-2.5-flash-image"] + assert key_b.calls == [] + assert backup.calls == ["gemini-2.5-flash-image"] + + +def test_520_does_not_retry(): + ad = FakeImageAdapter({"gemini-2.5-flash-image": [BILLED, PNG]}) + gw = _make_gw(ad, image_fallbacks="") + with pytest.raises(RuntimeError, match="520"): + gw.gen_image("p", []) + assert ad.calls == ["gemini-2.5-flash-image"] + + +def test_empty_image_then_fallback(): + empty = AdapterResult(ok=False, error_type=ModelErrorType.INVALID_RESPONSE) + ad = FakeImageAdapter({ + "gemini-2.5-flash-image": [empty, empty, empty], + "gemini-2.5-flash-image-alt": [PNG], + }) + gw = _make_gw(ad, image_fallbacks="gemini-2.5-flash-image-alt") + gw.gen_image("p", []) + assert ad.calls.count("gemini-2.5-flash-image") == 3 + assert ad.calls[-1] == "gemini-2.5-flash-image-alt" + + +def test_success_trace_has_latency_and_null_cost_by_default(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + ad = FakeImageAdapter({"gemini-2.5-flash-image": [PNG]}) + gw = _make_gw(ad, image_fallbacks="") + gw.gen_image("p", []) + assert "total_latency_ms" in caplog.text + assert '"cost": null' in caplog.text or '"cost":null' in caplog.text + + +def test_skip_open_model_circuit_sets_fallback_used(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + ad = FakeImageAdapter({ + "gemini-2.5-flash-image": [PNG], + "gemini-2.5-flash-image-alt": [PNG], + }) + br = CircuitBreaker(cooldown_s=60) + br.open("model:gemini-2.5-flash-image") + gw = _make_gw(ad, image_fallbacks="gemini-2.5-flash-image-alt", circuit=br) + assert gw.gen_image("p", []).startswith(b"\x89PNG") + assert ad.calls == ["gemini-2.5-flash-image-alt"] + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + assert success, caplog.text + line = success[-1] + assert line["fallback_used"] is True + assert line["outcome"] == "fallback_success" + assert line["route_reason"] == "skip_circuit_open" + assert not any( + r.get("outcome") == "success" and r.get("fallback_used") is False + for r in records + ) diff --git a/backend/tests/test_gateway_ledger_models.py b/backend/tests/test_gateway_ledger_models.py new file mode 100644 index 00000000..be52e124 --- /dev/null +++ b/backend/tests/test_gateway_ledger_models.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from sqlalchemy import create_engine, inspect + +from windup_framework.db import Base +from windup_framework.gateway.models import AIGatewayAttempt, AIGatewayAttemptDetail + + +def test_gateway_ledger_tables_are_registered_and_split_hot_detail(): + assert AIGatewayAttempt.__tablename__ == "windup_ai_gateway_attempt" + assert AIGatewayAttemptDetail.__tablename__ == "windup_ai_gateway_attempt_detail" + + attempt_cols = set(AIGatewayAttempt.__table__.columns.keys()) + detail_cols = set(AIGatewayAttemptDetail.__table__.columns.keys()) + + # Hot table: compact router/cost fields used by key/url/model health queries. + assert { + "request_id", + "attempt_id", + "route_id", + "route_group", + "candidate_index", + "provider_name", + "base_url_id", + "base_url_host", + "api_key_id", + "model", + "route_layer", + "error_type", + "maybe_billed", + "estimated_cost", + } <= attempt_cols + + # Cold table: larger troubleshooting fields stay out of the hot path. + assert { + "attempt_id", + "edge_fingerprint", + "error_message", + "provider_request_id", + "provider_usage", + "submit_ms", + "poll_ms", + "download_ms", + "extra", + } <= detail_cols + assert "provider_usage" not in attempt_cols + assert "edge_fingerprint" not in attempt_cols + + +def test_gateway_ledger_tables_can_be_created_in_test_db(): + engine = create_engine("sqlite:///:memory:") + try: + Base.metadata.create_all( + engine, + tables=[AIGatewayAttempt.__table__, AIGatewayAttemptDetail.__table__], + ) + tables = set(inspect(engine).get_table_names()) + assert "windup_ai_gateway_attempt" in tables + assert "windup_ai_gateway_attempt_detail" in tables + finally: + engine.dispose() diff --git a/backend/tests/test_gateway_ledger_persistence.py b/backend/tests/test_gateway_ledger_persistence.py new file mode 100644 index 00000000..2f3eab29 --- /dev/null +++ b/backend/tests/test_gateway_ledger_persistence.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import uuid +from datetime import datetime, timezone + +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +from windup_framework.db import Base +from windup_framework.gateway.ledger import persist_attempt +from windup_framework.gateway.models import AIGatewayAttempt, AIGatewayAttemptDetail +from windup_framework.gateway.routes import GatewayRoute +from windup_framework.gateway.trace import AttemptDetail, AttemptTrace +from windup_framework.gateway.types import Scene + + +def test_persist_attempt_splits_hot_and_detail_fields(): + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all( + engine, + tables=[AIGatewayAttempt.__table__, AIGatewayAttemptDetail.__table__], + ) + session_factory = sessionmaker(bind=engine, expire_on_commit=False) + attempt_id = str(uuid.uuid4()) + + persist_attempt( + AttemptTrace( + request_id="gw-1", + attempt_id=attempt_id, + task_id="42", + user_id="7", + scene=Scene.CHARACTER_IMAGE, + model="gemini-2.5-flash-image", + family="image.chat_data_uri", + route=GatewayRoute( + route_id="backup.fallback", + route_group="character_image", + candidate_index=1, + provider_name="openai-compatible", + base_url_id="backup", + base_url="https://backup.example.com/v1", + api_key_id="backup", + api_key="k", + ), + attempt_index=2, + retry_count=1, + route_reason="base_url_unreached", + outcome="fallback_success", + maybe_billed=True, + cost=0.25, + price_version="2026-08-16", + started_at=datetime.now(timezone.utc).isoformat(), + ended_at=datetime.now(timezone.utc).isoformat(), + attempt_latency_ms=123, + detail=AttemptDetail( + edge_fingerprint="cf-ray=abc", + provider_usage={"total_tokens": 12}, + ), + ), + session_factory=session_factory, + ) + + with session_factory() as session: + hot = session.scalar(select(AIGatewayAttempt)) + detail = session.scalar(select(AIGatewayAttemptDetail)) + + assert hot is not None + assert detail is not None + assert hot.request_id == "gw-1" + assert hot.task_id == 42 + assert hot.user_id == 7 + assert hot.base_url_id == "backup" + assert hot.route_layer == "base_url" + assert hot.outcome == "success" + assert str(hot.attempt_id) == attempt_id + assert detail.edge_fingerprint == "cf-ray=abc" + assert detail.provider_usage == {"total_tokens": 12} diff --git a/backend/tests/test_gateway_policy.py b/backend/tests/test_gateway_policy.py new file mode 100644 index 00000000..ddaac166 --- /dev/null +++ b/backend/tests/test_gateway_policy.py @@ -0,0 +1,83 @@ +import threading +import time + +from windup_common.enums.model import ModelErrorType +from windup_framework.gateway.circuit import CircuitBreaker +from windup_framework.gateway.policy import decide +from windup_framework.gateway.types import NextStep + + +def test_522_retries_once_then_opens_aggregator(): + assert decide(error_type=ModelErrorType.UNREACHED, retry_count=0, has_job_id=False) is NextStep.RETRY_SAME + assert decide(error_type=ModelErrorType.UNREACHED, retry_count=1, has_job_id=False) is NextStep.OPEN_AGGREGATOR + + +def test_429_retries_twice_then_fallback_key(): + assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=0, has_job_id=False) is NextStep.RETRY_SAME + assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=1, has_job_id=False) is NextStep.RETRY_SAME + assert decide(error_type=ModelErrorType.RATE_LIMIT, retry_count=2, has_job_id=False) is NextStep.FALLBACK_KEY + + +def test_520_never_retries(): + assert decide(error_type=ModelErrorType.MAYBE_BILLED, retry_count=0, has_job_id=False) is NextStep.FAIL + + +def test_empty_image_retries_then_fallback(): + assert decide(error_type=ModelErrorType.INVALID_RESPONSE, retry_count=0, has_job_id=False) is NextStep.RETRY_SAME + assert decide(error_type=ModelErrorType.INVALID_RESPONSE, retry_count=2, has_job_id=False) is NextStep.FALLBACK + + +def test_job_id_blocks_fallback_on_unreached(): + assert decide(error_type=ModelErrorType.UNREACHED, retry_count=0, has_job_id=True) is NextStep.RETRY_SAME + assert decide(error_type=ModelErrorType.UNREACHED, retry_count=1, has_job_id=True) is NextStep.FAIL + + +def test_upstream_job_failure_fallbacks(): + assert decide(error_type=ModelErrorType.UPSTREAM_FAILED, retry_count=0, has_job_id=True) is NextStep.FALLBACK + + +def test_poll_timeout_fails_without_new_job(): + assert decide(error_type=ModelErrorType.TIMEOUT, retry_count=0, has_job_id=True) is NextStep.FAIL + + +def test_circuit_opens_and_cools_down(monkeypatch): + clock = {"t": 0.0} + br = CircuitBreaker(cooldown_s=60, monotonic=lambda: clock["t"]) + assert not br.is_open("aggregator") + br.open("aggregator") + assert br.is_open("aggregator") + clock["t"] = 59.0 + assert br.is_open("aggregator") + clock["t"] = 60.0 + assert not br.is_open("aggregator") + + +def test_circuit_expiry_is_thread_safe(): + clock = {"t": 0.0} + + def now() -> float: + time.sleep(0.002) + return clock["t"] + + br = CircuitBreaker(cooldown_s=60, monotonic=now) + errors: list[BaseException] = [] + + def expire_once(barrier: threading.Barrier) -> None: + try: + barrier.wait() + br.is_open("k") + except BaseException as exc: + errors.append(exc) + + clock["t"] = 0.0 + br.open("k") + clock["t"] = 60.0 + n = 8 + barrier = threading.Barrier(n) + threads = [threading.Thread(target=expire_once, args=(barrier,)) for _ in range(n)] + for t in threads: + t.start() + for t in threads: + t.join() + assert errors == [] + assert not br.is_open("k") diff --git a/backend/tests/test_gateway_registry.py b/backend/tests/test_gateway_registry.py new file mode 100644 index 00000000..a0da0f39 --- /dev/null +++ b/backend/tests/test_gateway_registry.py @@ -0,0 +1,38 @@ +import pytest +from windup_framework.config.provider import AIProviderSettings +from windup_framework.gateway.registry import ModelRegistry, RegistryError +from windup_framework.gateway.types import Family, Scene + +def _cfg(**kw) -> AIProviderSettings: + return AIProviderSettings( + image_model="gemini-2.5-flash-image", + video_model="kling-v2-5-turbo", + **kw, + ) + +def test_default_chains(): + r = ModelRegistry.from_settings(_cfg(video_fallbacks="kling-v2-6")) + assert r.chain(Scene.CHARACTER_IMAGE) == ("gemini-2.5-flash-image",) + assert r.chain(Scene.CHARACTER_ACTION) == ("kling-v2-5-turbo", "kling-v2-6") + assert r.family_of("kling-v2-6") is Family.VIDEO_INPUT_REFERENCE + +def test_rejects_image_list_in_video_chain(): + with pytest.raises(RegistryError, match="family"): + ModelRegistry.from_settings(_cfg(video_fallbacks="kling-video-o1")) + +def test_rejects_unknown_model(): + with pytest.raises(RegistryError, match="未登记"): + ModelRegistry.from_settings(_cfg(image_fallbacks="not-a-real-model")) + +def test_empty_fallbacks_ok(): + r = ModelRegistry.from_settings(_cfg(image_fallbacks="", video_fallbacks="")) + assert r.chain(Scene.CHARACTER_ACTION) == ("kling-v2-5-turbo",) + + +def test_image_alt_can_be_fallback(): + r = ModelRegistry.from_settings(_cfg(image_fallbacks="gemini-2.5-flash-image-alt")) + assert r.chain(Scene.CHARACTER_IMAGE) == ( + "gemini-2.5-flash-image", + "gemini-2.5-flash-image-alt", + ) + assert r.family_of("gemini-2.5-flash-image-alt") is Family.IMAGE_CHAT_DATA_URI diff --git a/backend/tests/test_gateway_route_config.py b/backend/tests/test_gateway_route_config.py new file mode 100644 index 00000000..e4117335 --- /dev/null +++ b/backend/tests/test_gateway_route_config.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from windup_framework.config.provider import AIProviderSettings +from windup_framework.gateway.routes import routes_from_settings + + +def test_gateway_route_env_fields_are_live(): + cfg = AIProviderSettings( + route_primary_name="qnaigc", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="primary-key", + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="backup-key", + ) + + assert cfg.route_primary_name == "qnaigc" + assert cfg.route_primary_base_url == "https://api.qnaigc.com/v1" + assert cfg.route_primary_api_key == "primary-key" + assert cfg.route_fallback_name == "backup" + assert cfg.route_fallback_base_url == "https://backup.example.com/v1" + assert cfg.route_fallback_api_key == "backup-key" + + +def test_empty_gateway_route_values_disable_fallback_route(): + cfg = AIProviderSettings( + base_url="https://api.qnaigc.com/v1/", + api_key="legacy-key", + route_primary_base_url="", + route_primary_api_key="", + route_fallback_name="", + route_fallback_base_url="", + route_fallback_api_key="", + ) + + assert cfg.effective_route_primary_base_url == "https://api.qnaigc.com/v1" + assert cfg.effective_route_primary_api_key == "legacy-key" + assert cfg.route_fallback_enabled is False + + +def test_routes_expand_extra_keys_on_same_base_url_before_fallback_url(): + cfg = AIProviderSettings( + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="key-a", + route_primary_api_keys="key-b", + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="key-c", + ) + routes = routes_from_settings(cfg, route_group="character_image") + + assert [(r.base_url_id, r.api_key, r.base_url) for r in routes] == [ + ("primary", "key-a", "https://api.qnaigc.com/v1"), + ("primary", "key-b", "https://api.qnaigc.com/v1"), + ("backup", "key-c", "https://backup.example.com/v1"), + ] + assert routes[0].api_key_id != routes[1].api_key_id + assert routes[0].candidate_index == 0 + assert routes[1].candidate_index == 1 + assert routes[2].candidate_index == 2 diff --git a/backend/tests/test_gateway_trace.py b/backend/tests/test_gateway_trace.py new file mode 100644 index 00000000..ed61d484 --- /dev/null +++ b/backend/tests/test_gateway_trace.py @@ -0,0 +1,141 @@ +import json +import logging +from dataclasses import fields + +from windup_framework.gateway.context import bind_call_context, current_call_context +from windup_framework.gateway.routes import GatewayRoute +from windup_framework.gateway.trace import AttemptDetail, AttemptTrace, emit, estimate_cost +from windup_framework.gateway.types import Scene + +REQUIRED = { + "request_id", "attempt_id", "task_id", "user_id", "scene", "model", "family", + "route_id", "route_group", "candidate_index", "provider_name", "base_url_id", + "base_url_host", "api_key_id", "attempt_index", "retry_count", "route_reason", + "route_layer", "circuit_scope", + "error_type", "http_status", "edge_fingerprint", "job_id", "fallback_used", + "outcome", "job_status", "started_at", "ended_at", "attempt_latency_ms", + "total_latency_ms", "submit_ms", "poll_ms", "download_ms", "poll_count", + "retry_after_ms", "resend_spent", "output_bytes", "expected_bytes", + "input_hash", "output_hash", "maybe_billed", "cost", "price_version", + "provider_usage", +} + +COLD_FIELDS = { + "input_hash", "output_hash", "output_bytes", "expected_bytes", + "retry_after_ms", "submit_ms", "poll_ms", "download_ms", "poll_count", + "resend_spent", "job_status", "edge_fingerprint", "provider_usage", +} + + +def _route(**overrides) -> GatewayRoute: + fields_ = dict( + route_id="primary.key0", + route_group="character_image", + candidate_index=0, + provider_name="openai-compatible", + base_url_id="primary", + base_url="https://api.qnaigc.com/v1", + api_key_id="primary.key0", + api_key="k", + ) + fields_.update(overrides) + return GatewayRoute(**fields_) + + +def _trace(**overrides) -> AttemptTrace: + fields_ = dict( + request_id="r1", + scene=Scene.CHARACTER_IMAGE, + model="gemini-2.5-flash-image", + route=_route(), + attempt_index=0, + retry_count=0, + route_reason="primary", + outcome="success", + ) + fields_.update(overrides) + return AttemptTrace(**fields_) + + +def test_cold_fields_live_on_detail_not_trace(): + names = {f.name for f in fields(AttemptTrace)} + assert "route" in names + assert "detail" in names + assert COLD_FIELDS.isdisjoint(names) + assert "route_id" not in names + assert "route_layer" not in names + + +def test_trace_as_dict_has_required_keys(): + t = _trace() + keys = set(t.as_dict()) + missing = REQUIRED - keys + assert not missing, missing + assert "route" not in keys + assert "detail" not in keys + + +def test_as_dict_flattens_route_and_detail(): + t = _trace( + route=_route(base_url_id="backup", route_id="backup.key0"), + route_reason="base_url_unreached", + detail=AttemptDetail(input_hash="abc", submit_ms=12, provider_usage={"n": 1}), + ) + d = t.as_dict() + assert d["base_url_id"] == "backup" + assert d["route_id"] == "backup.key0" + assert d["route_layer"] == "base_url" + assert d["input_hash"] == "abc" + assert d["submit_ms"] == 12 + assert d["provider_usage"] == {"n": 1} + + +def test_cost_null_when_unpriced(): + assert estimate_cost(Scene.CHARACTER_IMAGE, billed=True, seconds=5, + image_unit_cost=None, video_unit_cost_per_second=None) is None + assert estimate_cost(Scene.CHARACTER_IMAGE, billed=True, seconds=5, + image_unit_cost=0.02, video_unit_cost_per_second=None) == 0.02 + assert estimate_cost(Scene.CHARACTER_IMAGE, billed=False, seconds=5, + image_unit_cost=0.02, video_unit_cost_per_second=None) is None + assert estimate_cost(Scene.CHARACTER_ACTION, billed=True, seconds=5, + image_unit_cost=None, video_unit_cost_per_second=0.1) == 0.5 + + +def test_cost_never_emits_zero_for_missing_price(): + d = _trace(model="x", cost=None).as_dict() + assert d["cost"] is None + + +def test_context_bind_and_reset(): + assert current_call_context().request_id is None + tok = bind_call_context(request_id="abc", task_id="1", user_id="9", start_from_model="kling-v2-6") + try: + assert current_call_context().request_id == "abc" + assert current_call_context().start_from_model == "kling-v2-6" + finally: + tok() + assert current_call_context().request_id is None + + +def test_emit_fills_ids_from_context(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + tok = bind_call_context(request_id="abc", task_id="1", user_id="9") + try: + emit(_trace(request_id="r1", family="image.chat_data_uri")) + finally: + tok() + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + assert records + line = records[-1] + assert line["request_id"] == "r1" + assert line["task_id"] == "1" + assert line["user_id"] == "9" + assert line["attempt_id"] + assert line["price_version"] + assert line["family"] == "image.chat_data_uri" + + +def test_emit_logs_json_fields(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + emit(_trace(request_id="r1", model="m")) + assert "r1" in caplog.text diff --git a/backend/tests/test_gateway_video.py b/backend/tests/test_gateway_video.py new file mode 100644 index 00000000..a3885be4 --- /dev/null +++ b/backend/tests/test_gateway_video.py @@ -0,0 +1,274 @@ +import json +import logging + +import pytest +from windup_common.enums.model import ModelErrorType +from windup_framework.config.provider import AIProviderSettings +from windup_framework.gateway.circuit import CircuitBreaker +from windup_framework.gateway.registry import ModelRegistry +from windup_framework.gateway.types import AdapterResult +from windup_framework.gateway.video import VideoGateway + +UNREACHED = AdapterResult(ok=False, error_type=ModelErrorType.UNREACHED, http_status=522) +FAILED_JOB = AdapterResult( + ok=False, error_type=ModelErrorType.UPSTREAM_FAILED, job_id="j1", maybe_billed=True, +) +TIMEOUT = AdapterResult(ok=False, error_type=ModelErrorType.TIMEOUT, job_id="j1", maybe_billed=True) +MP4 = AdapterResult(ok=True, body=b"\x00\x00\x00\x18ftypmp42", maybe_billed=True) + +class FakeVideoAdapter: + def __init__(self, submits: dict[str, list[AdapterResult]], follows: dict[str, AdapterResult]): + self.submits = {k: list(v) for k, v in submits.items()} + self.follows = dict(follows) + self.submit_models: list[str] = [] + self.followed: list[str] = [] + + def submit_video(self, first_frame, prompt, seconds, size, model): + self.submit_models.append(model) + return self.submits[model].pop(0) + + def follow_job(self, job_id): + self.followed.append(job_id) + return self.follows[job_id] + +def _video_gw(adapter, circuit=None) -> VideoGateway: + cfg = AIProviderSettings(video_model="kling-v2-5-turbo", video_fallbacks="kling-v2-6") + return VideoGateway( + registry=ModelRegistry.from_settings(cfg), + adapter=adapter, + circuit=circuit or CircuitBreaker(cooldown_s=60), + settings=cfg, + ) + +def test_submit_522_retries_once_does_not_open_second_job_on_fallback_model(): + ad = FakeVideoAdapter( + submits={"kling-v2-5-turbo": [UNREACHED, UNREACHED], "kling-v2-6": [ + AdapterResult(ok=True, job_id="j-alt", maybe_billed=True) + ]}, + follows={}, + ) + with pytest.raises(RuntimeError, match="522"): + _video_gw(ad).i2v(b"frame", "walk") + assert ad.submit_models == ["kling-v2-5-turbo", "kling-v2-5-turbo"] + assert ad.followed == [] + + +def test_submit_522_switches_base_url_route_before_model_fallback(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + primary = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [UNREACHED, UNREACHED], + "kling-v2-6": [AdapterResult(ok=True, job_id="wrong", maybe_billed=True)], + }, + follows={}, + ) + backup = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j-backup", maybe_billed=True)], + "kling-v2-6": [], + }, + follows={"j-backup": MP4}, + ) + cfg = AIProviderSettings( + video_model="kling-v2-5-turbo", + video_fallbacks="kling-v2-6", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="primary-key", + route_fallback_name="backup", + route_fallback_base_url="https://backup.example.com/v1", + route_fallback_api_key="backup-key", + ) + gw = VideoGateway( + registry=ModelRegistry.from_settings(cfg), + adapter=primary, + circuit=CircuitBreaker(cooldown_s=60), + settings=cfg, + route_adapters={"primary": primary, "backup": backup}, + ) + + assert gw.i2v(b"frame", "walk").startswith(b"\x00\x00\x00\x18ftyp") + assert primary.submit_models == ["kling-v2-5-turbo", "kling-v2-5-turbo"] + assert backup.submit_models == ["kling-v2-5-turbo"] + assert "kling-v2-6" not in primary.submit_models + + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + assert success, caplog.text + line = success[-1] + assert line["route_reason"] == "base_url_unreached" + assert line["route_layer"] == "base_url" + assert line["base_url_id"] == "backup" + + +def test_submit_429_switches_key_on_same_base_url(monkeypatch): + monkeypatch.setattr("windup_framework.gateway.video.time.sleep", lambda _: None) + rate = AdapterResult(ok=False, error_type=ModelErrorType.RATE_LIMIT, http_status=429) + key_a = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [rate, rate, rate], + "kling-v2-6": [AdapterResult(ok=True, job_id="wrong", maybe_billed=True)], + }, + follows={}, + ) + key_b = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j-key-b", maybe_billed=True)], + "kling-v2-6": [], + }, + follows={"j-key-b": MP4}, + ) + cfg = AIProviderSettings( + video_model="kling-v2-5-turbo", + video_fallbacks="kling-v2-6", + route_primary_name="primary", + route_primary_base_url="https://api.qnaigc.com/v1", + route_primary_api_key="key-a", + route_primary_api_keys="key-b", + ) + gw = VideoGateway( + registry=ModelRegistry.from_settings(cfg), + adapter=key_a, + circuit=CircuitBreaker(cooldown_s=60), + settings=cfg, + route_adapters={"primary.key0": key_a, "primary.key1": key_b}, + ) + + assert gw.i2v(b"frame", "walk").startswith(b"\x00\x00\x00\x18ftyp") + assert key_a.submit_models == ["kling-v2-5-turbo"] * 3 + assert key_b.submit_models == ["kling-v2-5-turbo"] + assert "kling-v2-6" not in key_a.submit_models + + +def test_follow_failed_opens_new_job_on_fallback(): + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j1", maybe_billed=True)], + "kling-v2-6": [AdapterResult(ok=True, job_id="j2", maybe_billed=True)], + }, + follows={"j1": FAILED_JOB, "j2": MP4}, + ) + body = _video_gw(ad).i2v(b"frame", "walk") + assert body.startswith(b"\x00\x00\x00\x18ftyp") + assert ad.submit_models == ["kling-v2-5-turbo", "kling-v2-6"] + assert ad.followed == ["j1", "j2"] + +def test_timeout_does_not_submit_fallback(): + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j1", maybe_billed=True)], + "kling-v2-6": [AdapterResult(ok=True, job_id="j2", maybe_billed=True)], + }, + follows={"j1": TIMEOUT}, + ) + with pytest.raises(RuntimeError, match="timeout|超时"): + _video_gw(ad).i2v(b"frame", "walk") + assert ad.submit_models == ["kling-v2-5-turbo"] + assert ad.followed == ["j1"] + + +@pytest.mark.parametrize( + "error_type", + [ModelErrorType.RATE_LIMIT, ModelErrorType.INVALID_RESPONSE], +) +def test_follow_fallback_without_upstream_fail_does_not_open_second_job(error_type): + follow_result = AdapterResult( + ok=False, + error_type=error_type, + job_id="j1", + retry_after_s=0, + maybe_billed=True, + ) + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j1", maybe_billed=True)], + "kling-v2-6": [AdapterResult(ok=True, job_id="j2", maybe_billed=True)], + }, + follows={"j1": follow_result}, + ) + with pytest.raises(RuntimeError, match=error_type.value): + _video_gw(ad).i2v(b"frame", "walk") + assert ad.submit_models == ["kling-v2-5-turbo"] + assert ad.followed == ["j1", "j1", "j1"] + + +def test_success_trace_has_phase_timings(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + timed = AdapterResult( + ok=True, + body=b"\x00\x00\x00\x18ftypmp42", + maybe_billed=True, + job_id="j1", + poll_count=2, + poll_ms=1500, + download_ms=80, + ) + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j1", maybe_billed=True)], + "kling-v2-6": [], + }, + follows={"j1": timed}, + ) + _video_gw(ad).i2v(b"frame", "walk") + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") == "success"] + assert success, caplog.text + line = success[-1] + assert line["poll_count"] == 2 + assert line["poll_ms"] == 1500 + assert line["download_ms"] == 80 + assert line["submit_ms"] is not None + + +def test_skip_open_model_circuit_sets_fallback_used(caplog): + caplog.set_level(logging.INFO, logger="windup.gateway") + br = CircuitBreaker(cooldown_s=60) + br.open("model:kling-v2-5-turbo") + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [AdapterResult(ok=True, job_id="j1", maybe_billed=True)], + "kling-v2-6": [AdapterResult(ok=True, job_id="j2", maybe_billed=True)], + }, + follows={"j2": MP4}, + ) + body = _video_gw(ad, circuit=br).i2v(b"frame", "walk") + assert body.startswith(b"\x00\x00\x00\x18ftyp") + assert ad.submit_models == ["kling-v2-6"] + records = [json.loads(r.message) for r in caplog.records if r.name == "windup.gateway"] + success = [r for r in records if r.get("outcome") in ("success", "fallback_success")] + assert success, caplog.text + line = success[-1] + assert line["fallback_used"] is True + assert line["outcome"] == "fallback_success" + assert line["route_reason"] == "skip_circuit_open" + + +def test_submit_invalid_response_does_not_open_fallback_job(): + invalid = AdapterResult(ok=False, error_type=ModelErrorType.INVALID_RESPONSE) + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [invalid, invalid, invalid], + "kling-v2-6": [AdapterResult(ok=True, job_id="j-alt", maybe_billed=True)], + }, + follows={"j-alt": MP4}, + ) + with pytest.raises(RuntimeError, match="invalid_response"): + _video_gw(ad).i2v(b"frame", "walk") + assert "kling-v2-6" not in ad.submit_models + assert ad.submit_models == ["kling-v2-5-turbo"] * 3 + + +def test_submit_ok_without_job_id_is_invalid_response(): + no_id = AdapterResult(ok=True, body=b"") + ad = FakeVideoAdapter( + submits={ + "kling-v2-5-turbo": [no_id, no_id, no_id], + "kling-v2-6": [AdapterResult(ok=True, job_id="j-alt", maybe_billed=True)], + }, + follows={"j-alt": MP4}, + ) + with pytest.raises(RuntimeError, match="invalid_response"): + _video_gw(ad).i2v(b"frame", "walk") + assert "kling-v2-6" not in ad.submit_models + assert ad.submit_models == ["kling-v2-5-turbo"] * 3 diff --git a/backend/tests/test_quality_judge.py b/backend/tests/test_quality_judge.py index 6a070b91..9fec4d14 100644 --- a/backend/tests/test_quality_judge.py +++ b/backend/tests/test_quality_judge.py @@ -79,6 +79,13 @@ def handler(request: httpx.Request) -> httpx.Response: return handler +def _ok_verdict_response() -> httpx.Response: + return httpx.Response(200, json={"choices": [{"message": {"content": json.dumps({ + "subject_count": 1, "foreign_objects": [], + "action_matches": True, "clipped": False, + })}}]}) + + def _provider(monkeypatch, handler) -> SufyJudgeProvider: """把 provider 的 client 换成走 MockTransport 的,保留它自己组的 headers / base_url。""" provider = SufyJudgeProvider(config=_cfg()) @@ -162,6 +169,91 @@ def handler(request: httpx.Request) -> httpx.Response: assert request.headers["Authorization"] == "Bearer test-key" +# ── 判官走 _post 重试;出图不走这条 ───────────────────────────────────────── +# +# 合入 Gateway 时这段曾被揉进 submit_image 后丢掉。下面几条钉住:429/52x 才重发, +# 500 可能已计费所以不重发,404 要说出型号和目录。 + + +def test_judge_retries_429_then_succeeds(monkeypatch): + monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + if calls["n"] == 1: + return httpx.Response(429, headers={"Retry-After": "0"}) + return _ok_verdict_response() + + verdict = _provider(monkeypatch, handler).judge(_png(1), _png(1), "walk") + assert calls["n"] == 2 + assert verdict.subject_count == 1 + + +def test_judge_retries_522_then_succeeds(monkeypatch): + monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + if calls["n"] == 1: + return httpx.Response(522, headers={"server": "cloudflare"}) + return _ok_verdict_response() + + _provider(monkeypatch, handler).judge(_png(1), _png(1), "walk") + assert calls["n"] == 2 + + +def test_judge_does_not_retry_500(monkeypatch): + """500 无法排除请求已到上游,再打一枪就是为同一帧付两次。""" + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response(500, text="upstream") + + with pytest.raises(httpx.HTTPStatusError): + _provider(monkeypatch, handler).judge(_png(1), _png(1), "walk") + assert calls["n"] == 1 + + +def test_judge_404_names_the_model_and_models_catalog(monkeypatch): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(404, text="model not found") + + with pytest.raises(RuntimeError, match="judge-x") as e: + _provider(monkeypatch, handler).judge(_png(1), _png(1), "walk") + assert "/models" in str(e.value) + + +def test_judge_exhausted_429_says_rate_limited(monkeypatch): + monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response(429, text="quota exceeded") + + with pytest.raises(RuntimeError, match="限流"): + _provider(monkeypatch, handler).judge(_png(1), _png(1), "walk") + assert calls["n"] == 3 + + +def test_judge_stops_52x_when_resend_budget_is_spent(monkeypatch): + """52x 重发有重复计费风险,额度花完必须停,不能跟 429 一样打满 _POST_TRIES。""" + monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) + monkeypatch.setattr("windup_framework.providers.sufy._POST_TRIES", 9) + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response(522, headers={"server": "APISIX"}) + + with pytest.raises(RuntimeError, match="未能连上上游"): + _provider(monkeypatch, handler).judge(_png(1), _png(1), "walk") + assert calls["n"] == 3, "额度 2 次 + 发现花完的那一枪,不是把 _POST_TRIES 打满" + + # ── 读不出结论必须抛错,不得兜底成"通过" ──────────────────────────────────── diff --git a/backend/tests/test_sufy_video_download.py b/backend/tests/test_sufy_video_download.py index 2b9da701..bf2e20b5 100644 --- a/backend/tests/test_sufy_video_download.py +++ b/backend/tests/test_sufy_video_download.py @@ -15,12 +15,12 @@ import httpx import pytest +from windup_common.enums.model import ModelErrorType +from windup_framework.gateway.classify import _utc_now, retry_after_seconds from windup_framework.providers.sufy import ( IncompleteDownloadError, UnsafeDownloadUrlError, _download, - _retry_after_seconds, - _utc_now, ) VIDEO = b"\x00\x01mp4-bytes" * 64 @@ -263,6 +263,37 @@ def test_image_provider_extends_request_timeout_by_half(): assert client.timeout.pool == 30 +def test_submit_image_returns_png_on_200(): + r = _image_provider(lambda req: httpx.Response(200, json=_img_payload(_big_b64()))).submit_image("x", [], "gemini-2.5-flash-image") + assert r.ok and r.body.startswith(b"\x89PNG") + + +def test_submit_image_maps_disconnect_to_unreached(): + """对端拆连接且无状态行时不得把异常抛出执行器;Gateway 才能按 UNREACHED 重试。""" + + def h(request): + raise httpx.RemoteProtocolError("Server disconnected without sending a response") + + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert not r.ok + assert r.error_type is ModelErrorType.UNREACHED + assert r.http_status is None + assert r.maybe_billed is False + assert r.job_id is None + assert "disconnected" in r.edge_fingerprint + + +def test_submit_image_sends_the_model_argument(): + seen: dict = {} + + def h(request): + seen["body"] = json.loads(request.content) + return httpx.Response(200, json=_img_payload(_big_b64())) + + _image_provider(h).submit_image("x", [], "gemini-override") + assert seen["body"]["model"] == "gemini-override" + + def test_gen_image_returns_the_decoded_png(): """端点可达而 provider 必抛错 = 每个图像任务稳定 FAILED。实现后必须真能出图。""" def h(request): @@ -291,11 +322,8 @@ def h(request): assert content[1]["image_url"]["url"].startswith("data:image/png;base64,") -def test_response_without_an_image_is_retried_then_raises(): - """模型偶发返回一条不含图的正常响应。重试后仍拿不到必须抛，不能返回空 bytes—— - 上游会把返回值直接上传对象存储并写进任务结果，0 字节的"成功"就是用户看到的裂图。""" - import pytest - +def test_response_without_an_image_is_invalid_response(): + """2xx 但没有图 → INVALID_RESPONSE，一次 POST，不在 adapter 内连打。""" calls = {"n": 0} def h(request): @@ -303,39 +331,46 @@ def h(request): calls["n"] += 1 return httpx.Response(200, json={"choices": [{"message": {"content": "抱歉"}}]}) + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert not r.ok + assert r.error_type is ModelErrorType.INVALID_RESPONSE + with pytest.raises(RuntimeError, match="未取得有效图"): _image_provider(h).gen_image("x", []) - assert calls["n"] == 3, "应重试到上限而不是一次就放弃" def test_undersized_image_is_rejected_not_returned(): """响应里可能带一个几十字节的占位串，当图存下去就是打不开的文件。""" import base64 - import pytest - tiny = base64.b64encode(b"\x89PNG" + b"\x00" * 200).decode() + calls = {"n": 0} def h(request): import httpx + calls["n"] += 1 return httpx.Response(200, json=_img_payload(tiny)) + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert not r.ok + assert r.error_type is ModelErrorType.INVALID_RESPONSE + with pytest.raises(RuntimeError, match="字节"): _image_provider(h).gen_image("x", []) -def test_first_successful_attempt_stops_retrying(): +def test_first_successful_attempt_is_one_post(): calls = {"n": 0} def h(request): import httpx calls["n"] += 1 - if calls["n"] == 1: - return httpx.Response(200, json={"choices": [{"message": {"content": "空"}}]}) return httpx.Response(200, json=_img_payload(_big_b64())) assert _image_provider(h).gen_image("x", []) - assert calls["n"] == 2 + assert calls["n"] == 1 def test_image_client_retries_connection_failures(): @@ -354,27 +389,25 @@ def test_image_client_retries_connection_failures(): client.close() -def test_image_rate_limit_is_retried_after_retry_after(monkeypatch): - """429 表示请求未被网关接收，按 Retry-After 退避后应继续当前图片任务。""" +def test_image_rate_limit_is_retried_after_retry_after(): + """429 → RATE_LIMIT，解析 Retry-After 进 result；adapter 不 sleep（Gateway 才 sleep）。""" calls = {"n": 0} - sleeps: list[float] = [] def h(request): import httpx calls["n"] += 1 - if calls["n"] == 1: - return httpx.Response(429, headers={"Retry-After": "0.25"}) - return httpx.Response(200, json=_img_payload(_big_b64())) - - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", sleeps.append) + return httpx.Response(429, headers={"Retry-After": "0.25"}) - assert _image_provider(h).gen_image("x", []) - assert calls["n"] == 2 - assert sleeps == [0.25] + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.RATE_LIMIT + assert r.retry_after_s == 0.25 + assert not r.ok + assert r.maybe_billed is False -def test_image_rate_limit_exhaustion_has_actionable_error(monkeypatch): - """持续 429 不能泄漏 httpx 异常，也不能无限重试。""" +def test_image_rate_limit_exhaustion_has_actionable_error(): + """持续 429 不能泄漏 httpx 异常；adapter 一次一枪，重试留给 Gateway。""" calls = {"n": 0} def h(request): @@ -382,54 +415,44 @@ def h(request): calls["n"] += 1 return httpx.Response(429, text='{"error":{"message":"quota exceeded"}}') - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) - - with pytest.raises(RuntimeError, match="稍后重试或检查服务商额度"): - _image_provider(h).gen_image("x", []) - assert calls["n"] == 3 + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.RATE_LIMIT + assert not r.ok @pytest.mark.parametrize( - ("retry_after", "expected"), [("invalid", 2.0), ("NaN", 2.0), ("300", 30.0)] + ("retry_after", "expected"), [("invalid", None), ("NaN", None), ("300", 30.0)] ) -def test_image_rate_limit_wait_has_fallback_and_cap(monkeypatch, retry_after, expected): - calls = {"n": 0} - sleeps: list[float] = [] - +def test_image_rate_limit_wait_has_fallback_and_cap(retry_after, expected): def h(request): import httpx - calls["n"] += 1 - if calls["n"] == 1: - return httpx.Response(429, headers={"Retry-After": retry_after}) - return httpx.Response(200, json=_img_payload(_big_b64())) - - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", sleeps.append) + return httpx.Response(429, headers={"Retry-After": retry_after}) - assert _image_provider(h).gen_image("x", []) - assert sleeps == [expected] + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert r.error_type is ModelErrorType.RATE_LIMIT + assert r.retry_after_s == expected def test_image_rate_limit_accepts_http_date(monkeypatch): calls = {"n": 0} - sleeps: list[float] = [] def h(request): import httpx calls["n"] += 1 - if calls["n"] == 1: - return httpx.Response( - 429, headers={"Retry-After": "Thu, 13 Aug 2026 03:00:10 GMT"} - ) - return httpx.Response(200, json=_img_payload(_big_b64())) + return httpx.Response( + 429, headers={"Retry-After": "Thu, 13 Aug 2026 03:00:10 GMT"} + ) monkeypatch.setattr( - "windup_framework.providers.sufy._utc_now", + "windup_framework.gateway.classify._utc_now", lambda: datetime(2026, 8, 13, 3, 0, tzinfo=timezone.utc), ) - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", sleeps.append) - assert _image_provider(h).gen_image("x", []) - assert sleeps == [10.0] + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.RATE_LIMIT + assert r.retry_after_s == 10.0 def test_retry_after_clock_is_utc(): @@ -438,11 +461,11 @@ def test_retry_after_clock_is_utc(): def test_retry_after_accepts_date_without_timezone(monkeypatch): monkeypatch.setattr( - "windup_framework.providers.sufy._utc_now", + "windup_framework.gateway.classify._utc_now", lambda: datetime(2026, 8, 13, 3, 0, tzinfo=timezone.utc), ) - assert _retry_after_seconds("Thu, 13 Aug 2026 03:00:10") == 10.0 + assert retry_after_seconds("Thu, 13 Aug 2026 03:00:10") == 10.0 def test_request_path_comes_from_config_not_a_literal(): @@ -468,21 +491,22 @@ def h(request): pytest.param({"cf-ray": "8f2b1c4d5e6a7890-SJC", "server": "nginx"}, id="relayed-cf-ray"), ]) @pytest.mark.parametrize("code", [521, 522, 523]) -def test_52x_is_retried_whatever_the_edge_looks_like(monkeypatch, code, headers): +def test_52x_is_classified_unreached_in_one_call(code, headers): """判据只看码:``AI_BASE_URL`` 后面挂哪家网关不可知,靠响应头认 Cloudflare 会把真实 - 链路上的 52x 全判否(实测网关自报 ``server: APISIX``),整条重试等于不存在。 + 链路上的 52x 全判否(实测网关自报 ``server: APISIX``)。adapter 一次 POST 分类即可。 """ calls = {"n": 0} def h(request): calls["n"] += 1 - if calls["n"] <= 2: - return httpx.Response(code, headers=headers, text="Connection timed out") - return httpx.Response(200, json=_img_payload(_big_b64())) + return httpx.Response(code, headers=headers, text="Connection timed out") - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) - assert _image_provider(h).gen_image("x", []).startswith(b"\x89PNG") - assert calls["n"] == 3, "必须真的重发，而不是靠外层出图循环碰运气" + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.UNREACHED + assert not r.ok + assert r.http_status == code + assert r.maybe_billed is False @pytest.mark.parametrize("code", [520, 524]) @@ -494,103 +518,74 @@ def h(request): seen["n"] += 1 return httpx.Response(code, headers={"server": "cloudflare"}, text="ambiguous") - with pytest.raises(httpx.HTTPStatusError): - _image_provider(h).gen_image("x", []) + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") assert seen["n"] == 1, f"HTTP {code} 被重试了,会重复计费" + assert r.error_type is ModelErrorType.MAYBE_BILLED + assert r.maybe_billed is True + assert not r.ok -def test_retryable_set_excludes_the_codes_that_may_have_billed(): - """常量本身也钉一道:改集合的人不必先读懂 _post 才发现自己开了重复计费的洞。""" - from windup_framework.providers.sufy import _CLOUDFLARE_UNREACHED_STATUS - - assert _CLOUDFLARE_UNREACHED_STATUS == {521, 522, 523} - - -def test_unreached_resends_are_capped_across_the_whole_gen_image(monkeypatch): - """52x 的"没到上游"是大概率不是保证(CF 的 522 含"连上了但源站没及时确认"), - 所以可重复计费的重发次数按整次 gen_image 封顶,不跟着内外两层循环叠乘。 - """ - from windup_framework.providers.sufy import _UNREACHED_RESENDS - +def test_unreached_resends_are_capped_across_the_whole_gen_image(): + """adapter 不再连打 52x；一次 POST + UNREACHED，重发次数由 Gateway 封顶。""" calls = {"n": 0} def h(request): calls["n"] += 1 - if calls["n"] % 2: - return httpx.Response(522, headers={"server": "APISIX"}, text="timed out") - return httpx.Response(200, json={"choices": [{"message": {"content": "无图"}}]}) + return httpx.Response(522, headers={"server": "APISIX"}, text="timed out") - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) - - with pytest.raises(RuntimeError, match=r"已重发 2 次"): - _image_provider(h).gen_image("x", []) - # 预算若按 _post 调用各算一份,外层三轮就会重发 3 次而不是 2 次。 - assert calls["n"] == 2 * _UNREACHED_RESENDS + 1 + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.UNREACHED -def test_exhausted_retries_report_the_edge_fingerprint(monkeypatch): - """三次全 52x 正是最需要复盘的场景,而它唯一留下的就是这条异常文本。""" +def test_exhausted_retries_report_the_edge_fingerprint(): + """52x 复盘靠边缘指纹，不再依赖「已重发 N 次」异常文本。""" def h(request): return httpx.Response(522, headers={"server": "APISIX", "cf-ray": "8f2b-SJC"}) - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) - - with pytest.raises(RuntimeError, match=r"522.*已重发 2 次.*server=APISIX"): - _image_provider(h).gen_image("x", []) + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert r.error_type is ModelErrorType.UNREACHED + assert "server=APISIX" in r.edge_fingerprint + assert "cf-ray=8f2b-SJC" in r.edge_fingerprint -def test_rate_limit_exhaustion_also_reports_the_fingerprint(monkeypatch): - """限流与"网关连不上上游"要能一眼分开 —— 两者的处置完全不同。""" - from windup_framework.providers.sufy import _POST_TRIES - +def test_rate_limit_exhaustion_also_reports_the_fingerprint(): + """限流与「网关连不上上游」要能一眼分开。""" calls = {"n": 0} def h(request): calls["n"] += 1 return httpx.Response(429, headers={"server": "APISIX"}, text="slow down") - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", lambda _: None) - - with pytest.raises(RuntimeError, match=r"过于频繁.*连发 3 次.*server=APISIX"): - _image_provider(h).gen_image("x", []) - assert calls["n"] == _POST_TRIES - + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.RATE_LIMIT + assert "server=APISIX" in r.edge_fingerprint -def test_unreached_backoff_is_capped(monkeypatch): - """上游挂掉时不该把一个图像任务堵成长时间阻塞。""" - from windup_framework.providers.sufy import _MAX_RETRY_WAIT - sleeps: list[float] = [] +def test_unreached_backoff_is_capped(): + """Retry-After 过大时解析结果仍封顶，adapter 不 sleep。""" + from windup_framework.gateway.classify import _MAX_RETRY_WAIT def h(request): return httpx.Response(522, headers={**_CF_EDGE, "Retry-After": "9999"}) - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", sleeps.append) - - with pytest.raises(RuntimeError, match="522"): - _image_provider(h).gen_image("x", []) - assert sleeps and max(sleeps) <= _MAX_RETRY_WAIT - + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert r.error_type is ModelErrorType.UNREACHED + assert r.retry_after_s == _MAX_RETRY_WAIT -def test_worst_case_request_count_and_wait_are_bounded(monkeypatch): - """内外两层重试会叠乘,最坏情况必须是个说得出的数,而不是"看情况"。""" - from windup_framework.providers.sufy import _IMAGE_TRIES, _MAX_RETRY_WAIT, _POST_TRIES +def test_worst_case_request_count_and_wait_are_bounded(): + """adapter 一次一枪，最坏情况就是 1 次 POST。""" calls = {"n": 0} - sleeps: list[float] = [] def h(request): calls["n"] += 1 - if calls["n"] % _POST_TRIES: - return httpx.Response(429, text="slow down") - return httpx.Response(200, json={"choices": [{"message": {"content": "无图"}}]}) - - monkeypatch.setattr("windup_framework.providers.sufy.time.sleep", sleeps.append) + return httpx.Response(429, text="slow down") - with pytest.raises(RuntimeError, match="均未取得有效图"): - _image_provider(h).gen_image("x", []) - assert calls["n"] == _IMAGE_TRIES * _POST_TRIES == 9 - assert sum(sleeps) <= _IMAGE_TRIES * (_POST_TRIES - 1) * _MAX_RETRY_WAIT + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert calls["n"] == 1 + assert r.error_type is ModelErrorType.RATE_LIMIT @pytest.mark.parametrize("code", [400, 404]) @@ -602,6 +597,12 @@ def h(request): import httpx return httpx.Response(code, text='{"error":{"message":"model not found"}}') + r = _image_provider(h).submit_image("x", [], "gemini-2.5-flash-image") + assert r.error_type is ModelErrorType.UNKNOWN + assert "/models" in r.edge_fingerprint + assert r.http_status == code + assert not r.ok + with pytest.raises(RuntimeError, match=r"/models"): _image_provider(h).gen_image("x", []) @@ -656,6 +657,8 @@ def test_request_shape_is_not_configurable(): for banned in ("image_list_models", "fal_endpoints", "first_frame_field"): assert banned not in fields, f"{banned} 不该进配置，见本用例 docstring" assert {"video_model", "image_model"} <= fields + assert {"image_fallbacks", "video_fallbacks", "image_unit_cost", + "video_unit_cost_per_second", "price_version"} <= fields # ── i2v 主流程（付费路径，此前零覆盖）───────────────────────────────────────── @@ -694,12 +697,15 @@ def _video_provider(handler, **kw): poll_interval=30.0, **kw, ) - client = _httpx.Client( - base_url="https://gw.example.com/v1", - headers={"Authorization": "Bearer k"}, - transport=_httpx.MockTransport(handler), - ) - p._client = lambda: client + # submit_video 与 follow_job 各自 with _client()，必须每次返回新 client， + # 否则建单结束就会把同一实例 close 掉，跟单 GET 打到已关闭的连接。 + def make_client(): + return _httpx.Client( + base_url="https://gw.example.com/v1", + headers={"Authorization": "Bearer k"}, + transport=_httpx.MockTransport(handler), + ) + p._client = make_client return p @@ -731,6 +737,30 @@ def h(request): return h +def test_follow_job_records_poll_and_download_timings(): + p = _video_provider(_i2v_handler({}, statuses=("in_progress", "completed"))) + result = p.follow_job("job-1") + assert result.ok + assert result.poll_count == 2 + assert isinstance(result.poll_ms, int) + assert isinstance(result.download_ms, int) + + +def test_submit_video_maps_disconnect_to_unreached(): + """建单前对端拆连接:无 job_id,Gateway 才能同路重发,而不是冒新单。""" + + def h(request): + raise httpx.RemoteProtocolError("Server disconnected without sending a response") + + r = _video_provider(h).submit_video(_jpeg_first_frame(), "walk", 5, "1280x720", "kling-v2-5-turbo") + assert not r.ok + assert r.error_type is ModelErrorType.UNREACHED + assert r.http_status is None + assert r.maybe_billed is False + assert r.job_id is None + assert "disconnected" in r.edge_fingerprint + + def test_i2v_submits_polls_and_downloads(): """一条完整的付费路径：提交拿 job id → 轮询到 completed → 下载 mp4。""" seen: dict = {}